surgent 0.7.0-alpha.0

package/src/web-tools/providers/tavily.ts

@@ -0,0 +1,56 @@
+import { formatErrorMessage, normalizeFetchedContent } from "../web-fetch/helpers.js";
+import type { WebFetchResponse } from "../web-fetch/types.js";
+import { normalizeSearchResult } from "../web-search/helpers.js";
+import type { WebSearchResult } from "../web-search/types.js";
+import { isDefined } from "../../utils.js";
+import { tavily } from "@tavily/core";
+import type { WebFetchProvider, WebSearchProvider } from "./index.js";
+
+interface TavilySearchResponse {
+  results?: Array<{ title?: string; content?: string; url?: string }>;
+}
+
+interface TavilyExtractResponse {
+  failedResults?: Array<{ error?: string; url: string }>;
+  results?: Array<{ rawContent?: string; url: string }>;
+}
+
+export class TavilyProvider implements WebSearchProvider, WebFetchProvider {
+  constructor(private readonly apiKey: string) {}
+
+  async search(query: string, news: boolean, max: number): Promise<WebSearchResult[]> {
+    const client = tavily({ apiKey: this.apiKey });
+    const response = (await client.search(query, {
+      includeAnswer: false,
+      includeRawContent: false,
+      maxResults: max,
+      searchDepth: "basic",
+      topic: news ? "news" : "general",
+    })) as TavilySearchResponse;
+
+    return (response.results ?? [])
+      .map((item) =>
+        normalizeSearchResult({ description: item.content, title: item.title, url: item.url }),
+      )
+      .filter(isDefined);
+  }
+
+  async fetch(url: string): Promise<WebFetchResponse> {
+    const client = tavily({ apiKey: this.apiKey });
+    try {
+      const response = (await client.extract([url], {
+        format: "markdown",
+      })) as TavilyExtractResponse;
+      const content = normalizeFetchedContent(response.results?.[0]?.rawContent);
+
+      if (content) {
+        return { provider: "tavily", content, url };
+      }
+
+      const errorMsg = response.failedResults?.[0]?.error ?? "Tavily returned no content.";
+      return { provider: "tavily", url, error: errorMsg };
+    } catch (error) {
+      return { provider: "tavily", url, error: formatErrorMessage(error) };
+    }
+  }
+}

package/src/web-tools/settings.ts

@@ -0,0 +1,15 @@
+export const WEB_SEARCH_PROVIDERS = [
+  { name: "tavily", label: "Tavily" },
+  { name: "brave-search", label: "Brave Search" },
+  { name: "firecrawl", label: "Firecrawl" },
+] as const;
+
+export const WEB_FETCH_PROVIDERS = [
+  { name: "jina", label: "Jina", note: "optional, only helps increase rate limits" },
+  { name: "firecrawl", label: "Firecrawl" },
+  { name: "tavily", label: "Tavily" },
+] as const;
+
+export const WEB_TOOLS_PROVIDERS = [
+  ...new Map([...WEB_SEARCH_PROVIDERS, ...WEB_FETCH_PROVIDERS].map((p) => [p.name, p])).values(),
+];

package/src/web-tools/web-fetch/helpers.ts

@@ -0,0 +1,66 @@
+import { normalizeText } from "../../utils.js";
+import { parseWebFetchContent } from "./parser.js";
+import { getCacheFilePath } from "./storage.js";
+import type { WebFetchResponse } from "./types.js";
+
+export function normalizeFetchedContent(value: unknown): string {
+  return normalizeText(value).replace(/\r\n/g, "\n");
+}
+
+export function toCanonicalUrl(value: string): string {
+  const normalized = normalizeText(value);
+  try {
+    return new URL(normalized).href;
+  } catch {
+    return normalized;
+  }
+}
+
+export function isTextLikeContentType(contentType: string | null): boolean {
+  const normalized = normalizeText(contentType).toLowerCase();
+  if (!normalized) return true;
+  return (
+    normalized.startsWith("text/") ||
+    normalized.includes("json") ||
+    normalized.includes("xml") ||
+    normalized.includes("javascript") ||
+    normalized.includes("html")
+  );
+}
+
+export function formatFetchResult(result: WebFetchResponse, cacheDate: string): string {
+  const filePath = getCacheFilePath(result.url, cacheDate);
+  return `Provider: ${result.provider}\nOutput path: ${filePath}\nSections:\n${parseWebFetchContent(result.content!)}`;
+}
+
+export function formatErrorMessage(error: unknown): string {
+  if (error instanceof Error && error.message) {
+    return error.message;
+  }
+  return String(error);
+}
+
+export async function getHttpError(response: Response): Promise<string> {
+  const responseText = await response.text();
+  const body = responseText.trim();
+  if (!body) {
+    return `HTTP ${response.status}`;
+  }
+  return `HTTP ${response.status}: ${body}`;
+}
+
+export function getValidatedUrl(url: string): string {
+  const value = url.trim();
+  let parsed: URL;
+  try {
+    parsed = new URL(value);
+  } catch {
+    throw new Error(`Invalid URL: ${value}`);
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new Error(`Unsupported URL protocol: ${value}`);
+  }
+
+  return parsed.href;
+}

package/src/web-tools/web-fetch/index.ts

@@ -0,0 +1,91 @@
+import { defineTool } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { formatErrorMessage, formatFetchResult, getValidatedUrl } from "./helpers.js";
+import {
+  getCurrentCacheDate,
+  pruneExpiredCacheDirs,
+  readCachedContent,
+  writeFetchedResult,
+} from "./storage.js";
+import { WebToolsFactory } from "../providers/index.js";
+import type { WebFetchResponse } from "./types.js";
+import { WEB_FETCH_PROVIDERS } from "../settings.js";
+import { getApiKey } from "../web-login/helpers.js";
+import { renderCallText } from "../../utils.js";
+
+const webToolsFactory = new WebToolsFactory();
+
+const webFetchTool = defineTool({
+  name: "web_fetch",
+  label: "Web Fetch",
+  description: "Fetch a public URL, cache markdown locally, return file path and heading outline.",
+  promptSnippet: "Fetch a known URL. Returns metadata and heading outline.",
+  promptGuidelines: [
+    "Don't use web_fetch when relevant info is already in web_search.",
+    "Use web_fetch for known URLs, not discovery.",
+    "Use web_fetch output path when page body is needed.",
+    "Web content tends to be very big. Use grep to search for needed content in output first before read ",
+  ],
+  parameters: Type.Object({
+    url: Type.String({ description: "An HTTP(S) URL" }),
+  }),
+  async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+    const url = getValidatedUrl(params.url);
+    const cacheDate = getCurrentCacheDate();
+    const attempts: string[] = [];
+    const nativeFetch = { name: "native", label: "Native fetch" } as const;
+
+    pruneExpiredCacheDirs(cacheDate);
+    if (signal?.aborted) {
+      throw new Error("web_fetch was cancelled.");
+    }
+
+    const cached = await readCachedContent(url, cacheDate);
+    if (cached !== undefined) {
+      const result: WebFetchResponse = { provider: "native", content: cached, url };
+      return {
+        content: [{ type: "text", text: formatFetchResult(result, cacheDate) }],
+        details: result satisfies WebFetchResponse,
+      };
+    }
+
+    for (const provider of [nativeFetch, ...WEB_FETCH_PROVIDERS]) {
+      if (signal?.aborted) {
+        throw new Error("web_fetch was cancelled.");
+      }
+
+      const apiKey =
+        provider.name === "native" ? undefined : await getApiKey(ctx.modelRegistry, provider.name);
+
+      if ((provider.name === "firecrawl" || provider.name === "tavily") && !apiKey) {
+        attempts.push(`${provider.label}: not configured`);
+        continue;
+      }
+
+      try {
+        const response = await webToolsFactory.createWebFetcher(provider.name, apiKey).fetch(url);
+        if (response.error === undefined) {
+          await writeFetchedResult(url, response.content, cacheDate);
+          return {
+            content: [{ type: "text", text: formatFetchResult(response, cacheDate) }],
+            details: response satisfies WebFetchResponse,
+          };
+        }
+
+        attempts.push(`${provider.label}: ${response.error}`);
+      } catch (error) {
+        attempts.push(`${provider.label}: ${formatErrorMessage(error)}`);
+      }
+    }
+
+    throw new Error(`Web fetch failed.\n|- ${attempts.join("\n|- ")}`);
+  },
+  renderCall(args, theme, { isPartial }) {
+    return renderCallText(
+      `${theme.fg("toolTitle", "web_fetch")} ${theme.underline(theme.fg("accent", args.url))}`,
+      isPartial,
+    );
+  },
+});
+
+export default webFetchTool;

package/src/web-tools/web-fetch/parser.ts

@@ -0,0 +1,51 @@
+export function parseWebFetchContent(content: string): string {
+  const lines = content.split("\n");
+  const headings: Array<{ endLine?: number; level: number; line: number; title: string }> = [];
+  let fenceMarker: string | undefined;
+
+  for (const [index, line] of lines.entries()) {
+    const fenceMatch = line.match(/^(```+|~~~+)/);
+    if (fenceMatch) {
+      const marker = fenceMatch[1]!;
+      if (!fenceMarker) {
+        fenceMarker = marker[0]!.repeat(marker.length);
+      } else if (line.startsWith(fenceMarker)) {
+        fenceMarker = undefined;
+      }
+      continue;
+    }
+
+    if (fenceMarker) {
+      continue;
+    }
+
+    const headingMatch = line.match(/^(#{1,5})[ \t]+(.+?)(?:[ \t]+#+[ \t]*)?$/);
+    if (!headingMatch) {
+      continue;
+    }
+
+    headings.push({
+      level: headingMatch[1]!.length,
+      line: index + 1,
+      title: headingMatch[2]!.trim(),
+    });
+  }
+
+  if (headings.length === 0) {
+    return "";
+  }
+
+  for (const [index, heading] of headings.entries()) {
+    const nextSiblingOrAncestor = headings
+      .slice(index + 1)
+      .find((candidate) => candidate.level <= heading.level);
+    heading.endLine = nextSiblingOrAncestor ? nextSiblingOrAncestor.line - 1 : lines.length;
+  }
+
+  return headings
+    .map((heading) => {
+      const indent = "  ".repeat(heading.level - 1);
+      return `${indent}${"#".repeat(heading.level)} ${heading.title} (L${heading.line}-${heading.endLine})`;
+    })
+    .join("\n");
+}

package/src/web-tools/web-fetch/storage.ts

@@ -0,0 +1,65 @@
+import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { dirname, join } from "node:path";
+import { toCanonicalUrl } from "./helpers.js";
+import { getPiPath } from "../../utils.js";
+
+export function getCurrentCacheDate(): string {
+  const now = new Date();
+  const year = String(now.getFullYear());
+  const month = String(now.getMonth() + 1).padStart(2, "0");
+  const day = String(now.getDate()).padStart(2, "0");
+  return `${year}-${month}-${day}`;
+}
+
+export function getCacheFilePath(url: string, date = getCurrentCacheDate()): string {
+  const canonicalUrl = toCanonicalUrl(url);
+  const fileName = `${createHash("md5").update(canonicalUrl).digest("hex")}.md`;
+  return getPiPath("web", "global", date, fileName);
+}
+
+export async function pruneExpiredCacheDirs(today = getCurrentCacheDate()) {
+  const cacheRoot = getPiPath("web");
+  try {
+    const entries = await readdir(cacheRoot, { withFileTypes: true });
+    await Promise.all(
+      entries
+        .filter((entry) => entry.isDirectory() && entry.name !== today)
+        .map((entry) => rm(join(cacheRoot, entry.name), { force: true, recursive: true })),
+    );
+  } catch (error) {
+    if (isMissingFileError(error)) return;
+    throw error;
+  }
+}
+
+export async function readCachedContent(
+  url: string,
+  date = getCurrentCacheDate(),
+): Promise<string | undefined> {
+  try {
+    return await readFile(getCacheFilePath(url, date), "utf8");
+  } catch (error) {
+    if (isMissingFileError(error)) return;
+    throw error;
+  }
+}
+
+export async function writeFetchedResult(
+  url: string,
+  content: string,
+  date = getCurrentCacheDate(),
+) {
+  const filePath = getCacheFilePath(url, date);
+  await mkdir(dirname(filePath), { recursive: true });
+  await writeFile(filePath, content, "utf8");
+}
+
+function isMissingFileError(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    (error as { code?: unknown }).code === "ENOENT"
+  );
+}

package/src/web-tools/web-fetch/types.ts

@@ -0,0 +1,8 @@
+import type { WEB_FETCH_PROVIDERS } from "../settings.js";
+
+export type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number]["name"] | "native";
+
+export type WebFetchResponse = {
+  url: string;
+  provider: WebFetchProviderId;
+} & ({ content: string; error?: never } | { error: string; content?: never });

package/src/web-tools/web-login/helpers.ts

@@ -0,0 +1,79 @@
+import type { CredentialStore } from "@earendil-works/pi-ai";
+import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
+import { WEB_TOOLS_PROVIDERS } from "../settings.js";
+import type { WebToolsProvider, WebToolsProviderId } from "./types.js";
+
+export function findWebToolsProvider(input: string): WebToolsProvider | undefined {
+  const normalized = input.trim().toLowerCase();
+  if (!normalized) return;
+  return WEB_TOOLS_PROVIDERS.find((provider) => provider.name === normalized);
+}
+
+export function getWebToolsProviderOptions(): string[] {
+  return WEB_TOOLS_PROVIDERS.map((provider) => provider.label);
+}
+
+export function getWebToolsProviderByLabel(label: string): WebToolsProvider | undefined {
+  return WEB_TOOLS_PROVIDERS.find((provider) => provider.label === label);
+}
+
+function maskApiKey(key: string): string {
+  if (key.length <= 4) return key;
+  return key.slice(0, 4) + "*".repeat(key.length - 4);
+}
+
+function getCredentialStore(modelRegistry: ModelRegistry): CredentialStore {
+  return (
+    (modelRegistry as unknown as { runtime: unknown }).runtime as {
+      credentials: CredentialStore;
+    }
+  ).credentials;
+}
+
+export async function getApiKey(
+  modelRegistry: ModelRegistry,
+  providerId: WebToolsProviderId,
+): Promise<string | undefined> {
+  const credential = await getCredentialStore(modelRegistry).read(providerId);
+  return credential?.type === "api_key" ? credential.key : undefined;
+}
+
+export async function formatProviderStatus(
+  modelRegistry: ModelRegistry,
+  provider: WebToolsProvider,
+): Promise<string> {
+  const apiKey = await getApiKey(modelRegistry, provider.name);
+  return apiKey
+    ? `${provider.label} (configured — ${maskApiKey(apiKey)})`
+    : `${provider.label} (not configured)`;
+}
+
+export async function setApiKey(
+  modelRegistry: ModelRegistry,
+  providerId: WebToolsProviderId,
+  apiKey: string,
+) {
+  await getCredentialStore(modelRegistry).modify(providerId, async () => ({
+    type: "api_key",
+    key: apiKey,
+  }));
+}
+
+export async function clearApiKey(modelRegistry: ModelRegistry, providerId: WebToolsProviderId) {
+  await getCredentialStore(modelRegistry).delete(providerId);
+}
+
+export function getArgumentCompletions(prefix: string) {
+  const normalized = prefix.trim().toLowerCase();
+  const matches = WEB_TOOLS_PROVIDERS.filter((provider) => provider.name.startsWith(normalized));
+
+  if (matches.length === 0) {
+    return null;
+  }
+
+  return matches.map((provider) => ({ value: provider.name, label: provider.label }));
+}
+
+export function getSupportedProviderNames(): string {
+  return WEB_TOOLS_PROVIDERS.map((provider) => provider.name).join(", ");
+}

package/src/web-tools/web-login/index.ts

@@ -0,0 +1,100 @@
+import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import {
+  clearApiKey,
+  findWebToolsProvider,
+  formatProviderStatus,
+  getApiKey,
+  getSupportedProviderNames,
+  getWebToolsProviderByLabel,
+  getWebToolsProviderOptions,
+  setApiKey,
+} from "./helpers.js";
+import type { WebToolsProvider } from "./types.js";
+
+async function selectProvider(ctx: ExtensionCommandContext): Promise<WebToolsProvider | undefined> {
+  const selected = await ctx.ui.select(
+    "Configure web provider authentication",
+    getWebToolsProviderOptions(),
+  );
+
+  if (!selected) return;
+  return getWebToolsProviderByLabel(selected);
+}
+
+async function chooseAction(
+  ctx: ExtensionCommandContext,
+  provider: WebToolsProvider,
+): Promise<"save" | "clear" | undefined> {
+  if (await getApiKey(ctx.modelRegistry, provider.name)) {
+    const selected = await ctx.ui.select(`${provider.label} credentials`, [
+      "Save new API key",
+      "Clear saved API key",
+    ]);
+    if (!selected) return;
+    return selected === "Clear saved API key" ? "clear" : "save";
+  }
+  return "save";
+}
+
+async function saveProviderKey(ctx: ExtensionCommandContext, provider: WebToolsProvider) {
+  const note = provider.name === "jina" ? ` (${provider.note})` : "";
+
+  if (await getApiKey(ctx.modelRegistry, provider.name)) {
+    const replace = await ctx.ui.confirm(
+      `${provider.label} API key`,
+      `${provider.label} already has a saved API key. Replace it?`,
+    );
+    if (!replace) return;
+  }
+
+  const apiKey = await ctx.ui.input(
+    `${provider.label} API key${note}`,
+    `Paste your ${provider.label} API key`,
+  );
+
+  if (!apiKey) {
+    ctx.ui.notify(`No ${provider.label} API key was saved`, "warning");
+    return;
+  }
+
+  await setApiKey(ctx.modelRegistry, provider.name, apiKey.trim());
+  ctx.ui.notify(`Saved ${provider.label} API key`, "info");
+}
+
+async function clearProviderKey(ctx: ExtensionCommandContext, provider: WebToolsProvider) {
+  const confirmed = await ctx.ui.confirm(
+    `Clear ${provider.label} API key`,
+    `Remove the saved ${provider.label} API key from shared auth storage?`,
+  );
+  if (!confirmed) return;
+
+  await clearApiKey(ctx.modelRegistry, provider.name);
+  ctx.ui.notify(`Cleared ${provider.label} API key`, "info");
+}
+
+export default async function webLoginCommand(args: string, ctx: ExtensionCommandContext) {
+  const arg = args.trim();
+  const provider = arg ? findWebToolsProvider(arg) : await selectProvider(ctx);
+
+  if (!provider) {
+    if (arg) {
+      ctx.ui.notify(
+        `Unknown provider \"${arg}\". Supported providers: ${getSupportedProviderNames()}.`,
+        "error",
+      );
+    }
+    return;
+  }
+
+  ctx.ui.notify(await formatProviderStatus(ctx.modelRegistry, provider), "info");
+
+  const action = await chooseAction(ctx, provider);
+
+  if (!action) return;
+  if (action === "clear") {
+    await clearProviderKey(ctx, provider);
+    return;
+  }
+
+  await saveProviderKey(ctx, provider);
+}

package/src/web-tools/web-login/types.ts

@@ -0,0 +1,4 @@
+import { WEB_TOOLS_PROVIDERS } from "../settings.js";
+
+export type WebToolsProvider = (typeof WEB_TOOLS_PROVIDERS)[number];
+export type WebToolsProviderId = WebToolsProvider["name"];

package/src/web-tools/web-search/helpers.ts

@@ -0,0 +1,36 @@
+import { normalizeText } from "../../utils.js";
+import type { WebSearchResult, WebSearchResultInput } from "./types.js";
+
+export function normalizeSearchResult(item: WebSearchResultInput): WebSearchResult | undefined {
+  const title = normalizeText(item.title);
+  const url = normalizeText(item.url);
+  if (!title || !url) return;
+  return { description: normalizeText(item.description), title, url };
+}
+
+export function joinSnippets(snippets: string[] | undefined): string {
+  if (!Array.isArray(snippets)) {
+    return "";
+  }
+
+  return snippets
+    .filter((snippet) => typeof snippet === "string" && snippet.trim().length > 0)
+    .join(" ")
+    .trim();
+}
+
+export function formatErrorMessage(error: unknown): string {
+  if (error instanceof Error && error.message) {
+    return error.message;
+  }
+  return String(error);
+}
+
+export async function getHttpError(response: Response): Promise<string> {
+  const responseText = await response.text();
+  const body = responseText.trim();
+  if (!body) {
+    return `HTTP ${response.status}`;
+  }
+  return `HTTP ${response.status}: ${body}`;
+}

package/src/web-tools/web-search/index.ts

@@ -0,0 +1,98 @@
+import { defineTool } from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import { Type } from "typebox";
+import { WEB_SEARCH_PROVIDERS } from "../settings.js";
+import { WebToolsFactory } from "../providers/index.js";
+import { formatErrorMessage } from "./helpers.js";
+import { getApiKey } from "../web-login/helpers.js";
+import type { WebSearchResult } from "./types.js";
+import { renderCallText, renderResultText } from "../../utils.js";
+
+const webToolsFactory = new WebToolsFactory();
+
+const webSearchTool = defineTool({
+  name: "web_search",
+  label: "Web Search",
+  description: "Search web or news. Returns ranked title/summary/url results.",
+  promptSnippet: "Search web or news. Returns ranked results.",
+  promptGuidelines: [
+    "Use web_search for external/unavailable info.",
+    "Use web_search with news=true for recent reporting; default is web search.",
+    "Use web_search with small max first; results are re-ranked best-first.",
+  ],
+  parameters: Type.Object({
+    query: Type.String({ description: "Search query" }),
+    max: Type.Optional(
+      Type.Number({ description: "Max results (default: 5)", minimum: 1, maximum: 10 }),
+    ),
+    news: Type.Optional(Type.Boolean({ description: "Search news when true; web when false" })),
+  }),
+  async execute(_toolCallId, { query, max = 5, news = false }, signal, _onUpdate, ctx) {
+    const trimmed = query.trim();
+    if (!trimmed) {
+      throw new Error("Query must not be empty.");
+    }
+
+    const attempts: string[] = [];
+    let anyConfiguredProvider = false;
+
+    for (const provider of WEB_SEARCH_PROVIDERS) {
+      if (signal?.aborted) {
+        throw new Error("web_search was cancelled.");
+      }
+
+      const apiKey = await getApiKey(ctx.modelRegistry, provider.name);
+
+      if (!apiKey) {
+        attempts.push(`${provider.label}: not configured`);
+        continue;
+      }
+
+      anyConfiguredProvider = true;
+
+      try {
+        const results = await webToolsFactory
+          .createWebSearcher(provider.name, apiKey)
+          .search(trimmed, news, max);
+
+        if (results.length === 0) {
+          attempts.push(`${provider.label}: returned no results`);
+          continue;
+        }
+
+        return {
+          content: [{ type: "text", text: JSON.stringify(results, null, 2) }],
+          details: { results } satisfies { results: WebSearchResult[] },
+        };
+      } catch (error) {
+        attempts.push(`${provider.label}: ${formatErrorMessage(error)}`);
+      }
+    }
+
+    if (!anyConfiguredProvider) {
+      throw new Error(
+        "No configured web search providers available. Use /web-login to configure Tavily, Brave Search, or Firecrawl.",
+      );
+    }
+
+    throw new Error(
+      `Web search failed across all configured providers.\n|- ${attempts.join("\n|- ")}`,
+    );
+  },
+  renderCall(args, theme, { isPartial }) {
+    return renderCallText(
+      `${theme.fg("toolTitle", "web_search")} ${theme.fg("accent", args.query)}`,
+      isPartial,
+    );
+  },
+  renderResult(result, { expanded, isPartial }, theme) {
+    if (isPartial) {
+      return new Text(theme.fg("warning", "Searching..."), 0, 0);
+    }
+    const details = result.details as { results: WebSearchResult[] } | undefined;
+    const text = (details?.results.map((item) => item.url) ?? ["No search results"]).join("\n");
+    return renderResultText(text, theme, expanded);
+  },
+});
+
+export default webSearchTool;