webfox 4.10.0 → 4.11.0

package/README.md

@@ -11,18 +11,6 @@ Search the web, extract pages, get grounded answers, and run deep research with
 one command: `web`. Choose your providers, bring your API keys, and use the same
 configuration in your terminal, TypeScript applications, and pi.
 
-> [!IMPORTANT]
-> **Coming from pi-web-providers?** Webfox continues the project as **v4.0.0**.
-> Replace the old Pi package:
->
-> ```sh
-> pi remove npm:pi-web-providers
-> pi install npm:webfox
-> ```
->
-> Recreate your provider defaults in the [Webfox configuration](./docs/reference.md#configuration)
-> and restart pi. The old `~/.pi/agent/web-providers.json` settings aren't imported.
-
 ## 🚀 Installation
 
 - **Terminal:** Install the `web` command with [Homebrew](https://github.com/mavam/homebrew-tap):
@@ -152,6 +140,7 @@ Use different providers for different tasks:
 | [Perplexity] |   ✔︎    |          |   ✔︎    |    ✔︎     |
 | [SerpBase]   |   ✔︎    |          |        |          |
 | [Serper]     |   ✔︎    |          |        |          |
+| [Sofya]      |   ✔︎    |    ✔︎     |   ✔︎    |    ✔︎     |
 | [Tavily]     |   ✔︎    |    ✔︎     |        |          |
 | [Valyu]      |   ✔︎    |    ✔︎     |   ✔︎    |    ✔︎     |
 | [You.com]    |   ✔︎    |          |        |          |
@@ -170,6 +159,7 @@ Use different providers for different tasks:
 [Perplexity]: ./docs/provider.md#perplexity
 [SerpBase]: ./docs/provider.md#serpbase
 [Serper]: ./docs/provider.md#serper
+[Sofya]: ./docs/provider.md#sofya
 [Tavily]: ./docs/provider.md#tavily
 [Valyu]: ./docs/provider.md#valyu
 [You.com]: ./docs/provider.md#youcom

package/dist/{adapter-UIFBJKKF.js → adapter-44ZBIK47.js}

@@ -2,8 +2,9 @@ import {
   trimSnippet
 } from "./chunk-FSTCLBSF.js";
 import {
+  omittedContents,
   orderedContents
-} from "./chunk-YZXM4U2C.js";
+} from "./chunk-TYJ6RHKB.js";
 import "./chunk-U4BLULLV.js";
 
 // src/providers/tavily/adapter.ts
@@ -20,7 +21,7 @@ var tavilyImplementation = {
     return {
       provider: "tavily",
       results: response.results.slice(0, maxResults).map((result) => ({
-        title: result.title || result.url || "Untitled",
+        title: result.title,
         url: result.url || "",
         snippet: trimSnippet(result.content ?? result.rawContent),
         score: typeof result.score === "number" ? result.score : void 0,
@@ -57,10 +58,7 @@ var tavilyImplementation = {
             error: failedResult.error || "Content extraction failed."
           };
         }
-        return {
-          url,
-          error: "No content returned for this URL."
-        };
+        return omittedContents(url);
       })
     };
   }

package/dist/{adapter-HTMXAJTU.js → adapter-57MELLYI.js}

@@ -4,7 +4,7 @@ import {
 } from "./chunk-FSTCLBSF.js";
 import {
   orderedContents
-} from "./chunk-YZXM4U2C.js";
+} from "./chunk-TYJ6RHKB.js";
 import {
   asWebfoxError,
   httpError
@@ -36,7 +36,7 @@ async function searchOllama(query, maxResults, config, context) {
   return {
     provider: "ollama",
     results: results.slice(0, clampMaxResults(maxResults)).map((result) => ({
-      title: result.title || result.url || "Untitled",
+      title: result.title,
       url: result.url ?? "",
       snippet: trimSnippet(result.content)
     }))
@@ -69,12 +69,6 @@ async function fetchOllamaContents(urls, config, context) {
           }
           const data = await response.json();
           const content = normalizeContentText(data.content);
-          if (!content) {
-            return {
-              url,
-              error: "No content returned for this URL."
-            };
-          }
           const metadata = buildFetchMetadata(data);
           return {
             url,

package/dist/{adapter-VMMS4YQG.js → adapter-BZO5UEI6.js}

@@ -1,6 +1,6 @@
 import {
   executeAsyncResearch
-} from "./chunk-EF4VMDZL.js";
+} from "./chunk-LDVDVA4Z.js";
 import "./chunk-I2UIACY3.js";
 import "./chunk-WCTOYZVB.js";
 import {
@@ -315,23 +315,11 @@ function ensureCompletedResponse(response, operation) {
   throw new Error(`${operation} did not complete (status: ${status})`);
 }
 function formatResponseOutput(response, operation) {
-  const lines = [];
-  lines.push(
-    response.output_text?.trim() || `OpenAI ${operation} completed without textual output.`
-  );
   const citations = extractUrlCitations(response);
-  if (citations.length > 0) {
-    lines.push("");
-    lines.push("Sources:");
-    for (const [index, citation] of citations.entries()) {
-      lines.push(`${index + 1}. ${citation.title}`);
-      lines.push(`   ${citation.url}`);
-    }
-  }
   return {
     provider: "openai",
-    text: lines.join("\n").trimEnd(),
-    itemCount: citations.length,
+    text: response.output_text ?? "",
+    sources: citations.map(({ title, url }) => ({ title, url })),
     metadata: {
       responseId: response.id,
       model: response.model,

package/dist/{adapter-OK2PTP6R.js → adapter-D4EXZEDL.js}

@@ -38,7 +38,6 @@ var perplexityImplementation = {
       config,
       context,
       DEFAULT_ANSWER_MODEL,
-      "Answer",
       options
     );
   },
@@ -48,12 +47,11 @@ var perplexityImplementation = {
       config,
       context,
       DEFAULT_RESEARCH_MODEL,
-      "Research",
       options
     );
   }
 };
-async function runSilentForegroundChatTool(input, config, context, fallbackModel, label, options) {
+async function runSilentForegroundChatTool(input, config, context, fallbackModel, options) {
   const client = createClient(config);
   const request = {
     ...options ?? {},
@@ -67,23 +65,13 @@ async function runSilentForegroundChatTool(input, config, context, fallbackModel
   );
   const content = extractMessageText(response.choices[0]?.message?.content);
   const sources = dedupeSources(extractSources(response));
-  const lines = [];
-  lines.push(content || `No ${label.toLowerCase()} returned.`);
-  if (sources.length > 0) {
-    lines.push("");
-    lines.push("Sources:");
-    for (const [index, source] of sources.entries()) {
-      lines.push(`${index + 1}. ${source.title}`);
-      lines.push(`   ${source.url}`);
-    }
-  }
   return {
     provider: "perplexity",
-    text: lines.join("\n").trimEnd(),
-    itemCount: sources.length
+    text: content,
+    sources
   };
 }
-async function runStreamingForegroundChatTool(input, config, context, fallbackModel, label, options) {
+async function runStreamingForegroundChatTool(input, config, context, fallbackModel, options) {
   const client = createClient(config);
   const request = {
     ...options ?? {},
@@ -106,21 +94,11 @@ async function runStreamingForegroundChatTool(input, config, context, fallbackMo
     }
     sources.push(...extractSources(chunk));
   }
-  const finalText = partialText.trim() || extractMessageText(lastChunk?.choices?.[0]?.message?.content) || `No ${label.toLowerCase()} returned.`;
-  const dedupedSources = dedupeSources(sources);
-  const lines = [finalText];
-  if (dedupedSources.length > 0) {
-    lines.push("");
-    lines.push("Sources:");
-    for (const [index, source] of dedupedSources.entries()) {
-      lines.push(`${index + 1}. ${source.title}`);
-      lines.push(`   ${source.url}`);
-    }
-  }
+  const finalText = partialText.trim() || extractMessageText(lastChunk?.choices?.[0]?.message?.content);
   return {
     provider: "perplexity",
-    text: lines.join("\n").trimEnd(),
-    itemCount: dedupedSources.length
+    text: finalText,
+    sources: dedupeSources(sources)
   };
 }
 function createClient(config) {
@@ -182,32 +160,22 @@ function extractDeltaText(content) {
 }
 function dedupeSources(sources) {
   const seen = /* @__PURE__ */ new Set();
-  const unique = [];
-  for (const source of sources) {
-    const title = source.title.trim() || source.url.trim() || "Untitled";
-    const url = source.url.trim();
-    if (!url) continue;
-    const key = `${title.toLowerCase()}::${url.toLowerCase()}`;
-    if (seen.has(key)) continue;
+  return sources.filter((source) => {
+    const key = `${source.title?.trim().toLowerCase() ?? ""}::${source.url?.trim().toLowerCase()}`;
+    if (seen.has(key)) return false;
     seen.add(key);
-    unique.push({ title, url });
-  }
-  return unique;
+    return true;
+  });
 }
 function extractSources(response) {
-  const searchResults = response.search_results?.flatMap((result) => {
-    const url = result.url?.trim() ?? "";
-    if (!url) {
-      return [];
-    }
-    return [{ title: result.title?.trim() ?? url, url }];
-  }) ?? [];
+  const searchResults = response.search_results?.flatMap(
+    (result) => result.url?.trim() ? [{ title: result.title ?? void 0, url: result.url }] : []
+  ) ?? [];
   if (searchResults.length > 0) {
     return searchResults;
   }
   return response.citations?.flatMap((citation) => {
-    const url = citation?.trim() ?? "";
-    return url ? [{ title: url, url }] : [];
+    return citation?.trim() ? [{ url: citation }] : [];
   }) ?? [];
 }
 function buildRequestOptions(context) {

package/dist/{adapter-G3I7JEO5.js → adapter-DJCTEEOA.js}

@@ -3,8 +3,9 @@ import {
   trimSnippet
 } from "./chunk-FSTCLBSF.js";
 import {
+  omittedContents,
   orderedContents
-} from "./chunk-YZXM4U2C.js";
+} from "./chunk-TYJ6RHKB.js";
 import "./chunk-U4BLULLV.js";
 
 // src/providers/parallel/adapter.ts
@@ -56,10 +57,7 @@ var parallelImplementation = {
         return error ? {
           url,
           error: formatJson(error)
-        } : {
-          url,
-          error: "No content returned for this URL."
-        };
+        } : omittedContents(url);
       })
     };
   }

package/dist/{adapter-764OQFKY.js → adapter-DLNMW34W.js}

@@ -1,6 +1,6 @@
 import {
   executeAsyncResearch
-} from "./chunk-EF4VMDZL.js";
+} from "./chunk-LDVDVA4Z.js";
 import "./chunk-I2UIACY3.js";
 import {
   sleep,
@@ -11,8 +11,9 @@ import {
   trimSnippet
 } from "./chunk-FSTCLBSF.js";
 import {
+  omittedContents,
   orderedContents
-} from "./chunk-YZXM4U2C.js";
+} from "./chunk-TYJ6RHKB.js";
 import "./chunk-U4BLULLV.js";
 
 // src/providers/valyu/adapter.ts
@@ -73,10 +74,7 @@ var valyuImplementation = {
       answers: urls.map((url) => {
         const result = resultsByUrl.get(url) ?? (urls.length === 1 && finalResponse.results?.length === 1 ? finalResponse.results[0] : void 0);
         if (!result) {
-          return {
-            url,
-            error: "No content returned for this URL."
-          };
+          return omittedContents(url);
         }
         if (result.status === "failed" || result.status === "not_found") {
           return {
@@ -105,22 +103,10 @@ var valyuImplementation = {
         "error" in response && typeof response.error === "string" ? response.error : "answer failed"
       );
     }
-    const lines = [];
-    const contents = typeof response.contents === "string" ? response.contents : formatJson(response.contents);
-    lines.push(contents);
-    const sources = response.search_results ?? [];
-    if (sources.length > 0) {
-      lines.push("");
-      lines.push("Sources:");
-      for (const [index, result] of sources.entries()) {
-        lines.push(`${index + 1}. ${result.title}`);
-        lines.push(`   ${result.url}`);
-      }
-    }
     return {
       provider: "valyu",
-      text: lines.join("\n").trimEnd(),
-      itemCount: sources.length
+      text: typeof response.contents === "string" ? response.contents : response.contents == null ? "" : formatJson(response.contents),
+      sources: response.search_results ?? []
     };
   },
   async research(input, config, context, options) {
@@ -155,25 +141,12 @@ var valyuImplementation = {
       throw new Error(result.error || "deep research failed");
     }
     if (result.status === "completed") {
-      const lines = [];
-      lines.push(
-        typeof result.output === "string" ? result.output : result.output ? formatJson(result.output) : "Valyu deep research completed without textual output."
-      );
-      const sources = result.sources ?? [];
-      if (sources.length > 0) {
-        lines.push("");
-        lines.push("Sources:");
-        for (const [index, source] of sources.entries()) {
-          lines.push(`${index + 1}. ${source.title}`);
-          lines.push(`   ${source.url}`);
-        }
-      }
       return {
         status: "completed",
         output: {
           provider: "valyu",
-          text: lines.join("\n").trimEnd(),
-          itemCount: sources.length
+          text: typeof result.output === "string" ? result.output : result.output ? formatJson(result.output) : "",
+          sources: result.sources ?? []
         }
       };
     }

package/dist/{adapter-HRI5ZMGK.js → adapter-GFXYSHKP.js}

@@ -5,11 +5,10 @@ import {
   trimSnippet
 } from "./chunk-FSTCLBSF.js";
 import {
+  omittedContents,
   orderedContents
-} from "./chunk-YZXM4U2C.js";
-import {
-  WebfoxError
-} from "./chunk-U4BLULLV.js";
+} from "./chunk-TYJ6RHKB.js";
+import "./chunk-U4BLULLV.js";
 
 // src/providers/exa/adapter.ts
 var DEFAULT_BASE_URL = "https://api.exa.ai";
@@ -28,7 +27,7 @@ var exaImplementation = {
     return {
       provider: "exa",
       results: (response.results ?? []).slice(0, maxResults).map((result) => ({
-        title: String(result.title ?? result.url ?? "Untitled"),
+        title: typeof result.title === "string" ? result.title : void 0,
         url: String(result.url ?? ""),
         snippet: trimSnippet(
           typeof result.text === "string" ? result.text : Array.isArray(result.highlights) ? result.highlights.join(" ") : typeof result.summary === "string" ? result.summary : ""
@@ -50,10 +49,7 @@ var exaImplementation = {
       answers: urls.map((url, index) => {
         const result = results[index];
         if (!result) {
-          return {
-            url,
-            error: "No content returned for this URL."
-          };
+          return omittedContents(url);
         }
         return {
           url: typeof result.url === "string" ? result.url : url,
@@ -79,25 +75,10 @@ var exaImplementation = {
       },
       context
     );
-    const lines = [];
-    lines.push(
-      typeof response.answer === "string" ? response.answer : formatJson(response.answer)
-    );
-    const citations = response.citations ?? [];
-    if (citations.length > 0) {
-      lines.push("");
-      lines.push("Sources:");
-      for (const [index, citation] of citations.entries()) {
-        lines.push(
-          `${index + 1}. ${String(citation.title ?? citation.url ?? "Untitled")}`
-        );
-        lines.push(`   ${String(citation.url ?? "")}`);
-      }
-    }
     return {
       provider: "exa",
-      text: lines.join("\n").trimEnd(),
-      itemCount: citations.length
+      text: typeof response.answer === "string" ? response.answer : response.answer == null ? "" : formatJson(response.answer),
+      sources: response.citations ?? []
     };
   },
   async research(input, config, context, options) {
@@ -117,25 +98,10 @@ var exaImplementation = {
       context
     );
     const content = result.output?.content;
-    if (typeof content !== "string" || !content.trim())
-      throw new WebfoxError(
-        "PROVIDER_FAILURE",
-        "Exa returned no research report."
-      );
-    const sources = result.results ?? [];
-    const lines = [content.trim()];
-    if (sources.length) {
-      lines.push("", "Sources:");
-      for (const [index, source] of sources.entries())
-        lines.push(
-          `${index + 1}. ${source.title || source.url}`,
-          `   ${source.url}`
-        );
-    }
     return {
       provider: "exa",
-      text: lines.join("\n"),
-      itemCount: sources.length
+      text: typeof content === "string" ? content : "",
+      sources: result.results ?? []
     };
   }
 };

package/dist/{adapter-2UVAARTV.js → adapter-GJOUFE2U.js}

@@ -67,7 +67,7 @@ function joinUrl(baseUrl) {
 function collectResults(results, section, searchMetadata) {
   return array(results).map((entry) => asRecord(entry)).filter((entry) => Object.keys(entry).length > 0).map((entry) => {
     const url = string(entry.url) ?? "";
-    const title = string(entry.title) ?? string(entry.name) ?? (url || "Untitled");
+    const title = string(entry.title) ?? string(entry.name);
     return {
       title,
       url,

package/dist/{adapter-C2VZHFUB.js → adapter-MZ7USB6Q.js}

@@ -334,10 +334,9 @@ function toSearchResult(entry, searchContext, mode) {
     record.query,
     record.value,
     user?.name,
-    formatReviewTitle(record, user),
-    resultUrl
-  ) ?? "Untitled";
-  const url = resultUrl || (mode === "autocomplete" ? toGoogleSearchUrl(title) : "");
+    formatReviewTitle(record, user)
+  );
+  const url = resultUrl || (mode === "autocomplete" && title ? toGoogleSearchUrl(title) : "");
   const snippet = trimSnippet(
     firstNonEmptyString(
       record.snippet,

package/dist/adapter-ODDFBGJG.js

@@ -0,0 +1,167 @@
+import {
+  joinUrl,
+  requestJson,
+  trimSnippet
+} from "./chunk-FSTCLBSF.js";
+import {
+  orderedContents
+} from "./chunk-TYJ6RHKB.js";
+import {
+  WebfoxError,
+  asWebfoxError
+} from "./chunk-U4BLULLV.js";
+
+// src/providers/sofya/adapter.ts
+var DEFAULT_BASE_URL = "https://sofya.co";
+var MAX_RESULTS = 20;
+var adapter = {
+  async search(request, config, context) {
+    const maxResults = Math.max(
+      1,
+      Math.min(MAX_RESULTS, Math.trunc(request.maxResults || 0))
+    );
+    const payload = await sofyaRequest(config, context, "v1/search", {
+      ...asRecord(request.options),
+      query: request.query,
+      max_results: maxResults
+    });
+    const entries = payload.results;
+    if (!Array.isArray(entries)) throw invalidResponse("search");
+    const results = entries.map(toSearchResult).filter((result) => result !== void 0);
+    return { provider: "sofya", results: results.slice(0, maxResults) };
+  },
+  async contents(request, config, context) {
+    const answers = await Promise.all(
+      request.urls.map(async (url) => {
+        try {
+          const payload = await sofyaRequest(config, context, "v1/fetch", {
+            ...asRecord(request.options),
+            urls: [url]
+          });
+          const result = Array.isArray(payload.results) ? asRecord(payload.results[0]) : void 0;
+          if (!result) throw invalidResponse("fetch");
+          if (result.success === false) {
+            return { url, error: text(result.error) ?? "Fetch failed." };
+          }
+          const {
+            url: finalUrl,
+            content,
+            success: _s,
+            error: _e,
+            ...rest
+          } = result;
+          const metadata = Object.fromEntries(
+            Object.entries(rest).filter(
+              ([, value]) => value !== null && value !== ""
+            )
+          );
+          return {
+            url: text(finalUrl) ?? url,
+            content: typeof content === "string" ? content : "",
+            ...Object.keys(metadata).length > 0 ? { metadata } : {}
+          };
+        } catch (error) {
+          return { url, error: asWebfoxError(error).toJSON() };
+        }
+      })
+    );
+    return orderedContents({ provider: "sofya", answers });
+  },
+  async answer(request, config, context) {
+    const payload = await sofyaRequest(config, context, "v1/search", {
+      ...asRecord(request.options),
+      query: request.query,
+      include_answer: true
+    });
+    const sources = (Array.isArray(payload.results) ? payload.results : []).map(toSearchResult).filter((result) => result !== void 0).map(({ title, url }) => ({ title, url }));
+    return output(payload.answer, sources, queryMetadata(payload));
+  },
+  async research(request, config, context) {
+    context.onProgress?.("Sofya is researching and writing the report.");
+    const payload = await sofyaRequest(config, context, "v1/research", {
+      ...asRecord(request.options),
+      query: request.input
+    });
+    const sources = (Array.isArray(payload.sources) ? payload.sources : []).map(asRecord).filter((source) => source !== void 0).map((source) => ({ title: text(source.title), url: text(source.url) }));
+    return output(
+      payload.report,
+      sources,
+      Array.isArray(payload.sub_queries) ? { sub_queries: payload.sub_queries } : {}
+    );
+  }
+};
+async function sofyaRequest(config, context, path, body) {
+  const apiKey = config.credentials?.api;
+  if (!apiKey) throw new Error("Sofya is missing an API key");
+  const payload = asRecord(
+    await requestJson(
+      joinUrl(config.baseUrl ?? DEFAULT_BASE_URL, path),
+      {
+        method: "POST",
+        headers: { Authorization: `Bearer ${apiKey}` },
+        body,
+        signal: context.signal
+      },
+      describeError
+    )
+  );
+  if (!payload) throw invalidResponse(path.replace("v1/", ""));
+  return payload;
+}
+function output(body, sources, metadata) {
+  return {
+    provider: "sofya",
+    text: typeof body === "string" ? body : "",
+    sources,
+    ...Object.keys(metadata).length > 0 ? { metadata } : {}
+  };
+}
+function queryMetadata(payload) {
+  return Object.fromEntries(
+    ["altered_query", "relaxed_query"].flatMap((key) => {
+      const value = text(payload[key]);
+      return value ? [[key, value]] : [];
+    })
+  );
+}
+function invalidResponse(endpoint) {
+  return new WebfoxError(
+    "PROVIDER_FAILURE",
+    `Sofya returned an invalid ${endpoint} response.`,
+    { retryable: false }
+  );
+}
+function describeError(payload, response) {
+  const record = asRecord(payload);
+  const detail = text(record?.detail) ?? (typeof payload === "string" ? text(payload) : void 0) ?? (record ? JSON.stringify(record.detail ?? record) : void 0);
+  const trimmed = detail?.trim().slice(0, 500);
+  return `Sofya request failed (HTTP ${response.status})${trimmed ? `: ${trimmed}` : "."}`;
+}
+function toSearchResult(entry) {
+  const record = asRecord(entry);
+  const url = text(record?.url);
+  if (!record || !url) return void 0;
+  const { title, url: _url, content, description, fetched, ...rest } = record;
+  const metadata = Object.fromEntries(
+    Object.entries(rest).filter(
+      ([, value]) => value !== null && !(Array.isArray(value) && value.length === 0) && !(asRecord(value) && Object.keys(value).length === 0)
+    )
+  );
+  if (fetched === true && text(content)) metadata.content = content;
+  const excerpt = text(description) ?? text(content);
+  return {
+    title: text(title),
+    url,
+    snippet: excerpt ? trimSnippet(excerpt) : "",
+    ...Object.keys(metadata).length > 0 ? { metadata } : {}
+  };
+}
+function asRecord(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
+}
+function text(value) {
+  return typeof value === "string" && value.trim() ? value : void 0;
+}
+export {
+  adapter
+};