pi-web-providers 3.1.0 → 3.3.0

package/README.md

@@ -66,9 +66,9 @@ reported when the tool is actually called.
 | **Brave**      |   ✔    |          |   ✔    |    ✔     | `BRAVE_SEARCH_API_KEY` / `BRAVE_ANSWERS_API_KEY` |
 | **Cloudflare** |        |    ✔     |        |          | `CLOUDFLARE_API_TOKEN` + `CLOUDFLARE_ACCOUNT_ID` |
 | **Exa**        |   ✔    |    ✔     |   ✔    |    ✔     | `EXA_API_KEY`                                    |
-| **Firecrawl**  |   ✔    |    ✔     |        |          | `FIRECRAWL_API_KEY`                              |
+| **Firecrawl**  |   ✔    |    ✔     |   ✔    |          | `FIRECRAWL_API_KEY`                              |
 | **Gemini**     |   ✔    |          |   ✔    |    ✔     | `GOOGLE_API_KEY`                                 |
-| **Linkup**     |   ✔    |    ✔     |        |          | `LINKUP_API_KEY`                                 |
+| **Linkup**     |   ✔    |    ✔     |        |    ✔     | `LINKUP_API_KEY`                                 |
 | **Ollama**     |   ✔    |    ✔     |        |          | `OLLAMA_API_KEY`                                 |
 | **OpenAI**     |   ✔    |          |   ✔    |    ✔     | `OPENAI_API_KEY`                                 |
 | **Parallel**   |   ✔    |    ✔     |        |          | `PARALLEL_API_KEY`                               |
@@ -315,13 +315,19 @@ scope, or account ID is usually wrong.
 <summary><strong>Firecrawl</strong></summary>
 
 - SDK: `@mendable/firecrawl-js`
-- Supports `web_search` and `web_contents`
+- Supports `web_search`, `web_contents`, and page-scoped `web_answer`
 - Search can optionally include Firecrawl scrape-backed result enrichment
 - Contents extraction uses Firecrawl scrape with markdown-first defaults
+- Answers use Firecrawl scrape's `question` format against one explicit page URL;
+  set `options.url` in the `web_answer` call or
+  `providers.firecrawl.options.answer.url` as a default
 - Exposes search options such as `lang`, `country`, `sources`, `categories`,
   `location`, `timeout`, and `scrapeOptions`
 - Exposes contents options such as `formats`, `onlyMainContent`, `includeTags`,
   `excludeTags`, `waitFor`, `headers`, `location`, `mobile`, and `proxy`
+- Exposes answer options `url`, `onlyMainContent`, `includeTags`,
+  `excludeTags`, `waitFor`, `headers`, `location`, `mobile`, and `proxy`
+- Firecrawl charges 5 credits per page for the `question` format
 - Optional `baseUrl` overrides are supported for self-hosted Firecrawl
   instances, proxies, and testing. API keys are required for Firecrawl Cloud,
   but can be omitted for self-hosted endpoints that do not enforce
@@ -352,10 +358,17 @@ scope, or account ID is usually wrong.
 - SDK: `linkup-sdk`
 - Supports `web_search` via Linkup Search with fixed `searchResults` output
 - Supports `web_contents` via Linkup Fetch and always returns markdown
+- Supports `web_research` via Linkup's async Research API
 - Exposes search options `depth`, `includeImages`, `includeDomains`,
   `excludeDomains`, `fromDate`, and `toDate`
 - Exposes contents options `renderJs`, `includeRawHtml`, and `extractImages`
-- Good fit for a simple search-plus-markdown setup without extra provider wiring
+- Exposes research options `outputType`, `mode`, `reasoningDepth`,
+  `includeDomains`, `excludeDomains`, `fromDate`, `toDate`, and
+  `structuredOutputSchema`
+- Research defaults to `sourcedAnswer`; provide `structuredOutputSchema` for
+  structured output
+- Good fit for search, markdown extraction, and long-running sourced research
+  without extra provider wiring
 
 </details>
 

package/dist/index.js

@@ -3097,6 +3097,8 @@ var exaProvider = defineProvider({
 import FirecrawlClient from "@mendable/firecrawl-js";
 import { Type as Type7 } from "typebox";
 var FIRECRAWL_CLOUD_HOST = "api.firecrawl.dev";
+var FIRECRAWL_DEFAULT_API_URL = "https://api.firecrawl.dev";
+var FIRECRAWL_QUESTION_LIMIT = 1e4;
 var firecrawlSearchOptionsSchema = Type7.Object(
   {
     lang: Type7.Optional(
@@ -3203,6 +3205,55 @@ var firecrawlScrapeOptionsSchema = Type7.Object(
   },
   { description: "Firecrawl scrape options." }
 );
+var firecrawlAnswerOptionsSchema = Type7.Object(
+  {
+    url: Type7.String({
+      minLength: 1,
+      description: "URL of the page to ask about."
+    }),
+    onlyMainContent: Type7.Optional(
+      Type7.Boolean({ description: "Extract only the main content." })
+    ),
+    includeTags: Type7.Optional(
+      Type7.Array(Type7.String(), { description: "CSS selectors to include." })
+    ),
+    excludeTags: Type7.Optional(
+      Type7.Array(Type7.String(), { description: "CSS selectors to exclude." })
+    ),
+    waitFor: Type7.Optional(
+      Type7.Integer({
+        minimum: 0,
+        description: "Milliseconds to wait before scraping."
+      })
+    ),
+    headers: Type7.Optional(
+      Type7.Record(Type7.String(), Type7.String(), {
+        description: "Headers to send when scraping."
+      })
+    ),
+    location: Type7.Optional(
+      Type7.Object(
+        {
+          country: Type7.Optional(Type7.String({ description: "Country hint." })),
+          region: Type7.Optional(Type7.String({ description: "Region hint." })),
+          city: Type7.Optional(Type7.String({ description: "City hint." }))
+        },
+        { description: "Location hint for scraping." }
+      )
+    ),
+    mobile: Type7.Optional(
+      Type7.Boolean({ description: "Use a mobile browser profile." })
+    ),
+    proxy: Type7.Optional(
+      Type7.String({
+        description: "Proxy mode passed through to Firecrawl."
+      })
+    )
+  },
+  {
+    description: "Firecrawl page-question options. The URL is required; the question comes from the web_answer query."
+  }
+);
 var firecrawlImplementation = {
   id: "firecrawl",
   label: "Firecrawl",
@@ -3213,6 +3264,8 @@ var firecrawlImplementation = {
         return firecrawlSearchOptionsSchema;
       case "contents":
         return firecrawlScrapeOptionsSchema;
+      case "answer":
+        return firecrawlAnswerOptionsSchema;
       default:
         return void 0;
     }
@@ -3279,6 +3332,34 @@ var firecrawlImplementation = {
         })
       )
     };
+  },
+  async answer(query2, config, _context, options) {
+    const question = validateQuestion(query2);
+    const defaults = asJsonObject(config.options?.scrape);
+    const answerDefaults = asJsonObject(config.options?.answer);
+    const mergedOptions = {
+      onlyMainContent: true,
+      ...defaults,
+      ...answerDefaults,
+      ...options ?? {}
+    };
+    const url2 = validateUrl(mergedOptions.url);
+    const scrapeOptions = stripAnswerOnlyOptions(mergedOptions);
+    const response = await scrapeQuestion(config, url2, question, scrapeOptions);
+    const document = getFirecrawlDocument(response);
+    const answer = readString2(document.answer);
+    if (!answer?.trim()) {
+      throw new Error("No answer returned for this URL.");
+    }
+    return {
+      provider: firecrawlImplementation.id,
+      text: answer.trim(),
+      itemCount: 1,
+      metadata: {
+        url: url2,
+        ...asRecord(document.metadata) ? { metadata: document.metadata } : {}
+      }
+    };
   }
 };
 function createClient3(config) {
@@ -3302,6 +3383,88 @@ function getFirecrawlCapabilityStatus(config, options) {
 function isFirecrawlCloudApiUrl(apiUrl) {
   return !apiUrl || apiUrl.includes(FIRECRAWL_CLOUD_HOST);
 }
+function validateQuestion(query2) {
+  const question = query2.trim();
+  if (!question) {
+    throw new Error("question must be a non-empty string.");
+  }
+  if (question.length > FIRECRAWL_QUESTION_LIMIT) {
+    throw new Error(
+      `Firecrawl question must be at most ${FIRECRAWL_QUESTION_LIMIT} characters.`
+    );
+  }
+  return question;
+}
+function validateUrl(value) {
+  if (typeof value !== "string" || !value.trim()) {
+    throw new Error("Firecrawl answer requires options.url.");
+  }
+  return value.trim();
+}
+function stripAnswerOnlyOptions(options) {
+  const { url: _url, formats: _formats, ...scrapeOptions } = options;
+  return scrapeOptions;
+}
+async function scrapeQuestion(config, url2, question, options) {
+  const apiUrl = resolveConfigValue(config.baseUrl) ?? FIRECRAWL_DEFAULT_API_URL;
+  const apiKey = resolveConfigValue(config.credentials?.api);
+  if (isFirecrawlCloudApiUrl(apiUrl) && !apiKey) {
+    throw new Error("is missing an API key");
+  }
+  const response = await fetch(joinUrl(apiUrl, "/v2/scrape"), {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      ...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
+    },
+    body: JSON.stringify({
+      ...options,
+      url: url2,
+      formats: [{ type: "question", question }]
+    })
+  });
+  const payload = await readJsonResponse(response);
+  if (!response.ok) {
+    throw new Error(readFirecrawlError(payload, response.statusText));
+  }
+  if (isFirecrawlFailure(payload)) {
+    throw new Error(readFirecrawlError(payload, "Firecrawl scrape failed."));
+  }
+  return payload;
+}
+function joinUrl(baseUrl, path) {
+  return `${baseUrl.replace(/\/+$/g, "")}/${path.replace(/^\/+/g, "")}`;
+}
+async function readJsonResponse(response) {
+  const text = await response.text();
+  if (!text) {
+    return {};
+  }
+  try {
+    return JSON.parse(text);
+  } catch {
+    return text;
+  }
+}
+function isFirecrawlFailure(value) {
+  const record = asRecord(value);
+  return record?.success === false || record?.error !== void 0;
+}
+function readFirecrawlError(value, fallback) {
+  const record = asRecord(value);
+  return readString2(record?.error) ?? readString2(record?.message) ?? (typeof value === "string" ? value : void 0) ?? fallback;
+}
+function getFirecrawlDocument(value) {
+  const record = asRecord(value);
+  const data = asRecord(record?.data);
+  if (data) {
+    return data;
+  }
+  if (record) {
+    return record;
+  }
+  throw new Error(`Unexpected Firecrawl response: ${formatJson(value)}`);
+}
 function flattenSearchResults(response) {
   return ["web", "news", "images"].flatMap(
     (source) => (response[source] ?? []).map((entry) => toSearchResult(source, entry)).filter((entry) => entry !== null)
@@ -3389,6 +3552,21 @@ var firecrawlProvider = defineProvider({
           input.options
         );
       }
+    }),
+    answer: defineCapability({
+      options: firecrawlImplementation.getToolOptionsSchema?.("answer"),
+      promptGuidelines: [
+        "Firecrawl web_answer is page-scoped: set options.url to the specific page URL to ask about.",
+        "Do not use Firecrawl web_answer for general multi-source answers; use web_search plus web_contents or web_research instead."
+      ],
+      async execute(input, ctx) {
+        return await firecrawlImplementation.answer(
+          input.query,
+          ctx.config,
+          ctx,
+          input.options
+        );
+      }
     })
   }
 });
@@ -4241,6 +4419,45 @@ var linkupContentsOptionsSchema = Type9.Object(
   },
   { description: "Linkup fetch options." }
 );
+var linkupResearchOptionsSchema = Type9.Object(
+  {
+    outputType: Type9.Optional(
+      literalUnion(["sourcedAnswer", "structured"], {
+        description: "Research output type. Defaults to 'sourcedAnswer' unless structuredOutputSchema is provided."
+      })
+    ),
+    mode: Type9.Optional(
+      literalUnion(["answer", "auto", "investigate", "research"], {
+        description: "Research mode. Use 'answer' for precise verified answers, 'investigate' for focused deep dives, 'research' for broad reports, or omit/auto to let Linkup classify the task."
+      })
+    ),
+    reasoningDepth: Type9.Optional(
+      literalUnion(["S", "M", "L", "XL"], {
+        description: "Reasoning depth. Higher values trade latency for more thorough investigation."
+      })
+    ),
+    includeDomains: Type9.Optional(
+      Type9.Array(Type9.String(), {
+        description: "Restrict research to these domains."
+      })
+    ),
+    excludeDomains: Type9.Optional(
+      Type9.Array(Type9.String(), { description: "Exclude these domains." })
+    ),
+    fromDate: Type9.Optional(
+      Type9.String({ description: "ISO date string for earliest result date." })
+    ),
+    toDate: Type9.Optional(
+      Type9.String({ description: "ISO date string for latest result date." })
+    ),
+    structuredOutputSchema: Type9.Optional(
+      Type9.Record(Type9.String(), Type9.Any(), {
+        description: "JSON schema object required when outputType is 'structured'."
+      })
+    )
+  },
+  { description: "Linkup research options." }
+);
 var linkupImplementation = {
   id: "linkup",
   label: "Linkup",
@@ -4251,6 +4468,8 @@ var linkupImplementation = {
         return linkupSearchOptionsSchema;
       case "contents":
         return linkupContentsOptionsSchema;
+      case "research":
+        return linkupResearchOptionsSchema;
       default:
         return void 0;
     }
@@ -4307,6 +4526,51 @@ var linkupImplementation = {
         })
       )
     };
+  },
+  async research(input, config, context, options) {
+    return await executeAsyncResearch({
+      providerLabel: linkupImplementation.label,
+      providerId: linkupImplementation.id,
+      context,
+      start: (researchContext) => linkupImplementation.startResearch(
+        input,
+        config,
+        researchContext,
+        options
+      ),
+      poll: (id, researchContext) => linkupImplementation.pollResearch(id, config, researchContext)
+    });
+  },
+  async startResearch(input, config, _context, options) {
+    const client = createClient4(config);
+    const defaults = asJsonObject(config.options?.research) ?? {};
+    const task = await client.research(
+      buildResearchParams(input, {
+        ...defaults,
+        ...options ?? {}
+      })
+    );
+    return { id: task.id };
+  },
+  async pollResearch(id, config, _context) {
+    const client = createClient4(config);
+    const task = await client.getResearch(id);
+    if (task.status === "completed") {
+      return {
+        status: "completed",
+        output: formatResearchTaskOutput(task)
+      };
+    }
+    if (task.status === "failed") {
+      return {
+        status: "failed",
+        error: task.error ?? "research failed"
+      };
+    }
+    return {
+      status: "in_progress",
+      statusText: task.status
+    };
   }
 };
 function buildSearchParams(query2, maxResults, options) {
@@ -4351,6 +4615,45 @@ function buildFetchParams(url2, options) {
     ...fetchOptions.extractImages !== void 0 ? { extractImages: fetchOptions.extractImages } : {}
   };
 }
+function buildResearchParams(input, options) {
+  const researchOptions = options;
+  if (researchOptions.q !== void 0 || researchOptions.query !== void 0 || researchOptions.input !== void 0) {
+    throw new Error(
+      "Linkup research options cannot override the managed input."
+    );
+  }
+  const outputType = researchOptions.outputType ?? (researchOptions.structuredOutputSchema !== void 0 ? "structured" : "sourcedAnswer");
+  if (outputType === "structured" && researchOptions.structuredOutputSchema === void 0) {
+    throw new Error(
+      "Linkup research outputType 'structured' requires structuredOutputSchema."
+    );
+  }
+  if (outputType === "sourcedAnswer" && researchOptions.structuredOutputSchema !== void 0) {
+    throw new Error(
+      "Linkup research structuredOutputSchema requires outputType 'structured'."
+    );
+  }
+  const commonParams = {
+    query: input,
+    ...researchOptions.includeDomains !== void 0 ? { includeDomains: researchOptions.includeDomains } : {},
+    ...researchOptions.excludeDomains !== void 0 ? { excludeDomains: researchOptions.excludeDomains } : {},
+    ...researchOptions.fromDate !== void 0 ? { fromDate: toDate(researchOptions.fromDate, "fromDate") } : {},
+    ...researchOptions.toDate !== void 0 ? { toDate: toDate(researchOptions.toDate, "toDate") } : {},
+    ...researchOptions.mode !== void 0 ? { mode: researchOptions.mode } : {},
+    ...researchOptions.reasoningDepth !== void 0 ? { reasoningDepth: researchOptions.reasoningDepth } : {}
+  };
+  if (outputType === "structured") {
+    return {
+      ...commonParams,
+      outputType,
+      structuredOutputSchema: researchOptions.structuredOutputSchema
+    };
+  }
+  return {
+    ...commonParams,
+    outputType
+  };
+}
 function createClient4(config) {
   const apiKey = resolveConfigValue(config.credentials?.api);
   if (!apiKey) {
@@ -4361,6 +4664,40 @@ function createClient4(config) {
     baseUrl: resolveConfigValue(config.baseUrl)
   });
 }
+function formatResearchTaskOutput(task) {
+  const output = task.output;
+  if (!output) {
+    return {
+      provider: linkupImplementation.id,
+      text: "Linkup research completed without textual output."
+    };
+  }
+  const outputRecord = asRecord2(output);
+  const inputRecord = asRecord2(task.input);
+  const outputType = inputRecord ? readString3(inputRecord.outputType) : void 0;
+  const answer = outputRecord ? readString3(outputRecord.answer) : void 0;
+  const sources = outputRecord ? readSources(outputRecord.sources) : [];
+  if (outputType !== "structured" && answer !== void 0) {
+    const lines = [answer];
+    if (sources.length > 0) {
+      lines.push("");
+      lines.push("Sources:");
+      for (const [index, source] of sources.entries()) {
+        lines.push(`${index + 1}. ${source.title}`);
+        lines.push(`   ${source.url}`);
+      }
+    }
+    return {
+      provider: linkupImplementation.id,
+      text: lines.join("\n").trimEnd(),
+      itemCount: sources.length
+    };
+  }
+  return {
+    provider: linkupImplementation.id,
+    text: formatJson(output)
+  };
+}
 function toSearchResult2(value) {
   const entry = asRecord2(value);
   if (!entry) {
@@ -4382,6 +4719,27 @@ function toSearchResult2(value) {
     metadata: Object.keys(metadata).length > 0 ? metadata : void 0
   };
 }
+function readSources(value) {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  return value.flatMap((entry) => {
+    const source = asRecord2(entry);
+    if (!source) {
+      return [];
+    }
+    const url2 = readString3(source.url);
+    if (!url2) {
+      return [];
+    }
+    return [
+      {
+        title: readString3(source.name) ?? url2,
+        url: url2
+      }
+    ];
+  });
+}
 function asRecord2(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
 }
@@ -4435,6 +4793,17 @@ var linkupProvider = defineProvider({
           input.options
         );
       }
+    }),
+    research: defineCapability({
+      options: linkupImplementation.getToolOptionsSchema?.("research"),
+      async execute(input, ctx) {
+        return await linkupImplementation.research(
+          input.input,
+          ctx.config,
+          ctx,
+          input.options
+        );
+      }
     })
   }
 });
@@ -5849,7 +6218,7 @@ var serperImplementation = {
       requestOptions
     );
     const response = await fetch(
-      joinUrl(resolveConfigValue(config.baseUrl), requestOptions.mode),
+      joinUrl2(resolveConfigValue(config.baseUrl), requestOptions.mode),
       {
         method: "POST",
         headers: {
@@ -5884,7 +6253,7 @@ var serperImplementation = {
     };
   }
 };
-function joinUrl(baseUrl, mode = "search") {
+function joinUrl2(baseUrl, mode = "search") {
   const base2 = (baseUrl ?? DEFAULT_BASE_URL3).replace(/\/+$/, "");
   if (mode === "webpage" && base2 === DEFAULT_BASE_URL3) {
     return DEFAULT_SCRAPE_URL;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-web-providers",
-  "version": "3.1.0",
+  "version": "3.3.0",
   "description": "Configurable web access extension for pi with per-tool provider routing and explicit provider option schemas for search, contents, quick grounded answers, and research.",
   "type": "module",
   "files": [
@@ -73,7 +73,7 @@
     "@tavily/core": "^0.7.3",
     "cloudflare": "^5.2.0",
     "exa-js": "^2.12.1",
-    "linkup-sdk": "^2.7.0",
+    "linkup-sdk": "^3.2.0",
     "openai": "^6.35.0",
     "parallel-web": "^0.4.1",
     "valyu-js": "^2.7.14"