website-api 1.1.3 → 1.1.4

package/dist/src/sites/gemini.google.com/openapi.yaml

@@ -0,0 +1,39 @@
+# Generated by `pnpm generate:openapi` — do not edit by hand.
+openapi: 3.1.0
+info:
+  title: Gemini Usage
+  description: Fetches Gemini account usage/quota details directly over HTTP (no browser).
+  version: 1.1.3
+servers:
+  - url: https://gemini.google.com
+paths:
+  /usage:
+    get:
+      summary: "Gemini Usage: GET /usage"
+      description: Fetches Gemini account usage/quota details directly over HTTP (no browser).
+      operationId: gemini_usage_get__usage
+      responses:
+        "200":
+          description: JSON response body (shape defined by the site, see its transform)
+      security:
+        - chromeSession: []
+components:
+  securitySchemes:
+    chromeSession:
+      type: apiKey
+      in: cookie
+      name: session
+      description: "Authenticated via the user's real Chrome session: website-api injects decrypted Chrome
+        cookies for google.com into every request."
+x-website-api:
+  id: gemini-usage
+  domain: gemini.google.com
+  cookieDomain: google.com
+  transport: http
+  cookies: required
+  requiresLogin: true
+  imperative: false
+  cli:
+    command: website-api gemini-usage
+    positionals: []
+    parameters: []

package/dist/src/sites/google.com/google-helpers.js

@@ -1 +1,255 @@
-export function formEncode(t){let e="";const r=String(t??"");for(let t=0;t<r.length;t++){const n=r.codePointAt(t);if(void 0===n)continue;const o=String.fromCodePoint(n);if(n>65535&&t++," "===o)e+="+";else if("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~".includes(o))e+=o;else for(const t of utf8Bytes(n))e+=`%${t.toString(16).toUpperCase().padStart(2,"0")}`}return e}export function utf8Bytes(t){return t<=127?[t]:t<=2047?[192|t>>6,128|63&t]:t<=65535?[224|t>>12,128|t>>6&63,128|63&t]:[240|t>>18,128|t>>12&63,128|t>>6&63,128|63&t]}export function utf8String(t){let e="";for(let r=0;r<t.length;r++){const n=t[r];let o=n;192==(224&n)?o=(31&n)<<6|63&t[++r]:224==(240&n)?o=(15&n)<<12|(63&t[++r])<<6|63&t[++r]:240==(248&n)&&(o=(7&n)<<18|(63&t[++r])<<12|(63&t[++r])<<6|63&t[++r]),e+=String.fromCodePoint(o)}return e}export function percentDecode(t){const e=String(t??"").replace(/\+/g," ");let r="";for(let t=0;t<e.length;t++){if("%"!==e[t]||!/[0-9a-fA-F]{2}/.test(e.slice(t+1,t+3))){r+=e[t];continue}const n=[];for(;"%"===e[t]&&/[0-9a-fA-F]{2}/.test(e.slice(t+1,t+3));)n.push(Number.parseInt(e.slice(t+1,t+3),16)),t+=3;t--;try{r+=utf8String(n)}catch{r+=n.map(t=>`%${t.toString(16).toUpperCase().padStart(2,"0")}`).join("")}}return r}export function parseQueryString(t){const e={};for(const r of String(t??"").replace(/^\?/,"").split("&")){if(!r)continue;const t=r.indexOf("="),n=-1===t?r:r.slice(0,t),o=-1===t?"":r.slice(t+1);e[percentDecode(n)]=percentDecode(o)}return e}export function buildQueryString(t){return Object.entries(t).filter(([,t])=>null!=t&&""!==t).map(([t,e])=>`${formEncode(t)}=${formEncode(e)}`).join("&")}export function googlePath(t){const e=/^https?:\/\/([^/]+)(\/[^?#]*)?/i.exec(String(t??""));return e&&/(^|\.)google\.[^/]+$/i.test(e[1])?e[2]||"/":null}export function stripXssi(t){return t.replace(/^\s*\)\]\}'\s*\n?/,"")}export function parseJsonMaybe(t){try{return JSON.parse(t)}catch{return}}export function parseGoogleRecordStream(t){const e=[];for(const r of stripXssi(t).split(/\r?\n/)){const t=r.trim();if(!t)continue;const n=/^([a-zA-Z0-9_-]+);(.*)$/.exec(t);if(!n)continue;const o=n[2].trim();e.push({id:n[1],value:parseJsonMaybe(o)??o})}return e}export function decodeGoogleBody({body:t,contentType:e=""}){const r=String(t??""),n=stripXssi(r).trimStart(),o=/^\s*\)\]\}'/.test(r),i=/^[\[{]/.test(n)?parseJsonMaybe(n):void 0,s=void 0===i?parseGoogleRecordStream(r):[];let c="text";return(e.includes("html")||/^\s*</.test(n))&&(c="html"),o&&void 0!==i?c="google-xssi-json":o&&s.length?c="google-xssi-record-stream":void 0!==i?c="json":s.length&&(c="google-record-stream"),{format:c,xssiPrefixed:o,parsed:i,records:s}}export function cleanText(t,e=4e3){return String(t??"").replace(/\u0000/g,"").replace(/[ \t]+\n/g,"\n").trim().slice(0,e)}export function extractAnswerFromText(t){const e=String(t??"").split("\n").map(t=>t.trim()).filter(Boolean),r=e.findIndex(t=>/AI Mode response is ready/i.test(t));if(r>0)return e[r-1];const n=/^(Skip to main content|Accessibility help|Accessibility feedback|AI Mode|All|Images|Videos|News|More|Search Results|Sources|Related)$/i;return e.find(t=>!n.test(t))??null}export function cleanHtml(t){let e=t.replace(/&lt;/g,"<").replace(/&gt;/g,">").replace(/&amp;/g,"&").replace(/&nbsp;/g," ");return e=function(t){let e="",r=0;for(;r<t.length;){const n=t.toLowerCase().indexOf("<style",r);if(-1===n){e+=t.slice(r);break}e+=t.slice(r,n);const o=t.toLowerCase().indexOf("</style>",n);if(-1===o)break;r=o+8}let n="";for(r=0;r<e.length;){const t=e.toLowerCase().indexOf("<script",r);if(-1===t){n+=e.slice(r);break}n+=e.slice(r,t);const o=e.toLowerCase().indexOf("<\/script>",t);if(-1===o)break;r=o+9}return n}(e),e=e.replace(/<[^>]+>/g," "),e.replace(/\s+/g," ").trim()}export function findHtmlInObject(t){if(!t)return null;if("string"==typeof t){const e=t.trim();return e.startsWith("<")||e.includes("class=")||e.includes("id=")?t:null}if(Array.isArray(t))for(const e of t){const t=findHtmlInObject(e);if(t)return t}else if("object"==typeof t){if("string"==typeof t.html)return t.html;if("string"==typeof t.aimc_block?.html)return t.aimc_block.html;if("string"==typeof t.value&&(t.value.startsWith("<")||t.value.includes("class=")))return t.value;for(const e of Object.values(t)){const t=findHtmlInObject(e);if(t)return t}}return null}export function extractAnswerFromRecordStream(t){try{const e=stripXssi(t).trimStart();if(e.startsWith("<")||e.includes("class=")||e.includes("id="))return cleanHtml(e);for(const e of parseGoogleRecordStream(t)){const t=findHtmlInObject(e.value);if(t)return cleanHtml(t)}}catch{}return null}
+// Pure encoders/decoders for Google's search + batchexecute responses.
+// No Playwright or I/O here, so every function is directly unit-testable.
+export function formEncode(value) {
+    const safe = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~";
+    let out = "";
+    const input = String(value ?? "");
+    for (let i = 0; i < input.length; i++) {
+        const codePoint = input.codePointAt(i);
+        if (codePoint === undefined)
+            continue;
+        const ch = String.fromCodePoint(codePoint);
+        if (codePoint > 0xffff)
+            i++;
+        if (ch === " ")
+            out += "+";
+        else if (safe.includes(ch))
+            out += ch;
+        else {
+            for (const byte of utf8Bytes(codePoint)) {
+                out += `%${byte.toString(16).toUpperCase().padStart(2, "0")}`;
+            }
+        }
+    }
+    return out;
+}
+export function utf8Bytes(codePoint) {
+    if (codePoint <= 0x7f)
+        return [codePoint];
+    if (codePoint <= 0x7ff)
+        return [0xc0 | (codePoint >> 6), 0x80 | (codePoint & 0x3f)];
+    if (codePoint <= 0xffff) {
+        return [0xe0 | (codePoint >> 12), 0x80 | ((codePoint >> 6) & 0x3f), 0x80 | (codePoint & 0x3f)];
+    }
+    return [
+        0xf0 | (codePoint >> 18),
+        0x80 | ((codePoint >> 12) & 0x3f),
+        0x80 | ((codePoint >> 6) & 0x3f),
+        0x80 | (codePoint & 0x3f),
+    ];
+}
+export function utf8String(bytes) {
+    let out = "";
+    for (let i = 0; i < bytes.length; i++) {
+        const b1 = bytes[i];
+        let cp = b1;
+        if ((b1 & 0xe0) === 0xc0)
+            cp = ((b1 & 0x1f) << 6) | (bytes[++i] & 0x3f);
+        else if ((b1 & 0xf0) === 0xe0)
+            cp = ((b1 & 0x0f) << 12) | ((bytes[++i] & 0x3f) << 6) | (bytes[++i] & 0x3f);
+        else if ((b1 & 0xf8) === 0xf0)
+            cp =
+                ((b1 & 0x07) << 18) | ((bytes[++i] & 0x3f) << 12) | ((bytes[++i] & 0x3f) << 6) | (bytes[++i] & 0x3f);
+        out += String.fromCodePoint(cp);
+    }
+    return out;
+}
+export function percentDecode(value) {
+    const input = String(value ?? "").replace(/\+/g, " ");
+    let out = "";
+    for (let i = 0; i < input.length; i++) {
+        if (input[i] !== "%" || !/[0-9a-fA-F]{2}/.test(input.slice(i + 1, i + 3))) {
+            out += input[i];
+            continue;
+        }
+        const bytes = [];
+        while (input[i] === "%" && /[0-9a-fA-F]{2}/.test(input.slice(i + 1, i + 3))) {
+            bytes.push(Number.parseInt(input.slice(i + 1, i + 3), 16));
+            i += 3;
+        }
+        i--;
+        try {
+            out += utf8String(bytes);
+        }
+        catch {
+            out += bytes.map((b) => `%${b.toString(16).toUpperCase().padStart(2, "0")}`).join("");
+        }
+    }
+    return out;
+}
+export function parseQueryString(search) {
+    const query = {};
+    for (const part of String(search ?? "")
+        .replace(/^\?/, "")
+        .split("&")) {
+        if (!part)
+            continue;
+        const eq = part.indexOf("=");
+        const rawKey = eq === -1 ? part : part.slice(0, eq);
+        const rawValue = eq === -1 ? "" : part.slice(eq + 1);
+        query[percentDecode(rawKey)] = percentDecode(rawValue);
+    }
+    return query;
+}
+export function buildQueryString(query) {
+    return Object.entries(query)
+        .filter(([, value]) => value != null && value !== "")
+        .map(([key, value]) => `${formEncode(key)}=${formEncode(value)}`)
+        .join("&");
+}
+export function googlePath(url) {
+    const m = /^https?:\/\/([^/]+)(\/[^?#]*)?/i.exec(String(url ?? ""));
+    if (!m || !/(^|\.)google\.[^/]+$/i.test(m[1]))
+        return null;
+    return m[2] || "/";
+}
+export function stripXssi(text) {
+    return text.replace(/^\s*\)\]\}'\s*\n?/, "");
+}
+export function parseJsonMaybe(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+export function parseGoogleRecordStream(text) {
+    const records = [];
+    for (const rawLine of stripXssi(text).split(/\r?\n/)) {
+        const line = rawLine.trim();
+        if (!line)
+            continue;
+        const m = /^([a-zA-Z0-9_-]+);(.*)$/.exec(line);
+        if (!m)
+            continue;
+        const valueText = m[2].trim();
+        records.push({ id: m[1], value: parseJsonMaybe(valueText) ?? valueText });
+    }
+    return records;
+}
+export function decodeGoogleBody({ body, contentType = "" }) {
+    const text = String(body ?? "");
+    const unprefixed = stripXssi(text).trimStart();
+    const xssiPrefixed = /^\s*\)\]\}'/.test(text);
+    const parsed = /^[[{]/.test(unprefixed) ? parseJsonMaybe(unprefixed) : undefined;
+    const records = parsed === undefined ? parseGoogleRecordStream(text) : [];
+    let format = "text";
+    if (contentType.includes("html") || /^\s*</.test(unprefixed))
+        format = "html";
+    if (xssiPrefixed && parsed !== undefined)
+        format = "google-xssi-json";
+    else if (xssiPrefixed && records.length)
+        format = "google-xssi-record-stream";
+    else if (parsed !== undefined)
+        format = "json";
+    else if (records.length)
+        format = "google-record-stream";
+    return { format, xssiPrefixed, parsed, records };
+}
+export function cleanText(s, limit = 4000) {
+    return String(s ?? "")
+        .replace(/\u0000/g, "")
+        .replace(/[ \t]+\n/g, "\n")
+        .trim()
+        .slice(0, limit);
+}
+export function extractAnswerFromText(text) {
+    const lines = String(text ?? "")
+        .split("\n")
+        .map((s) => s.trim())
+        .filter(Boolean);
+    const ready = lines.findIndex((l) => /AI Mode response is ready/i.test(l));
+    if (ready > 0)
+        return lines[ready - 1];
+    const ignored = /^(Skip to main content|Accessibility help|Accessibility feedback|AI Mode|All|Images|Videos|News|More|Search Results|Sources|Related)$/i;
+    return lines.find((l) => !ignored.test(l)) ?? null;
+}
+function stripStyleAndScriptTags(html) {
+    let clean = "";
+    let currentIndex = 0;
+    while (currentIndex < html.length) {
+        const styleStartIndex = html.toLowerCase().indexOf("<style", currentIndex);
+        if (styleStartIndex === -1) {
+            clean += html.slice(currentIndex);
+            break;
+        }
+        clean += html.slice(currentIndex, styleStartIndex);
+        const styleEndIndex = html.toLowerCase().indexOf("</style>", styleStartIndex);
+        if (styleEndIndex === -1)
+            break;
+        currentIndex = styleEndIndex + "</style>".length;
+    }
+    let clean2 = "";
+    currentIndex = 0;
+    while (currentIndex < clean.length) {
+        const scriptStartIndex = clean.toLowerCase().indexOf("<script", currentIndex);
+        if (scriptStartIndex === -1) {
+            clean2 += clean.slice(currentIndex);
+            break;
+        }
+        clean2 += clean.slice(currentIndex, scriptStartIndex);
+        const scriptEndIndex = clean.toLowerCase().indexOf("</script>", scriptStartIndex);
+        if (scriptEndIndex === -1)
+            break;
+        currentIndex = scriptEndIndex + "</script>".length;
+    }
+    return clean2;
+}
+export function cleanHtml(html) {
+    let clean = html
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&amp;/g, "&")
+        .replace(/&nbsp;/g, " ");
+    clean = stripStyleAndScriptTags(clean);
+    clean = clean.replace(/<[^>]+>/g, " ");
+    return clean.replace(/\s+/g, " ").trim();
+}
+export function findHtmlInObject(obj) {
+    if (!obj)
+        return null;
+    if (typeof obj === "string") {
+        const trimmed = obj.trim();
+        if (trimmed.startsWith("<") || trimmed.includes("class=") || trimmed.includes("id="))
+            return obj;
+        return null;
+    }
+    if (Array.isArray(obj)) {
+        for (const item of obj) {
+            const found = findHtmlInObject(item);
+            if (found)
+                return found;
+        }
+    }
+    else if (typeof obj === "object") {
+        if (typeof obj.html === "string")
+            return obj.html;
+        if (typeof obj.aimc_block?.html === "string")
+            return obj.aimc_block.html;
+        if (typeof obj.value === "string" && (obj.value.startsWith("<") || obj.value.includes("class="))) {
+            return obj.value;
+        }
+        for (const value of Object.values(obj)) {
+            const found = findHtmlInObject(value);
+            if (found)
+                return found;
+        }
+    }
+    return null;
+}
+export function extractAnswerFromRecordStream(text) {
+    try {
+        const unprefixed = stripXssi(text).trimStart();
+        if (unprefixed.startsWith("<") || unprefixed.includes("class=") || unprefixed.includes("id=")) {
+            return cleanHtml(unprefixed);
+        }
+        for (const record of parseGoogleRecordStream(text)) {
+            const html = findHtmlInObject(record.value);
+            if (html)
+                return cleanHtml(html);
+        }
+    }
+    catch { }
+    return null;
+}

package/dist/src/sites/google.com/index.js

@@ -1 +1,253 @@
-import{defineSite as e}from"../../core/define-site.js";import{buildQueryString as t,cleanText as o,decodeGoogleBody as n,extractAnswerFromRecordStream as r,extractAnswerFromText as s,formEncode as i,googlePath as a,parseQueryString as l}from"./google-helpers.js";function u(e,t){return e.map(e=>{const o=null==e.body?null:n({body:e.body,contentType:e.mimeType});return{method:e.method,type:e.type,url:e.url,status:e.status,mimeType:e.mimeType,decodedFormat:o?.format??null,recordCount:o?.records?.length??0,bodyPrefix:null==e.body?null:e.body.slice(0,t)}})}export default e({id:"google-ai",name:"Google AI Overview",domain:"google.com",description:"Fetches Google's AI Overview and AI Mode answers using browser-attached Playwright.",transport:"browser",cookies:"optional",endpoints:[{url:"https://www.google.com"}],positionals:[{name:"question",description:"The search query or question to ask Google",required:!0,variadic:!0}],parameters:[{name:"raw-limit",type:"number",description:"Max raw response chars to include",default:12e3},{name:"timeout",type:"number",description:"Playwright timeout in milliseconds",default:9e4},{name:"text",type:"boolean",description:"Print only the extracted AI Overview answer text",short:"t"}],run:async e=>{const d=e.options.question,c=void 0!==e.options.rawLimit?Number(e.options.rawLimit):12e3,m=void 0!==e.options.timeout?Number(e.options.timeout):9e4,p=await e.browser(),y=await p.context().newCDPSession(p);await y.send("Network.enable",{maxTotalBufferSize:1e8,maxResourceBufferSize:1e8});const w=[],g=new Map;y.on("Network.requestWillBeSent",e=>{const t=e.request||{};(e=>{const t=a(e);return"/search"===t||t?.startsWith("/async/")||t?.includes("batchexecute")})(t.url)&&(g.set(e.requestId,w.length),w.push({id:e.requestId,type:e.type,method:t.method,url:t.url,postData:t.postData||null,status:null,mimeType:null,body:null}))}),y.on("Network.responseReceived",e=>{const t=g.get(e.requestId);null!=t&&(w[t].status=e.response.status,w[t].mimeType=e.response.mimeType)}),y.on("Network.loadingFinished",async e=>{const t=g.get(e.requestId);if(null!=t)try{const o=w[t].mimeType||"";if(!/text|json|html|javascript|x-protobuf/.test(o))return;const n=await y.send("Network.getResponseBody",{requestId:e.requestId});w[t].body=n.base64Encoded?null:(n.body||"").slice(0,c)}catch{}});const f=`https://www.google.com/search?${t({q:d,udm:"50"})}`;e.debug&&console.log(`Navigating to Google Search: ${f}`),await p.goto(f,{waitUntil:"domcontentloaded"});const h=await async function(e,t){const o=Date.now();for(;Date.now()-o<Math.min(t,45e3);){await e.waitForTimeout(750);const t=await e.evaluate(()=>{const e=Array.from(document.querySelectorAll('[jsname="KFl8ub"], [data-attrid], .kp-wholepage')).map(e=>e.innerText?.trim()).filter(Boolean).find(e=>!/^(Sources|Related|AI Mode response is ready)$/i.test(e));if(e)return e;const t=(document.body?.innerText||"").split("\n").map(e=>e.trim()).filter(Boolean),o=t.findIndex(e=>/AI Mode response is ready/i.test(e));return o>0?t[o-1]:null});if(t)return t}return null}(p,m),b=await async function(e,t){return e.evaluate(e=>{const t=document.body?.innerText||"",o=document.documentElement?.outerHTML||"";return{title:document.title,url:document.location.href,bodyText:t.slice(0,e),htmlPrefix:o.slice(0,e)}},t)}(p,c),x=await async function(e){const o=await e.evaluate(()=>{const e=document.querySelector("[data-garc][data-lro-token][data-lro-signature][data-ei]");if(!e)return{url:null,error:"Missing AI Mode token container"};const t=document.getElementById("rKxeg")?.getAttribute("data-stkp")||null;return{origin:document.location.origin,search:document.location.search,stkp:t,fmt:document.querySelector("[data-madl]")?"madl":"adl",tokens:{ei:e.dataset.ei,garc:e.dataset.garc,lroToken:e.dataset.lroToken,lroSignature:e.dataset.lroSignature,xsrfFolwrToken:e.dataset.xsrfFolwrToken||null,srtst:e.dataset.srtst||null}}});if(!o?.origin)return o;const n=l(o.search),r={},s=["q","udm","mstk","csuir","mtid","ved","vet","sei","dpr","hl","gl","source","vsrid","lns_img","cinpts"];for(const e of s)n[e]&&(r[e]=n[e]);o.tokens.srtst&&(r.srtst=o.tokens.srtst),r.garc=o.tokens.garc,r.mlro=o.tokens.lroToken,r.mlros=o.tokens.lroSignature,r.ei=o.tokens.ei,o.stkp&&(r.stkp=o.stkp);const a={_fmt:o.fmt};o.tokens.xsrfFolwrToken&&(a._xsrf=o.tokens.xsrfFolwrToken);const u=t(r),d=Object.entries(a).map(([e,t])=>`${i(e)}:${i(t)}`).join(",");return{url:`${o.origin}/async/folwr?${u}&async=${d}`,tokens:o.tokens}}(p);let k=null;if(x?.url){e.debug&&console.log(`Discovered folwr endpoint: ${x.url}`);try{const e=await p.evaluate(async e=>(await fetch(e)).text(),x.url),t=r(e),o=s(e),i=!o||o.includes("<")||o.includes("class=")?null:o;k={title:b.title,url:x.url,bodyText:e,htmlPrefix:e.slice(0,c),answer:t||i,decoded:n({body:e,contentType:"text/plain"})}}catch(t){e.debug&&console.warn("Failed to query folwr endpoint in-page:",t)}}await p.waitForTimeout(500);const T=k?.answer||h||s(b.bodyText);return{question:d,answer:o(T,c)||null,finalUrl:p.url(),endpoint:x||null,searchPage:{title:b.title,url:b.url,bodyText:o(b.bodyText,c)},endpointResult:k?{title:k.title,url:k.url,bodyText:o(k.bodyText,c),htmlPrefix:k.htmlPrefix,decoded:k.decoded}:null,requests:u(w,c)}}});
+import { defineSite } from "../../core/define-site.js";
+import { buildQueryString, cleanText, decodeGoogleBody, extractAnswerFromRecordStream, extractAnswerFromText, formEncode, googlePath, parseQueryString, } from "./google-helpers.js";
+async function waitForAiAnswer(page, timeout) {
+    const started = Date.now();
+    while (Date.now() - started < Math.min(timeout, 45000)) {
+        await page.waitForTimeout(750);
+        const answer = await page.evaluate(() => {
+            const direct = Array.from(document.querySelectorAll('[jsname="KFl8ub"], [data-attrid], .kp-wholepage'))
+                .map((e) => e.innerText?.trim())
+                .filter(Boolean)
+                .find((t) => !/^(Sources|Related|AI Mode response is ready)$/i.test(t));
+            if (direct)
+                return direct;
+            const text = document.body?.innerText || "";
+            const lines = text
+                .split("\n")
+                .map((s) => s.trim())
+                .filter(Boolean);
+            const ready = lines.findIndex((l) => /AI Mode response is ready/i.test(l));
+            return ready > 0 ? lines[ready - 1] : null;
+        });
+        if (answer)
+            return answer;
+    }
+    return null;
+}
+async function collectPageState(page, limit) {
+    return page.evaluate((l) => {
+        const bodyText = document.body?.innerText || "";
+        const html = document.documentElement?.outerHTML || "";
+        return {
+            title: document.title,
+            url: document.location.href,
+            bodyText: bodyText.slice(0, l),
+            htmlPrefix: html.slice(0, l),
+        };
+    }, limit);
+}
+async function buildFolwrEndpoint(page) {
+    const pageData = await page.evaluate(() => {
+        const root = document.querySelector("[data-garc][data-lro-token][data-lro-signature][data-ei]");
+        if (!root)
+            return { url: null, error: "Missing AI Mode token container" };
+        const stkp = document.getElementById("rKxeg")?.getAttribute("data-stkp") || null;
+        return {
+            origin: document.location.origin,
+            search: document.location.search,
+            stkp,
+            fmt: document.querySelector("[data-madl]") ? "madl" : "adl",
+            tokens: {
+                ei: root.dataset.ei,
+                garc: root.dataset.garc,
+                lroToken: root.dataset.lroToken,
+                lroSignature: root.dataset.lroSignature,
+                xsrfFolwrToken: root.dataset.xsrfFolwrToken || null,
+                srtst: root.dataset.srtst || null,
+            },
+        };
+    });
+    if (!pageData?.origin)
+        return pageData;
+    const current = parseQueryString(pageData.search);
+    const query = {};
+    const copyKeys = [
+        "q",
+        "udm",
+        "mstk",
+        "csuir",
+        "mtid",
+        "ved",
+        "vet",
+        "sei",
+        "dpr",
+        "hl",
+        "gl",
+        "source",
+        "vsrid",
+        "lns_img",
+        "cinpts",
+    ];
+    for (const key of copyKeys)
+        if (current[key])
+            query[key] = current[key];
+    if (pageData.tokens.srtst)
+        query.srtst = pageData.tokens.srtst;
+    query.garc = pageData.tokens.garc;
+    query.mlro = pageData.tokens.lroToken;
+    query.mlros = pageData.tokens.lroSignature;
+    query.ei = pageData.tokens.ei;
+    if (pageData.stkp)
+        query.stkp = pageData.stkp;
+    const asyncParts = { _fmt: pageData.fmt };
+    if (pageData.tokens.xsrfFolwrToken)
+        asyncParts._xsrf = pageData.tokens.xsrfFolwrToken;
+    const queryString = buildQueryString(query);
+    const asyncString = Object.entries(asyncParts)
+        .map(([key, value]) => `${formEncode(key)}:${formEncode(value)}`)
+        .join(",");
+    return {
+        url: `${pageData.origin}/async/folwr?${queryString}&async=${asyncString}`,
+        tokens: pageData.tokens,
+    };
+}
+function summarizeNetwork(rows, rawLimit) {
+    return rows.map((r) => {
+        const decoded = r.body == null ? null : decodeGoogleBody({ body: r.body, contentType: r.mimeType });
+        return {
+            method: r.method,
+            type: r.type,
+            url: r.url,
+            status: r.status,
+            mimeType: r.mimeType,
+            decodedFormat: decoded?.format ?? null,
+            recordCount: decoded?.records?.length ?? 0,
+            bodyPrefix: r.body == null ? null : r.body.slice(0, rawLimit),
+        };
+    });
+}
+export default defineSite({
+    id: "google-ai",
+    name: "Google AI Overview",
+    domain: "google.com",
+    description: "Fetches Google's AI Overview and AI Mode answers using browser-attached Playwright.",
+    transport: "browser",
+    cookies: "optional",
+    endpoints: [{ url: "https://www.google.com" }],
+    positionals: [
+        {
+            name: "question",
+            description: "The search query or question to ask Google",
+            required: true,
+            variadic: true,
+        },
+    ],
+    parameters: [
+        { name: "raw-limit", type: "number", description: "Max raw response chars to include", default: 12000 },
+        { name: "timeout", type: "number", description: "Playwright timeout in milliseconds", default: 90000 },
+        {
+            name: "text",
+            type: "boolean",
+            description: "Print only the extracted AI Overview answer text",
+            short: "t",
+        },
+    ],
+    run: async (ctx) => {
+        const question = ctx.options.question;
+        const rawLimit = ctx.options.rawLimit !== undefined ? Number(ctx.options.rawLimit) : 12000;
+        const timeout = ctx.options.timeout !== undefined ? Number(ctx.options.timeout) : 90000;
+        const page = await ctx.browser();
+        const cdp = await page.context().newCDPSession(page);
+        await cdp.send("Network.enable", { maxTotalBufferSize: 100000000, maxResourceBufferSize: 100000000 });
+        const rows = [];
+        const byId = new Map();
+        const interesting = (url) => {
+            const path = googlePath(url);
+            return path === "/search" || path?.startsWith("/async/") || path?.includes("batchexecute");
+        };
+        cdp.on("Network.requestWillBeSent", (p) => {
+            const r = p.request || {};
+            if (!interesting(r.url))
+                return;
+            byId.set(p.requestId, rows.length);
+            rows.push({
+                id: p.requestId,
+                type: p.type,
+                method: r.method,
+                url: r.url,
+                postData: r.postData || null,
+                status: null,
+                mimeType: null,
+                body: null,
+            });
+        });
+        cdp.on("Network.responseReceived", (p) => {
+            const i = byId.get(p.requestId);
+            if (i == null)
+                return;
+            rows[i].status = p.response.status;
+            rows[i].mimeType = p.response.mimeType;
+        });
+        cdp.on("Network.loadingFinished", async (p) => {
+            const i = byId.get(p.requestId);
+            if (i == null)
+                return;
+            try {
+                const mt = rows[i].mimeType || "";
+                if (!/text|json|html|javascript|x-protobuf/.test(mt))
+                    return;
+                const b = await cdp.send("Network.getResponseBody", { requestId: p.requestId });
+                rows[i].body = b.base64Encoded ? null : (b.body || "").slice(0, rawLimit);
+            }
+            catch { }
+        });
+        const searchUrl = `https://www.google.com/search?${buildQueryString({ q: question, udm: "50" })}`;
+        if (ctx.debug)
+            console.log(`Navigating to Google Search: ${searchUrl}`);
+        await page.goto(searchUrl, { waitUntil: "domcontentloaded" });
+        const answer = await waitForAiAnswer(page, timeout);
+        const searchPage = await collectPageState(page, rawLimit);
+        const endpoint = await buildFolwrEndpoint(page);
+        let endpointResult = null;
+        if (endpoint?.url) {
+            if (ctx.debug)
+                console.log(`Discovered folwr endpoint: ${endpoint.url}`);
+            try {
+                const responseText = await page.evaluate(async (url) => {
+                    const res = await fetch(url);
+                    return res.text();
+                }, endpoint.url);
+                const decodedAnswer = extractAnswerFromRecordStream(responseText);
+                const rawTextAnswer = extractAnswerFromText(responseText);
+                const cleanFallbackAnswer = rawTextAnswer && !rawTextAnswer.includes("<") && !rawTextAnswer.includes("class=")
+                    ? rawTextAnswer
+                    : null;
+                endpointResult = {
+                    title: searchPage.title,
+                    url: endpoint.url,
+                    bodyText: responseText,
+                    htmlPrefix: responseText.slice(0, rawLimit),
+                    answer: decodedAnswer || cleanFallbackAnswer,
+                    decoded: decodeGoogleBody({ body: responseText, contentType: "text/plain" }),
+                };
+            }
+            catch (err) {
+                if (ctx.debug)
+                    console.warn("Failed to query folwr endpoint in-page:", err);
+            }
+        }
+        await page.waitForTimeout(500);
+        const finalAnswer = endpointResult?.answer || answer || extractAnswerFromText(searchPage.bodyText);
+        return {
+            question,
+            answer: cleanText(finalAnswer, rawLimit) || null,
+            finalUrl: page.url(),
+            endpoint: endpoint || null,
+            searchPage: {
+                title: searchPage.title,
+                url: searchPage.url,
+                bodyText: cleanText(searchPage.bodyText, rawLimit),
+            },
+            endpointResult: endpointResult
+                ? {
+                    title: endpointResult.title,
+                    url: endpointResult.url,
+                    bodyText: cleanText(endpointResult.bodyText, rawLimit),
+                    htmlPrefix: endpointResult.htmlPrefix,
+                    decoded: endpointResult.decoded,
+                }
+                : null,
+            requests: summarizeNetwork(rows, rawLimit),
+        };
+    },
+});

package/dist/src/sites/google.com/openapi.yaml

@@ -0,0 +1,59 @@
+# Generated by `pnpm generate:openapi` — do not edit by hand.
+openapi: 3.1.0
+info:
+  title: Google AI Overview
+  description: Fetches Google's AI Overview and AI Mode answers using browser-attached Playwright.
+  version: 1.1.3
+servers:
+  - url: https://google.com
+  - url: https://www.google.com
+paths:
+  /:
+    servers:
+      - url: https://www.google.com
+    get:
+      summary: "Google AI Overview: GET /"
+      description: Fetches Google's AI Overview and AI Mode answers using browser-attached Playwright.
+      operationId: google_ai_get__
+      responses:
+        "200":
+          description: JSON response body (shape defined by the site, see its transform)
+      security: []
+components:
+  securitySchemes:
+    chromeSession:
+      type: apiKey
+      in: cookie
+      name: session
+      description: "Authenticated via the user's real Chrome session: website-api injects decrypted Chrome
+        cookies for google.com into every request."
+x-website-api:
+  id: google-ai
+  domain: google.com
+  cookieDomain: google.com
+  transport: browser
+  cookies: optional
+  requiresLogin: false
+  imperative: false
+  cli:
+    command: website-api google-ai
+    positionals:
+      - name: question
+        description: The search query or question to ask Google
+        required: true
+        variadic: true
+    parameters:
+      - flag: --raw-limit
+        type: number
+        description: Max raw response chars to include
+        default: 12000
+        required: false
+      - flag: --timeout
+        type: number
+        description: Playwright timeout in milliseconds
+        default: 90000
+        required: false
+      - flag: --text
+        type: boolean
+        description: Print only the extracted AI Overview answer text
+        required: false

package/dist/src/sites/ollama.com/index.js

@@ -1 +1,43 @@
-import{defineSite as e}from"../../core/define-site.js";export function extractPlan(e){return e.match(/Cloud Usage[\s\S]*?<\/span>[\s\S]*?<span[^>]*>([\s\S]*?)<\/span/i)?.[1]?.trim()??"unknown"}export function extractUsage(e,a){const s=a.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");const t=new RegExp(`<div[\\s\\S]*?<span[^>]*>\\s*${s}\\s*<\\/span>[\\s\\S]*?aria-label="${s}\\s+([^"]+)"[\\s\\S]*?data-time="([^"]+)"`,"i"),n=e.match(t);return{usage:n?.[1]?.replace(/\s+used$/i,"").trim()??"unknown",reset:n?.[2]?.trim()??"unknown"}}export function parseOllamaUsage(e){const a=extractUsage(e,"Session usage"),s=extractUsage(e,"Weekly usage");return{time:(new Date).toISOString(),Plan:extractPlan(e),"Session Usage":a.usage,"Session Reset":a.reset,"Weekly Usage":s.usage,"Weekly Reset":s.reset}}export default e({id:"ollama-usage",name:"Ollama Usage",domain:"ollama.com",description:"Fetches Ollama plan and usage details from the authenticated settings page.",endpoints:[{url:"https://ollama.com/settings",responseType:"html",transform:e=>parseOllamaUsage("string"==typeof e?e:String(e))}]});
+import { defineSite } from "../../core/define-site.js";
+function escapeRegex(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+export function extractPlan(html) {
+    const pattern = /Cloud Usage[\s\S]*?<\/span>[\s\S]*?<span[^>]*>([\s\S]*?)<\/span/i;
+    return html.match(pattern)?.[1]?.trim() ?? "unknown";
+}
+export function extractUsage(html, label) {
+    const escapedLabel = escapeRegex(label);
+    const pattern = new RegExp(`<div[\\s\\S]*?<span[^>]*>\\s*${escapedLabel}\\s*<\\/span>[\\s\\S]*?aria-label="${escapedLabel}\\s+([^"]+)"[\\s\\S]*?data-time="([^"]+)"`, "i");
+    const match = html.match(pattern);
+    return {
+        usage: match?.[1]?.replace(/\s+used$/i, "").trim() ?? "unknown",
+        reset: match?.[2]?.trim() ?? "unknown",
+    };
+}
+/** Parses the Ollama settings HTML into a usage summary. Pure + testable. */
+export function parseOllamaUsage(html) {
+    const sessionUsage = extractUsage(html, "Session usage");
+    const weeklyUsage = extractUsage(html, "Weekly usage");
+    return {
+        time: new Date().toISOString(),
+        Plan: extractPlan(html),
+        "Session Usage": sessionUsage.usage,
+        "Session Reset": sessionUsage.reset,
+        "Weekly Usage": weeklyUsage.usage,
+        "Weekly Reset": weeklyUsage.reset,
+    };
+}
+export default defineSite({
+    id: "ollama-usage",
+    name: "Ollama Usage",
+    domain: "ollama.com",
+    description: "Fetches Ollama plan and usage details from the authenticated settings page.",
+    endpoints: [
+        {
+            url: "https://ollama.com/settings",
+            responseType: "html",
+            transform: (body) => parseOllamaUsage(typeof body === "string" ? body : String(body)),
+        },
+    ],
+});