surf-cli 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/content/accessibility-tree.js +8 -8
- package/dist/content/accessibility-tree.js.map +1 -1
- package/dist/content/visual-indicator.js +17 -17
- package/dist/content/visual-indicator.js.map +1 -1
- package/dist/manifest.json +1 -1
- package/dist/service-worker/index.js +27 -27
- package/dist/service-worker/index.js.map +1 -1
- package/native/cli.cjs +12 -0
- package/native/gemini-client.cjs +304 -63
- package/native/host.cjs +51 -0
- package/package.json +8 -8
- package/skills/README.md +2 -2
- package/skills/surf/SKILL.md +40 -10
package/native/cli.cjs
CHANGED
|
@@ -2655,6 +2655,18 @@ delete toolArgs.output;
|
|
|
2655
2655
|
if (tool === "aistudio.build" && outputPath) {
|
|
2656
2656
|
toolArgs.output = path.resolve(outputPath);
|
|
2657
2657
|
}
|
|
2658
|
+
if (tool === "gemini") {
|
|
2659
|
+
if (outputPath) toolArgs.output = path.resolve(outputPath);
|
|
2660
|
+
if (toolArgs["generate-image"] && typeof toolArgs["generate-image"] === "string") {
|
|
2661
|
+
toolArgs["generate-image"] = path.resolve(toolArgs["generate-image"]);
|
|
2662
|
+
}
|
|
2663
|
+
if (toolArgs["edit-image"] && typeof toolArgs["edit-image"] === "string") {
|
|
2664
|
+
toolArgs["edit-image"] = path.resolve(toolArgs["edit-image"]);
|
|
2665
|
+
}
|
|
2666
|
+
if (toolArgs.file && typeof toolArgs.file === "string") {
|
|
2667
|
+
toolArgs.file = path.resolve(toolArgs.file);
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2658
2670
|
|
|
2659
2671
|
if (tool === "screenshot" && outputPath) {
|
|
2660
2672
|
if (typeof outputPath !== "string") {
|
package/native/gemini-client.cjs
CHANGED
|
@@ -143,7 +143,15 @@ function httpsGet(url, headers, opts = {}) {
|
|
|
143
143
|
}
|
|
144
144
|
|
|
145
145
|
function httpsPost(url, headers, body, opts = {}) {
|
|
146
|
-
|
|
146
|
+
return httpsSend("POST", url, headers, body, opts);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function httpsPut(url, headers, body, opts = {}) {
|
|
150
|
+
return httpsSend("PUT", url, headers, body, opts);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function httpsSend(method, url, headers, body, opts = {}) {
|
|
154
|
+
const { timeoutMs = 30000, log = null, label = "httpsSend" } = opts;
|
|
147
155
|
return new Promise((resolve, reject) => {
|
|
148
156
|
const urlObj = new URL(url);
|
|
149
157
|
const bodyBuffer = body == null
|
|
@@ -156,7 +164,7 @@ function httpsPost(url, headers, body, opts = {}) {
|
|
|
156
164
|
hostname: urlObj.hostname,
|
|
157
165
|
port: 443,
|
|
158
166
|
path: urlObj.pathname + urlObj.search,
|
|
159
|
-
method
|
|
167
|
+
method,
|
|
160
168
|
headers: {
|
|
161
169
|
"user-agent": USER_AGENT,
|
|
162
170
|
...headers,
|
|
@@ -224,6 +232,32 @@ async function fetchGeminiAccessToken(cookieMap, opts = {}) {
|
|
|
224
232
|
}
|
|
225
233
|
|
|
226
234
|
function trimGeminiJsonEnvelope(text) {
|
|
235
|
+
// Handle streaming chunk format: )]}\n\n<size>\n<json>\n<size>\n<json>...
|
|
236
|
+
const lines = text.split("\n");
|
|
237
|
+
const chunks = [];
|
|
238
|
+
for (let i = 0; i < lines.length; i++) {
|
|
239
|
+
const line = lines[i].trim();
|
|
240
|
+
if (!line || line === ")]}'" || /^\d+$/.test(line)) continue;
|
|
241
|
+
if (line.startsWith("[")) {
|
|
242
|
+
chunks.push(line);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (chunks.length > 1) {
|
|
247
|
+
const merged = [];
|
|
248
|
+
for (const chunk of chunks) {
|
|
249
|
+
try {
|
|
250
|
+
const parsed = JSON.parse(chunk);
|
|
251
|
+
if (Array.isArray(parsed)) {
|
|
252
|
+
merged.push(...parsed);
|
|
253
|
+
}
|
|
254
|
+
} catch {
|
|
255
|
+
// ignore
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return JSON.stringify(merged);
|
|
259
|
+
}
|
|
260
|
+
|
|
227
261
|
const start = text.indexOf("[");
|
|
228
262
|
const end = text.lastIndexOf("]");
|
|
229
263
|
if (start === -1 || end === -1 || end <= start) {
|
|
@@ -347,32 +381,48 @@ function parseGeminiStreamGenerateResponse(rawText) {
|
|
|
347
381
|
// File Upload
|
|
348
382
|
// ============================================================================
|
|
349
383
|
|
|
350
|
-
async function uploadGeminiFile(filePath, opts = {}) {
|
|
384
|
+
async function uploadGeminiFile(filePath, cookieMap, opts = {}) {
|
|
351
385
|
const absPath = path.resolve(process.cwd(), filePath);
|
|
352
386
|
const data = fs.readFileSync(absPath);
|
|
353
387
|
const fileName = path.basename(absPath);
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
const
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
Buffer.from(header, "utf-8"),
|
|
362
|
-
data,
|
|
363
|
-
Buffer.from(footer, "utf-8"),
|
|
364
|
-
]);
|
|
365
|
-
|
|
366
|
-
const res = await httpsPost(GEMINI_UPLOAD_URL, {
|
|
367
|
-
"content-type": `multipart/form-data; boundary=${boundary}`,
|
|
388
|
+
const cookieHeader = buildCookieHeader(cookieMap);
|
|
389
|
+
|
|
390
|
+
// Step 1: Initiate resumable upload
|
|
391
|
+
const initRes = await httpsPut(GEMINI_UPLOAD_URL, {
|
|
392
|
+
"authorization": "Basic c2F2ZXM6cyNMdGhlNmxzd2F2b0RsN3J1d1U=",
|
|
393
|
+
"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
|
|
394
|
+
"cookie": cookieHeader,
|
|
368
395
|
"push-id": GEMINI_UPLOAD_PUSH_ID,
|
|
369
|
-
|
|
396
|
+
"referer": "https://gemini.google.com/",
|
|
397
|
+
"x-goog-upload-command": "start",
|
|
398
|
+
"x-goog-upload-header-content-length": String(data.length),
|
|
399
|
+
"x-goog-upload-header-content-type": "application/octet-stream",
|
|
400
|
+
"x-goog-upload-protocol": "resumable",
|
|
401
|
+
"x-tenant-id": "bard-storage",
|
|
402
|
+
}, data, { ...opts, label: opts.label || "geminiUploadInit" });
|
|
403
|
+
|
|
404
|
+
const uploadId = initRes.headers["x-guploader-uploadid"];
|
|
405
|
+
if (!uploadId) {
|
|
406
|
+
throw new Error(`File upload init failed: no upload ID (${initRes.status})`);
|
|
407
|
+
}
|
|
370
408
|
|
|
371
|
-
|
|
372
|
-
|
|
409
|
+
// Step 2: Upload data and finalize
|
|
410
|
+
const uploadUrl = `${GEMINI_UPLOAD_URL}?upload_id=${encodeURIComponent(uploadId)}&upload_protocol=resumable`;
|
|
411
|
+
const finalRes = await httpsPut(uploadUrl, {
|
|
412
|
+
"content-type": "application/octet-stream",
|
|
413
|
+
"cookie": cookieHeader,
|
|
414
|
+
"origin": "https://gemini.google.com",
|
|
415
|
+
"referer": "https://gemini.google.com/",
|
|
416
|
+
"x-goog-upload-command": "upload, finalize",
|
|
417
|
+
"x-goog-upload-offset": "0",
|
|
418
|
+
"x-tenant-id": "bard-storage",
|
|
419
|
+
}, data, { ...opts, label: opts.label || "geminiUpload" });
|
|
420
|
+
|
|
421
|
+
if (finalRes.status < 200 || finalRes.status >= 300) {
|
|
422
|
+
throw new Error(`File upload failed: ${finalRes.status} (${finalRes.text.slice(0, 200)})`);
|
|
373
423
|
}
|
|
374
424
|
|
|
375
|
-
return { id:
|
|
425
|
+
return { id: finalRes.text, name: fileName };
|
|
376
426
|
}
|
|
377
427
|
|
|
378
428
|
// ============================================================================
|
|
@@ -402,18 +452,51 @@ async function downloadGeminiImage(url, cookieMap, outputPath, opts = {}) {
|
|
|
402
452
|
fs.writeFileSync(outputPath, res.buffer);
|
|
403
453
|
}
|
|
404
454
|
|
|
455
|
+
async function downloadGeminiImageViaExtension(url, outputPath, opts = {}) {
|
|
456
|
+
const { fetchUrl, log } = opts;
|
|
457
|
+
const fullUrl = ensureFullSizeImageUrl(url);
|
|
458
|
+
|
|
459
|
+
const result = await fetchUrl(fullUrl);
|
|
460
|
+
if (!result || result.error) throw new Error(`Image download failed: ${result?.error || "no response"}`);
|
|
461
|
+
if (!result.b64) throw new Error("Image download returned no data");
|
|
462
|
+
|
|
463
|
+
const dir = path.dirname(outputPath);
|
|
464
|
+
if (dir && !fs.existsSync(dir)) {
|
|
465
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
fs.writeFileSync(outputPath, Buffer.from(result.b64, "base64"));
|
|
469
|
+
}
|
|
470
|
+
|
|
405
471
|
async function saveFirstGeminiImage(output, cookieMap, outputPath, opts = {}) {
|
|
406
|
-
|
|
407
|
-
const
|
|
408
|
-
|
|
409
|
-
|
|
472
|
+
const useExtensionDownload = !!opts.fetchUrl;
|
|
473
|
+
const img = output.images?.find(i => i.kind === "generated") ?? output.images?.[0];
|
|
474
|
+
|
|
475
|
+
if (img?.b64) {
|
|
476
|
+
const dir = path.dirname(outputPath);
|
|
477
|
+
if (dir && !fs.existsSync(dir)) {
|
|
478
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
479
|
+
}
|
|
480
|
+
fs.writeFileSync(outputPath, Buffer.from(img.b64, "base64"));
|
|
481
|
+
return { saved: true, imageCount: output.images.length };
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
if (img?.url) {
|
|
485
|
+
if (useExtensionDownload) {
|
|
486
|
+
await downloadGeminiImageViaExtension(img.url, outputPath, opts);
|
|
487
|
+
} else {
|
|
488
|
+
await downloadGeminiImage(img.url, cookieMap, outputPath, opts);
|
|
489
|
+
}
|
|
410
490
|
return { saved: true, imageCount: output.images.length };
|
|
411
491
|
}
|
|
412
492
|
|
|
413
|
-
|
|
414
|
-
const ggdl = extractGgdlUrls(output.rawResponseText);
|
|
493
|
+
const ggdl = extractGgdlUrls(output.rawResponseText || "");
|
|
415
494
|
if (ggdl[0]) {
|
|
416
|
-
|
|
495
|
+
if (useExtensionDownload) {
|
|
496
|
+
await downloadGeminiImageViaExtension(ggdl[0], outputPath, opts);
|
|
497
|
+
} else {
|
|
498
|
+
await downloadGeminiImage(ggdl[0], cookieMap, outputPath, opts);
|
|
499
|
+
}
|
|
417
500
|
return { saved: true, imageCount: ggdl.length };
|
|
418
501
|
}
|
|
419
502
|
|
|
@@ -426,7 +509,7 @@ async function saveFirstGeminiImage(output, cookieMap, outputPath, opts = {}) {
|
|
|
426
509
|
|
|
427
510
|
function buildGeminiFReqPayload(prompt, uploaded, chatMetadata) {
|
|
428
511
|
const promptPayload = uploaded.length > 0
|
|
429
|
-
? [prompt, 0, null, uploaded.map(file => [[file.id, 1]])]
|
|
512
|
+
? [prompt, 0, null, uploaded.map(file => [[file.id, 1], file.name])]
|
|
430
513
|
: [prompt];
|
|
431
514
|
|
|
432
515
|
const innerList = [promptPayload, null, chatMetadata ?? null];
|
|
@@ -443,7 +526,7 @@ async function runGeminiWebOnce(input) {
|
|
|
443
526
|
// 2. Upload files
|
|
444
527
|
const uploaded = [];
|
|
445
528
|
for (const file of files ?? []) {
|
|
446
|
-
uploaded.push(await uploadGeminiFile(file, { timeoutMs, log, label: "geminiUpload" }));
|
|
529
|
+
uploaded.push(await uploadGeminiFile(file, cookieMap, { timeoutMs, log, label: "geminiUpload" }));
|
|
447
530
|
}
|
|
448
531
|
|
|
449
532
|
// 3. Build request
|
|
@@ -519,6 +602,137 @@ async function runGeminiWebWithFallback(input) {
|
|
|
519
602
|
return { ...attempt, effectiveModel: input.model };
|
|
520
603
|
}
|
|
521
604
|
|
|
605
|
+
// ============================================================================
|
|
606
|
+
// In-Page Execution (for image generation)
|
|
607
|
+
// ============================================================================
|
|
608
|
+
|
|
609
|
+
async function runGeminiWebViaPage(input) {
|
|
610
|
+
const { prompt, files, model, timeoutMs = 120000, log = null, createTab, closeTab, jsEval, fetchUrl, uploadFile } = input;
|
|
611
|
+
|
|
612
|
+
if (!createTab || !closeTab || !jsEval) {
|
|
613
|
+
throw new Error("In-page execution requires createTab, closeTab, and jsEval callbacks");
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
let tabId = null;
|
|
617
|
+
try {
|
|
618
|
+
if (log) log("Creating Gemini tab...");
|
|
619
|
+
const tabResult = await createTab();
|
|
620
|
+
tabId = tabResult?.tabId;
|
|
621
|
+
if (!tabId) throw new Error("Failed to create Gemini tab");
|
|
622
|
+
if (log) log(`Gemini tab created: ${tabId}`);
|
|
623
|
+
await new Promise(r => setTimeout(r, 12000));
|
|
624
|
+
|
|
625
|
+
if (files?.length && uploadFile) {
|
|
626
|
+
const absFiles = files.map(f => path.resolve(process.cwd(), f));
|
|
627
|
+
if (log) log(`Uploading ${absFiles.length} file(s) via file chooser...`);
|
|
628
|
+
const result = await uploadFile(tabId, absFiles);
|
|
629
|
+
if (result?.error) throw new Error(`File upload failed: ${result.error}`);
|
|
630
|
+
if (log) log("File uploaded, waiting for processing...");
|
|
631
|
+
await new Promise(r => setTimeout(r, 3000));
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
const checkJsResult = (result, context) => {
|
|
635
|
+
if (result?.error) throw new Error(`${context}: ${result.error}`);
|
|
636
|
+
if (result?.output === undefined) throw new Error(`${context}: no output`);
|
|
637
|
+
return result.output;
|
|
638
|
+
};
|
|
639
|
+
|
|
640
|
+
// Type prompt
|
|
641
|
+
const fullPrompt = prompt.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/\n/g, "\\n").replace(/\r/g, "\\r");
|
|
642
|
+
if (log) log("Typing prompt...");
|
|
643
|
+
const typeResult = await jsEval(tabId, `
|
|
644
|
+
const editor = document.querySelector('.ql-editor[contenteditable=true]');
|
|
645
|
+
if (!editor) return JSON.stringify({ error: "No editor found on page" });
|
|
646
|
+
editor.focus();
|
|
647
|
+
document.execCommand('selectAll', false, null);
|
|
648
|
+
document.execCommand('insertText', false, '${fullPrompt}');
|
|
649
|
+
return JSON.stringify({ ok: true, len: editor.textContent.length });
|
|
650
|
+
`);
|
|
651
|
+
const typed = JSON.parse(JSON.parse(checkJsResult(typeResult, "Type prompt")));
|
|
652
|
+
if (typed.error) throw new Error(typed.error);
|
|
653
|
+
|
|
654
|
+
const beforeResult = await jsEval(tabId, `
|
|
655
|
+
return String(Array.from(document.querySelectorAll('img[src*="gg-dl"]')).filter(i => i.naturalWidth >= 512).length);
|
|
656
|
+
`);
|
|
657
|
+
const imgCountBefore = parseInt(JSON.parse(checkJsResult(beforeResult, "Count images")) || "0", 10);
|
|
658
|
+
|
|
659
|
+
if (log) log("Submitting...");
|
|
660
|
+
const sendResult = await jsEval(tabId, `
|
|
661
|
+
const btn = document.querySelector('button[aria-label="Send message"]');
|
|
662
|
+
if (!btn) return 'no-btn';
|
|
663
|
+
btn.click();
|
|
664
|
+
return 'sent';
|
|
665
|
+
`);
|
|
666
|
+
const sendVal = JSON.parse(checkJsResult(sendResult, "Click send"));
|
|
667
|
+
if (sendVal === "no-btn") throw new Error("Send button not found on Gemini page");
|
|
668
|
+
|
|
669
|
+
// Poll for response
|
|
670
|
+
if (log) log("Waiting for response...");
|
|
671
|
+
const deadline = Date.now() + timeoutMs;
|
|
672
|
+
let imageUrls = [];
|
|
673
|
+
let responseText = "";
|
|
674
|
+
|
|
675
|
+
while (Date.now() < deadline) {
|
|
676
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
677
|
+
const pollResult = await jsEval(tabId, `
|
|
678
|
+
const ggImgs = Array.from(document.querySelectorAll('img[src*="gg-dl"]'))
|
|
679
|
+
.filter(i => i.naturalWidth >= 512)
|
|
680
|
+
.map(i => i.src);
|
|
681
|
+
const loading = !!document.querySelector('mat-progress-bar, .loading-indicator, message-loading');
|
|
682
|
+
const turns = document.querySelectorAll('message-content');
|
|
683
|
+
const lastTurn = turns.length ? turns[turns.length - 1] : null;
|
|
684
|
+
const text = lastTurn ? lastTurn.textContent?.trim() : "";
|
|
685
|
+
return JSON.stringify({ ggImgs, loading, text, turns: turns.length });
|
|
686
|
+
`);
|
|
687
|
+
const poll = JSON.parse(JSON.parse(checkJsResult(pollResult, "Poll response")));
|
|
688
|
+
const newImgs = poll.ggImgs.slice(imgCountBefore);
|
|
689
|
+
|
|
690
|
+
if (newImgs.length > 0) {
|
|
691
|
+
imageUrls = newImgs;
|
|
692
|
+
responseText = poll.text || "";
|
|
693
|
+
if (log) log(`Found ${newImgs.length} generated image(s)`);
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
if (!poll.loading && poll.text && poll.turns > 0) {
|
|
697
|
+
responseText = poll.text;
|
|
698
|
+
break;
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
if (!imageUrls.length && !responseText) {
|
|
703
|
+
throw new Error("Gemini response timed out");
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
// Download images via extension
|
|
707
|
+
const images = [];
|
|
708
|
+
if (fetchUrl) {
|
|
709
|
+
for (const url of imageUrls) {
|
|
710
|
+
if (log) log(`Downloading image (${url.slice(0, 60)}...)...`);
|
|
711
|
+
const dlResult = await fetchUrl(url);
|
|
712
|
+
if (dlResult?.b64) {
|
|
713
|
+
images.push({ url, b64: dlResult.b64, type: dlResult.type || "image/png" });
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
} else {
|
|
717
|
+
for (const url of imageUrls) {
|
|
718
|
+
images.push({ url });
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
return {
|
|
723
|
+
text: responseText,
|
|
724
|
+
thoughts: null,
|
|
725
|
+
metadata: null,
|
|
726
|
+
images,
|
|
727
|
+
effectiveModel: model,
|
|
728
|
+
_pageTabId: tabId,
|
|
729
|
+
};
|
|
730
|
+
} catch (err) {
|
|
731
|
+
if (tabId) { try { await closeTab(tabId); } catch {} }
|
|
732
|
+
throw err;
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
|
|
522
736
|
// ============================================================================
|
|
523
737
|
// Main Query Function
|
|
524
738
|
// ============================================================================
|
|
@@ -534,9 +748,15 @@ async function query(options) {
|
|
|
534
748
|
youtube,
|
|
535
749
|
aspectRatio,
|
|
536
750
|
getCookies,
|
|
751
|
+
createTab,
|
|
752
|
+
closeTab,
|
|
753
|
+
jsEval,
|
|
754
|
+
fetchUrl,
|
|
755
|
+
uploadFile,
|
|
537
756
|
timeout = 300000,
|
|
538
757
|
log = () => {},
|
|
539
758
|
} = options;
|
|
759
|
+
const hasPageCallbacks = !!(createTab && closeTab && jsEval);
|
|
540
760
|
|
|
541
761
|
const startTime = Date.now();
|
|
542
762
|
log("Starting Gemini query");
|
|
@@ -579,59 +799,80 @@ async function query(options) {
|
|
|
579
799
|
|
|
580
800
|
try {
|
|
581
801
|
if (editImage) {
|
|
582
|
-
//
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
802
|
+
// Image editing
|
|
803
|
+
if (!hasPageCallbacks) {
|
|
804
|
+
throw new Error("Image editing requires the Chrome extension. Make sure it's loaded.");
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
log("Uploading and editing image...");
|
|
808
|
+
const out = await runGeminiWebViaPage({
|
|
809
|
+
prompt: fullPrompt,
|
|
586
810
|
files: [editImage],
|
|
587
811
|
model: resolvedModel,
|
|
588
|
-
cookieMap,
|
|
589
|
-
chatMetadata: null,
|
|
590
|
-
timeoutMs: timeout,
|
|
591
|
-
log,
|
|
592
|
-
});
|
|
593
|
-
|
|
594
|
-
log("Sending edit request...");
|
|
595
|
-
const editPrompt = `Use image generation tool to ${fullPrompt}`;
|
|
596
|
-
const out = await runGeminiWebWithFallback({
|
|
597
|
-
prompt: editPrompt,
|
|
598
|
-
files,
|
|
599
|
-
model: resolvedModel,
|
|
600
|
-
cookieMap,
|
|
601
|
-
chatMetadata: intro.metadata,
|
|
602
812
|
timeoutMs: timeout,
|
|
603
813
|
log,
|
|
814
|
+
createTab,
|
|
815
|
+
closeTab,
|
|
816
|
+
jsEval,
|
|
817
|
+
fetchUrl,
|
|
818
|
+
uploadFile,
|
|
604
819
|
});
|
|
605
820
|
|
|
606
821
|
response = out;
|
|
607
822
|
|
|
608
823
|
// Save output image
|
|
609
824
|
const outputPath = output || generateImage || "edited.png";
|
|
610
|
-
const
|
|
611
|
-
if (
|
|
612
|
-
|
|
825
|
+
const saveOpts = { timeoutMs: timeout, log };
|
|
826
|
+
if (fetchUrl) saveOpts.fetchUrl = fetchUrl;
|
|
827
|
+
try {
|
|
828
|
+
const imageSave = await saveFirstGeminiImage(out, cookieMap, outputPath, saveOpts);
|
|
829
|
+
if (!imageSave.saved) {
|
|
830
|
+
throw new Error(`No images generated. Response: ${out.text?.slice(0, 200) || "(empty)"}`);
|
|
831
|
+
}
|
|
832
|
+
} finally {
|
|
833
|
+
if (out._pageTabId && closeTab) { try { await closeTab(out._pageTabId); } catch {} }
|
|
613
834
|
}
|
|
614
835
|
imagePath = outputPath;
|
|
615
836
|
|
|
616
837
|
} else if (generateImage) {
|
|
617
838
|
// Image generation
|
|
618
839
|
log("Generating image...");
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
840
|
+
let out;
|
|
841
|
+
if (hasPageCallbacks) {
|
|
842
|
+
out = await runGeminiWebViaPage({
|
|
843
|
+
prompt: fullPrompt,
|
|
844
|
+
model: resolvedModel,
|
|
845
|
+
timeoutMs: timeout,
|
|
846
|
+
log,
|
|
847
|
+
createTab,
|
|
848
|
+
closeTab,
|
|
849
|
+
jsEval,
|
|
850
|
+
fetchUrl,
|
|
851
|
+
});
|
|
852
|
+
} else {
|
|
853
|
+
out = await runGeminiWebWithFallback({
|
|
854
|
+
prompt: fullPrompt,
|
|
855
|
+
files,
|
|
856
|
+
model: resolvedModel,
|
|
857
|
+
cookieMap,
|
|
858
|
+
chatMetadata: null,
|
|
859
|
+
timeoutMs: timeout,
|
|
860
|
+
log,
|
|
861
|
+
});
|
|
862
|
+
}
|
|
628
863
|
|
|
629
864
|
response = out;
|
|
630
865
|
|
|
631
866
|
// Save output image
|
|
632
|
-
const
|
|
633
|
-
if (
|
|
634
|
-
|
|
867
|
+
const saveOpts = { timeoutMs: timeout, log };
|
|
868
|
+
if (fetchUrl) saveOpts.fetchUrl = fetchUrl;
|
|
869
|
+
try {
|
|
870
|
+
const imageSave = await saveFirstGeminiImage(out, cookieMap, generateImage, saveOpts);
|
|
871
|
+
if (!imageSave.saved) {
|
|
872
|
+
throw new Error(`No images generated. Response: ${out.text?.slice(0, 200) || "(empty)"}`);
|
|
873
|
+
}
|
|
874
|
+
} finally {
|
|
875
|
+
if (out._pageTabId && closeTab) { try { await closeTab(out._pageTabId); } catch {} }
|
|
635
876
|
}
|
|
636
877
|
imagePath = generateImage;
|
|
637
878
|
|
package/native/host.cjs
CHANGED
|
@@ -697,6 +697,57 @@ function handleToolRequest(msg, socket) {
|
|
|
697
697
|
});
|
|
698
698
|
writeMessage({ type: "GET_GOOGLE_COOKIES", id: cookieId });
|
|
699
699
|
}),
|
|
700
|
+
createTab: () => new Promise((resolve) => {
|
|
701
|
+
const tabCreateId = ++requestCounter;
|
|
702
|
+
pendingToolRequests.set(tabCreateId, {
|
|
703
|
+
socket: null,
|
|
704
|
+
originalId: null,
|
|
705
|
+
tool: "create_tab",
|
|
706
|
+
onComplete: (r) => resolve(r)
|
|
707
|
+
});
|
|
708
|
+
writeMessage({ type: "GEMINI_NEW_TAB", id: tabCreateId });
|
|
709
|
+
}),
|
|
710
|
+
closeTab: (tabIdToClose) => new Promise((resolve) => {
|
|
711
|
+
const tabCloseId = ++requestCounter;
|
|
712
|
+
pendingToolRequests.set(tabCloseId, {
|
|
713
|
+
socket: null,
|
|
714
|
+
originalId: null,
|
|
715
|
+
tool: "close_tab",
|
|
716
|
+
onComplete: (r) => resolve(r)
|
|
717
|
+
});
|
|
718
|
+
writeMessage({ type: "GEMINI_CLOSE_TAB", tabId: tabIdToClose, id: tabCloseId });
|
|
719
|
+
}),
|
|
720
|
+
jsEval: (tabId, code) => new Promise((resolve) => {
|
|
721
|
+
const jsId = ++requestCounter;
|
|
722
|
+
pendingToolRequests.set(jsId, {
|
|
723
|
+
socket: null,
|
|
724
|
+
originalId: null,
|
|
725
|
+
tool: "js_eval",
|
|
726
|
+
onComplete: (r) => resolve(r)
|
|
727
|
+
});
|
|
728
|
+
log(`[gemini] Sending EXECUTE_JAVASCRIPT id=${jsId} tabId=${tabId} code=${code.length} chars`);
|
|
729
|
+
writeMessage({ type: "EXECUTE_JAVASCRIPT", tabId, code, id: jsId });
|
|
730
|
+
}),
|
|
731
|
+
uploadFile: (tabId, filePaths) => new Promise((resolve) => {
|
|
732
|
+
const uploadId = ++requestCounter;
|
|
733
|
+
pendingToolRequests.set(uploadId, {
|
|
734
|
+
socket: null,
|
|
735
|
+
originalId: null,
|
|
736
|
+
tool: "upload_file",
|
|
737
|
+
onComplete: (r) => resolve(r)
|
|
738
|
+
});
|
|
739
|
+
writeMessage({ type: "UPLOAD_FILE_TO_TAB", tabId, filePaths, id: uploadId });
|
|
740
|
+
}),
|
|
741
|
+
fetchUrl: (url) => new Promise((resolve) => {
|
|
742
|
+
const fetchId = ++requestCounter;
|
|
743
|
+
pendingToolRequests.set(fetchId, {
|
|
744
|
+
socket: null,
|
|
745
|
+
originalId: null,
|
|
746
|
+
tool: "fetch_url",
|
|
747
|
+
onComplete: (r) => resolve(r)
|
|
748
|
+
});
|
|
749
|
+
writeMessage({ type: "GEMINI_FETCH_URL", url, id: fetchId });
|
|
750
|
+
}),
|
|
700
751
|
log: (msg) => log(`[gemini] ${msg}`)
|
|
701
752
|
});
|
|
702
753
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "surf-cli",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.7.0",
|
|
4
4
|
"description": "CLI for AI agents to control Chrome. Zero config, agent-agnostic, battle-tested.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"chrome",
|
|
@@ -51,21 +51,21 @@
|
|
|
51
51
|
},
|
|
52
52
|
"dependencies": {
|
|
53
53
|
"@google/generative-ai": "^0.24.1",
|
|
54
|
-
"@modelcontextprotocol/sdk": "^1.
|
|
54
|
+
"@modelcontextprotocol/sdk": "^1.26.0",
|
|
55
55
|
"buffer": "^6.0.3",
|
|
56
56
|
"crypto-browserify": "^3.12.1",
|
|
57
57
|
"events": "^3.3.0",
|
|
58
58
|
"stream-browserify": "^3.0.0",
|
|
59
59
|
"vite-plugin-node-polyfills": "^0.25.0",
|
|
60
|
-
"zod": "^4.3.
|
|
60
|
+
"zod": "^4.3.6"
|
|
61
61
|
},
|
|
62
62
|
"devDependencies": {
|
|
63
|
-
"@biomejs/biome": "^2.
|
|
64
|
-
"@types/chrome": "^0.
|
|
65
|
-
"@vitest/coverage-v8": "^4.0.
|
|
66
|
-
"@vitest/ui": "^4.0.
|
|
63
|
+
"@biomejs/biome": "^2.4.4",
|
|
64
|
+
"@types/chrome": "^0.1.37",
|
|
65
|
+
"@vitest/coverage-v8": "^4.0.18",
|
|
66
|
+
"@vitest/ui": "^4.0.18",
|
|
67
67
|
"typescript": "^5.7.2",
|
|
68
68
|
"vite": "^7.3.1",
|
|
69
|
-
"vitest": "^4.0.
|
|
69
|
+
"vitest": "^4.0.18"
|
|
70
70
|
}
|
|
71
71
|
}
|
package/skills/README.md
CHANGED
|
@@ -8,10 +8,10 @@ To use the surf skill with [Pi coding agent](https://github.com/badlogic/pi-mono
|
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
10
|
# Option 1: Symlink (auto-updates)
|
|
11
|
-
ln -s "$(pwd)/skills/surf" ~/.
|
|
11
|
+
ln -s "$(pwd)/skills/surf" ~/.agents/skills/surf
|
|
12
12
|
|
|
13
13
|
# Option 2: Copy
|
|
14
|
-
cp -r skills/surf ~/.
|
|
14
|
+
cp -r skills/surf ~/.agents/skills/
|
|
15
15
|
```
|
|
16
16
|
|
|
17
17
|
The skill will be available when pi detects browser automation tasks.
|
package/skills/surf/SKILL.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: surf
|
|
3
|
-
description: Control Chrome browser via CLI for testing, automation, and debugging. Use when the user needs browser automation, screenshots, form filling, page inspection, network/CPU emulation, DevTools streaming, or AI queries via ChatGPT/Gemini/Perplexity/Grok.
|
|
3
|
+
description: Control Chrome browser via CLI for testing, automation, and debugging. Use when the user needs browser automation, screenshots, form filling, page inspection, network/CPU emulation, DevTools streaming, or AI queries via ChatGPT/Gemini/Perplexity/Grok/AI Studio.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Surf Browser Automation
|
|
@@ -87,13 +87,42 @@ surf grok --validate
|
|
|
87
87
|
surf grok --validate --save-models
|
|
88
88
|
```
|
|
89
89
|
|
|
90
|
+
### AI Studio (via aistudio.google.com - requires Google login in Chrome)
|
|
91
|
+
```bash
|
|
92
|
+
surf aistudio "explain quantum computing"
|
|
93
|
+
surf aistudio "redteam this" --with-page # Include current page context
|
|
94
|
+
surf aistudio "quick answer" --model gemini-3-flash-preview # Model selection
|
|
95
|
+
surf aistudio "analyze" --timeout 600 # Custom timeout (default: 300s)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
**Why AI Studio over Gemini?** AI Studio gives access to less restricted Gemini models. For Gemini 3 Pro the difference can be significant with certain prompts. Downside: aggressive per-day rate limits on Pro and Flash models.
|
|
99
|
+
|
|
100
|
+
**Model selection is best-effort:** Pass any AI Studio model id (e.g. `gemini-3.1-pro-preview`, `gemini-3-flash-preview`, `gemini-flash-lite-latest`). If the model isn't found, AI Studio uses whatever model was last selected in the UI.
|
|
101
|
+
|
|
102
|
+
### AI Studio App Builder
|
|
103
|
+
```bash
|
|
104
|
+
surf aistudio.build "build a portfolio site"
|
|
105
|
+
surf aistudio.build "todo app" --model gemini-3.1-pro-preview # Model override
|
|
106
|
+
surf aistudio.build "crm dashboard" --output ./out # Extract zip to directory
|
|
107
|
+
surf aistudio.build "game" --keep-open --timeout 600 # Keep tab open, 10min timeout
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Automates AI Studio's App Builder at `aistudio.google.com/apps`. Types your prompt, clicks Build, waits for completion, downloads the generated zip, and optionally extracts it.
|
|
111
|
+
|
|
112
|
+
- `--output <dir>` extracts the zip to a directory
|
|
113
|
+
- `--model <id>` overrides the model in Advanced Settings
|
|
114
|
+
- `--timeout <seconds>` build timeout (default: 600s)
|
|
115
|
+
- `--keep-open` leaves the AI Studio tab open after completion
|
|
116
|
+
|
|
117
|
+
Returns `zipPath`, `extractedPath`, `model`, `buildDuration`, and `tookMs`.
|
|
118
|
+
|
|
90
119
|
### AI Tool Troubleshooting
|
|
91
120
|
|
|
92
121
|
When AI queries fail, check these common issues:
|
|
93
122
|
|
|
94
|
-
1. **Not logged in**: The error "login required" means you need to log into the service in Chrome
|
|
123
|
+
1. **Not logged in**: The error "login required" means you need to log into the service in Chrome (chatgpt.com, gemini.google.com, perplexity.ai, x.com, or aistudio.google.com)
|
|
95
124
|
2. **Model selection failed**: The UI may have changed. Run `surf grok --validate` to check
|
|
96
|
-
3. **Response timeout**: Thinking models (ChatGPT o1, Grok thinking) can take 45+ seconds
|
|
125
|
+
3. **Response timeout**: Thinking models (ChatGPT o1, Grok thinking) can take 45+ seconds. AI Studio builds can take several minutes.
|
|
97
126
|
4. **Element not found**: The service's UI changed. Check for surf-cli updates
|
|
98
127
|
|
|
99
128
|
**Debugging workflow for agents:**
|
|
@@ -527,14 +556,15 @@ surf wait.element ".missing" --auto-capture --timeout 2000
|
|
|
527
556
|
3. **JS method for contenteditable** - Modern editors (ChatGPT, Claude, Notion) need `--method js`
|
|
528
557
|
4. **Named tabs for workflows** - `tab.name app` then `tab.switch app`
|
|
529
558
|
5. **Auto-capture for debugging** - `--auto-capture` saves diagnostics on failure
|
|
530
|
-
6. **AI tools use browser session** - Must be logged into the service, no API keys needed
|
|
559
|
+
6. **AI tools use browser session** - Must be logged into the service (ChatGPT, Gemini, Perplexity, Grok, AI Studio), no API keys needed
|
|
531
560
|
7. **Grok validation** - Run `surf grok --validate` if queries fail to check UI changes
|
|
532
|
-
8. **Long timeouts for thinking models** - ChatGPT o1, Grok thinking can take 60+ seconds
|
|
533
|
-
9. **
|
|
534
|
-
10. **
|
|
535
|
-
11. **
|
|
536
|
-
12. **
|
|
537
|
-
13. **
|
|
561
|
+
8. **Long timeouts for thinking models** - ChatGPT o1, Grok thinking can take 60+ seconds. AI Studio builds default to 600s.
|
|
562
|
+
9. **AI Studio for unrestricted Gemini** - `surf aistudio` gives less filtered responses than `surf gemini` for the same models
|
|
563
|
+
10. **Use `surf do` for multi-step tasks** - Reduces token overhead and improves reliability
|
|
564
|
+
11. **Dry-run workflows first** - `surf do '...' --dry-run` validates without executing
|
|
565
|
+
12. **Window isolation** - Use `window.new` + `--window-id` to keep agent work separate from your browsing
|
|
566
|
+
13. **Semantic locators** - `locate.role`, `locate.text`, `locate.label` for more robust element finding
|
|
567
|
+
14. **Frame context** - Use `frame.switch` before interacting with iframe content
|
|
538
568
|
|
|
539
569
|
## Socket API
|
|
540
570
|
|