scavio 0.38.0 → 0.39.1

package/README.md

@@ -24,6 +24,40 @@ Structured JSON in, structured JSON out. No per-site parsers to maintain.
 npm install scavio
 ```
 
+## Sign in without copying a key
+
+```bash
+npx -y scavio@latest login
+```
+
+Prints a link and a code and opens your browser. Sign in at dashboard.scavio.dev
+(or create a free account, 50 credits, no card), check that the page shows the
+same code, and click Approve. The CLI saves an API key named after your machine
+(`CLI - <hostname>`) and never prints it:
+
+- `~/.config/scavio/credentials.json` (owner-only), and
+- `SCAVIO_API_KEY` in your shell startup file (`~/.zshrc` or `$ZDOTDIR/.zshrc`, `~/.bashrc` and on
+  macOS `~/.bash_profile`, fish `conf.d/scavio.fish`, or the Windows user
+  environment). Skip this with `--no-profile`.
+
+Logging in again on the same machine reuses that machine's key. Open a new
+terminal afterwards; `new Scavio()` reads `SCAVIO_API_KEY`.
+
+AI coding agents (Claude Code, Codex, Pi, Cursor and others) can run it for you.
+When its output is not a terminal, `login` prints the link and exits at once so
+the agent can show it to you; the agent then runs `login --wait`, which waits up
+to 60 seconds per run.
+
+| Command | What it does |
+|---|---|
+| `scavio login` | Start a login (in a terminal it also waits for approval) |
+| `scavio login --wait` | Finish a login an agent started. Exit 0 logged in, 2 still pending (run again), 3 denied, 1 expired or error |
+| `scavio status` | Check the saved or exported key works, without printing it |
+| `scavio logout` | Remove the saved key and the shell line (revoke it at dashboard.scavio.dev/api-keys) |
+
+Options: `--force` (log in even if `SCAVIO_API_KEY` already works),
+`--no-profile`, `--no-browser`, `--timeout <seconds>` for `--wait`.
+
 ## Quick Start
 
 ```typescript

package/dist/cli.js

@@ -0,0 +1,441 @@
+#!/usr/bin/env node
+
+// src/cli/index.ts
+import { spawn } from "child_process";
+import { homedir as homedir3 } from "os";
+
+// src/http.ts
+var BASE_URL = "https://api.scavio.dev";
+var SDK_VERSION = true ? "0.39.1" : "0.0.0";
+var USER_AGENT = `scavio-js/${SDK_VERSION}`;
+
+// src/cli/files.ts
+import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
+import { homedir } from "os";
+import { join } from "path";
+function configDir(env = process.env, platform = process.platform, home = homedir()) {
+  if (platform === "win32") return join(env.APPDATA || join(home, "AppData", "Roaming"), "scavio");
+  return join(env.XDG_CONFIG_HOME || join(home, ".config"), "scavio");
+}
+function writeSecret(path, dir, data) {
+  mkdirSync(dir, { recursive: true, mode: 448 });
+  try {
+    chmodSync(dir, 448);
+  } catch {
+  }
+  writeFileSync(path, JSON.stringify(data, null, 2) + "\n", { mode: 384 });
+  try {
+    chmodSync(path, 384);
+  } catch {
+  }
+}
+function readJson(path) {
+  if (!existsSync(path)) return null;
+  try {
+    return JSON.parse(readFileSync(path, "utf8"));
+  } catch {
+    return null;
+  }
+}
+var credentialsPath = (dir) => join(dir, "credentials.json");
+var pendingPath = (dir) => join(dir, "pending-login.json");
+function saveCredentials(dir, c) {
+  const p = credentialsPath(dir);
+  writeSecret(p, dir, c);
+  return p;
+}
+function readCredentials(dir) {
+  const c = readJson(credentialsPath(dir));
+  return c && typeof c.api_key === "string" ? c : null;
+}
+function savePending(dir, p) {
+  writeSecret(pendingPath(dir), dir, p);
+}
+function readPending(dir) {
+  const p = readJson(pendingPath(dir));
+  return p && typeof p.device_code === "string" ? p : null;
+}
+function clearPending(dir) {
+  rmSync(pendingPath(dir), { force: true });
+}
+function clearCredentials(dir) {
+  const p = credentialsPath(dir);
+  const had = existsSync(p);
+  rmSync(p, { force: true });
+  return had;
+}
+
+// src/cli/login.ts
+import { hostname as osHostname } from "os";
+function machineName(raw = osHostname()) {
+  return raw.split(".")[0].slice(0, 64) || "unknown";
+}
+async function startLogin(baseUrl, client, f = fetch) {
+  const res = await f(`${baseUrl}/api/v1/auth/device/code`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", "User-Agent": client.userAgent },
+    body: JSON.stringify({ hostname: client.hostname, os: client.os })
+  });
+  const body = await res.json().catch(() => ({}));
+  if (!res.ok || typeof body.device_code !== "string") {
+    throw new Error(body.error_description || `Could not start login (HTTP ${res.status}).`);
+  }
+  return body;
+}
+async function waitForApproval(opts) {
+  const f = opts.fetch ?? fetch;
+  const now = opts.now ?? Date.now;
+  const sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  const deadline = Math.min(now() + opts.budgetMs, opts.expiresAt);
+  let interval = Math.max(1, opts.intervalSec) * 1e3;
+  let netFailures = 0;
+  for (; ; ) {
+    if (now() >= opts.expiresAt) return { kind: "expired" };
+    let res;
+    let body;
+    try {
+      res = await f(`${opts.baseUrl}/api/v1/auth/device/token`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "User-Agent": opts.userAgent },
+        body: JSON.stringify({ device_code: opts.deviceCode })
+      });
+      body = await res.json().catch(() => ({}));
+      netFailures = 0;
+    } catch {
+      netFailures++;
+      const backoff2 = Math.min(interval * 2 ** netFailures, 3e4);
+      if (now() + backoff2 >= deadline) {
+        return now() >= opts.expiresAt ? { kind: "expired" } : { kind: "pending" };
+      }
+      await sleep(backoff2);
+      continue;
+    }
+    if (res.ok && typeof body.access_token === "string") {
+      return { kind: "approved", apiKey: body.access_token, keyName: body.key_name || "CLI" };
+    }
+    switch (body.error) {
+      case "authorization_pending":
+        break;
+      case "slow_down":
+        interval += 5e3;
+        break;
+      case "access_denied":
+        return { kind: "denied" };
+      case "expired_token":
+        return { kind: "expired" };
+      case "invalid_grant":
+        return { kind: "error", message: "This login was already used or is not valid. Run scavio login again." };
+      default:
+        return { kind: "error", message: `Login failed (HTTP ${res.status}${body.error ? `, ${body.error}` : ""}).` };
+    }
+    if (now() + interval >= deadline) {
+      return now() + interval >= opts.expiresAt ? { kind: "expired" } : { kind: "pending" };
+    }
+    await sleep(interval);
+  }
+}
+async function checkKey(baseUrl, apiKey, userAgent, f = fetch) {
+  try {
+    const res = await f(`${baseUrl}/api/v1/usage`, {
+      headers: { Authorization: `Bearer ${apiKey}`, "User-Agent": userAgent }
+    });
+    if (!res.ok) return { ok: false, status: res.status };
+    const b = await res.json().catch(() => ({}));
+    return { ok: true, plan: b.plan, credits: typeof b.credit_balance === "number" ? b.credit_balance : void 0 };
+  } catch {
+    return { ok: false, status: 0 };
+  }
+}
+
+// src/cli/profile.ts
+import { spawnSync } from "child_process";
+import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, writeFileSync as writeFileSync2 } from "fs";
+import { homedir as homedir2 } from "os";
+import { basename, dirname, join as join2 } from "path";
+var MARKER = "# added by scavio login";
+function profileTargets(env = process.env, platform = process.platform, home = homedir2()) {
+  if (platform === "win32") return [];
+  const shell = basename(env.SHELL || "");
+  const t = (kind, rel) => ({ kind, path: join2(home, rel), display: `~/${rel}` });
+  if (shell === "fish") return [t("fish", ".config/fish/conf.d/scavio.fish")];
+  if (shell === "zsh") {
+    const zdot = env.ZDOTDIR;
+    if (zdot && zdot !== home) {
+      return [{ kind: "posix", path: join2(zdot, ".zshrc"), display: join2(zdot, ".zshrc").replace(home, "~") }];
+    }
+    return [t("posix", ".zshrc")];
+  }
+  if (shell === "bash") {
+    return platform === "darwin" ? [t("posix", ".bashrc"), t("posix", ".bash_profile")] : [t("posix", ".bashrc")];
+  }
+  return [t("posix", ".profile")];
+}
+function assertSafeKey(key) {
+  if (!/^sk_[A-Za-z0-9_-]{16,128}$/.test(key)) throw new Error("Unexpected API key format; nothing was written.");
+}
+function lineFor(kind, key) {
+  return kind === "fish" ? `set -gx SCAVIO_API_KEY "${key}" ${MARKER}` : `export SCAVIO_API_KEY="${key}" ${MARKER}`;
+}
+function upsertLine(content, line) {
+  const lines = content.split("\n");
+  const kept = lines.filter((l) => !l.includes(MARKER));
+  while (kept.length && kept[kept.length - 1] === "") kept.pop();
+  return (kept.length ? kept.join("\n") + "\n" : "") + line + "\n";
+}
+function removeLine(content) {
+  return content.split("\n").filter((l) => !l.includes(MARKER)).join("\n");
+}
+function writeProfiles(key, targets) {
+  assertSafeKey(key);
+  const written = [];
+  for (const t of targets) {
+    mkdirSync2(dirname(t.path), { recursive: true });
+    const before = existsSync2(t.path) ? readFileSync2(t.path, "utf8") : "";
+    writeFileSync2(t.path, upsertLine(before, lineFor(t.kind, key)));
+    written.push(t.display);
+  }
+  return written;
+}
+function removeFromProfiles(targets) {
+  const changed = [];
+  for (const t of targets) {
+    if (!existsSync2(t.path)) continue;
+    const before = readFileSync2(t.path, "utf8");
+    if (!before.includes(MARKER)) continue;
+    writeFileSync2(t.path, removeLine(before));
+    changed.push(t.display);
+  }
+  return changed;
+}
+function setWindowsUserEnv(key) {
+  if (key !== null) assertSafeKey(key);
+  const r = key === null ? spawnSync("reg", ["delete", "HKCU\\Environment", "/v", "SCAVIO_API_KEY", "/f"], { stdio: "ignore" }) : spawnSync("setx", ["SCAVIO_API_KEY", key], { stdio: "ignore" });
+  return r.status === 0;
+}
+
+// src/cli/index.ts
+var KEYS_URL = "https://dashboard.scavio.dev/api-keys";
+var UA = `scavio-cli/${SDK_VERSION}`;
+var AGENT_WAIT_MS = 6e4;
+var EXIT = { ok: 0, error: 1, pending: 2, denied: 3 };
+var stdio = { log: (s) => process.stdout.write(s + "\n"), err: (s) => process.stderr.write(s + "\n") };
+var HELP = `scavio ${SDK_VERSION} - sign this machine in to the Scavio API
+
+Usage:
+  npx -y scavio@latest login            Open a sign-in link; saves an API key for this machine
+  npx -y scavio@latest login --wait     Finish a login started by an agent (exit 2 = still waiting)
+  npx -y scavio@latest status           Check that the saved key works
+  npx -y scavio@latest logout           Remove the saved key from this machine
+
+Options:
+  --force        Log in even if SCAVIO_API_KEY already works
+  --no-profile   Do not add SCAVIO_API_KEY to your shell startup file
+  --no-browser   Print the link without opening a browser
+  --timeout <s>  How long --wait waits (default 60)
+
+The key is never printed. Manage or revoke keys at ${KEYS_URL}`;
+function tildify(p) {
+  const h = homedir3();
+  return p.startsWith(h) ? "~" + p.slice(h.length) : p;
+}
+function openBrowser(url) {
+  if (process.env.SSH_CONNECTION || process.env.CI) return;
+  if (process.platform === "linux" && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY) return;
+  const [cmd, args] = process.platform === "darwin" ? ["open", [url]] : process.platform === "win32" ? ["cmd", ["/c", "start", "", url]] : ["xdg-open", [url]];
+  try {
+    const child = spawn(cmd, args, { stdio: "ignore", detached: true });
+    child.on("error", () => {
+    });
+    child.unref();
+  } catch {
+  }
+}
+function flag(args, name) {
+  return args.includes(name);
+}
+function option(args, name) {
+  const i = args.indexOf(name);
+  return i >= 0 ? args[i + 1] : void 0;
+}
+async function finish(result, dir, args, baseUrl, out) {
+  switch (result.kind) {
+    case "pending": {
+      const p = readPending(dir);
+      out.log("pending: not approved yet.");
+      if (p) out.log(`Open ${p.verification_uri_complete} and approve code ${p.user_code}.`);
+      out.log("Then run again: npx -y scavio@latest login --wait");
+      return EXIT.pending;
+    }
+    case "denied":
+      clearPending(dir);
+      out.err("The login was denied on the approval page. Nothing was saved.");
+      return EXIT.denied;
+    case "expired":
+      clearPending(dir);
+      out.err("The login code expired. Run: npx -y scavio@latest login");
+      return EXIT.error;
+    case "error":
+      clearPending(dir);
+      out.err(result.message);
+      return EXIT.error;
+    case "approved":
+      break;
+  }
+  clearPending(dir);
+  let credPath;
+  try {
+    credPath = saveCredentials(dir, { api_key: result.apiKey, key_name: result.keyName, saved_at: (/* @__PURE__ */ new Date()).toISOString() });
+  } catch (e) {
+    out.err(`Approved, but the key could not be saved (${e.message}).`);
+    out.err(`Copy the key "${result.keyName}" from ${KEYS_URL} instead.`);
+    return EXIT.error;
+  }
+  const places = [];
+  if (!flag(args, "--no-profile")) {
+    try {
+      if (process.platform === "win32") {
+        if (setWindowsUserEnv(result.apiKey)) places.push("your Windows user environment");
+      } else {
+        places.push(...writeProfiles(result.apiKey, profileTargets()));
+      }
+    } catch (e) {
+      out.err(`Could not update the shell startup file (${e.message}).`);
+    }
+  }
+  out.log(`Logged in. API key "${result.keyName}" saved (it is not shown).`);
+  out.log(`  Saved to: ${tildify(credPath)}`);
+  if (places.length) out.log(`  SCAVIO_API_KEY added to: ${places.join(", ")}`);
+  const check = await checkKey(baseUrl, result.apiKey, UA);
+  if (check.ok) {
+    out.log(`  Key works${check.plan ? `: ${check.plan} plan` : ""}${typeof check.credits === "number" ? `, ${check.credits} credits` : ""}.`);
+  }
+  if (places.length) {
+    out.log("Open a new terminal, or restart your AI agent, so it picks up SCAVIO_API_KEY.");
+  } else {
+    out.log("SCAVIO_API_KEY was not exported; read the key from the saved file or set it yourself.");
+  }
+  return EXIT.ok;
+}
+async function login(args, out) {
+  const dir = configDir();
+  const baseUrl = (process.env.SCAVIO_BASE_URL || BASE_URL).replace(/\/+$/, "");
+  const budget = Math.max(5, Number(option(args, "--timeout")) || AGENT_WAIT_MS / 1e3) * 1e3;
+  if (flag(args, "--wait")) {
+    const p = readPending(dir);
+    if (!p) {
+      out.err("No login in progress. Run: npx -y scavio@latest login");
+      return EXIT.error;
+    }
+    const r2 = await waitForApproval({
+      baseUrl: p.base_url,
+      deviceCode: p.device_code,
+      intervalSec: p.interval,
+      expiresAt: p.expires_at,
+      budgetMs: budget,
+      userAgent: UA
+    });
+    return finish(r2, dir, args, p.base_url, out);
+  }
+  if (!flag(args, "--force") && process.env.SCAVIO_API_KEY) {
+    const check = await checkKey(baseUrl, process.env.SCAVIO_API_KEY, UA);
+    if (check.ok) {
+      out.log("SCAVIO_API_KEY is already set and works. Nothing to do (use --force to log in again).");
+      return EXIT.ok;
+    }
+  }
+  let code;
+  try {
+    code = await startLogin(baseUrl, { hostname: machineName(), os: process.platform, userAgent: UA });
+  } catch (e) {
+    out.err(e.message);
+    return EXIT.error;
+  }
+  const expiresAt = Date.now() + code.expires_in * 1e3;
+  savePending(dir, {
+    device_code: code.device_code,
+    user_code: code.user_code,
+    verification_uri_complete: code.verification_uri_complete,
+    interval: code.interval,
+    expires_at: expiresAt,
+    base_url: baseUrl
+  });
+  const minutes = Math.round(code.expires_in / 60);
+  out.log("To connect this machine to Scavio, open:");
+  out.log("");
+  out.log(`  ${code.verification_uri_complete}`);
+  out.log("");
+  out.log(`and check that the page shows this code: ${code.user_code}`);
+  out.log(`Sign in or create a free account there. The link works for ${minutes} minutes.`);
+  if (!flag(args, "--no-browser")) openBrowser(code.verification_uri_complete);
+  const interactive = Boolean(process.stdout.isTTY && process.stdin.isTTY);
+  if (!interactive) {
+    out.log("");
+    out.log("After approving, run: npx -y scavio@latest login --wait");
+    return EXIT.ok;
+  }
+  out.log("");
+  out.log("Waiting for approval...");
+  const r = await waitForApproval({
+    baseUrl,
+    deviceCode: code.device_code,
+    intervalSec: code.interval,
+    expiresAt,
+    budgetMs: code.expires_in * 1e3,
+    userAgent: UA
+  });
+  return finish(r, dir, args, baseUrl, out);
+}
+async function status(out) {
+  const baseUrl = (process.env.SCAVIO_BASE_URL || BASE_URL).replace(/\/+$/, "");
+  const saved = readCredentials(configDir());
+  const fromEnv = process.env.SCAVIO_API_KEY;
+  const key = fromEnv || saved?.api_key;
+  if (!key) {
+    out.log("Not logged in: SCAVIO_API_KEY is not set and no saved key was found. Run: npx -y scavio@latest login");
+    return EXIT.error;
+  }
+  const source = fromEnv ? "SCAVIO_API_KEY" : `saved key "${saved?.key_name}"`;
+  const check = await checkKey(baseUrl, key, UA);
+  if (!check.ok) {
+    out.log(`The key from ${source} does not work (HTTP ${check.status || "network error"}). Run: npx -y scavio@latest login --force`);
+    return EXIT.error;
+  }
+  out.log(`Logged in via ${source}${check.plan ? `: ${check.plan} plan` : ""}${typeof check.credits === "number" ? `, ${check.credits} credits` : ""}.`);
+  if (!fromEnv) out.log("SCAVIO_API_KEY is not set in this shell yet: open a new terminal or restart your agent.");
+  return EXIT.ok;
+}
+function logout(out) {
+  const dir = configDir();
+  const hadFile = clearCredentials(dir);
+  clearPending(dir);
+  const changed = process.platform === "win32" ? setWindowsUserEnv(null) ? ["Windows user environment"] : [] : removeFromProfiles(profileTargets());
+  if (!hadFile && changed.length === 0) {
+    out.log("Nothing to remove: no key saved by scavio login on this machine.");
+    return EXIT.ok;
+  }
+  out.log("Removed the saved key from this machine" + (changed.length ? ` and from ${changed.join(", ")}.` : "."));
+  out.log(`The key still exists on your account; revoke it at ${KEYS_URL} if you no longer need it.`);
+  return EXIT.ok;
+}
+async function main(argv = process.argv.slice(2), out = stdio) {
+  const [cmd, ...args] = argv;
+  if (cmd === "--version" || cmd === "-v") {
+    out.log(SDK_VERSION);
+    return EXIT.ok;
+  }
+  if (cmd === "login") return login(args, out);
+  if (cmd === "status" || cmd === "whoami") return status(out);
+  if (cmd === "logout") return logout(out);
+  out.log(HELP);
+  return cmd && cmd !== "help" && cmd !== "--help" && cmd !== "-h" ? EXIT.error : EXIT.ok;
+}
+
+// src/cli/bin.ts
+main().then(
+  (code) => process.exit(code),
+  (err) => {
+    process.stderr.write(`scavio: ${err?.message ?? String(err)}
+`);
+    process.exit(1);
+  }
+);

package/dist/index.cjs

@@ -265,7 +265,7 @@ function parseRetryAfter(header) {
 var BASE_URL = "https://api.scavio.dev";
 var DEFAULT_TIMEOUT = 3e4;
 var DEFAULT_MAX_RETRIES = 2;
-var SDK_VERSION = true ? "0.38.0" : "0.0.0";
+var SDK_VERSION = true ? "0.39.1" : "0.0.0";
 var USER_AGENT = `scavio-js/${SDK_VERSION}`;
 function isServerRuntime() {
   const g = globalThis;

package/dist/index.js

@@ -145,7 +145,7 @@ function parseRetryAfter(header) {
 var BASE_URL = "https://api.scavio.dev";
 var DEFAULT_TIMEOUT = 3e4;
 var DEFAULT_MAX_RETRIES = 2;
-var SDK_VERSION = true ? "0.38.0" : "0.0.0";
+var SDK_VERSION = true ? "0.39.1" : "0.0.0";
 var USER_AGENT = `scavio-js/${SDK_VERSION}`;
 function isServerRuntime() {
   const g = globalThis;

package/package.json

@@ -1,11 +1,14 @@
 {
   "name": "scavio",
-  "version": "0.38.0",
+  "version": "0.39.1",
   "description": "TypeScript SDK for the Scavio API — real-time structured web data across 61 platforms on one API key: Google SERP, ChatGPT, Gemini, Amazon, Walmart, Costco, Nordstrom, Nordstrom Rack, Lowe's, Sam's Club, Canadian Tire, Dollar General, Sephora, Ulta Beauty, ALDI, Micro Center, Walgreens, CVS, Whole Foods, Harbor Freight, PetSmart, Petco, iHerb, Chewy, Menards, HEB, Kroger, Mercado Libre, eBay, Zillow, Redfin, Realtor.com, Rightmove, Booking, Airbnb, Yelp, Trustpilot, Indeed, Glassdoor, G2, App Store, Google Play, SEC EDGAR, Google Ads and Meta Ad Library, YouTube, TikTok, Instagram, X, LinkedIn, Reddit and more, plus extract() to read any URL as clean Markdown and extract.map/crawl/batch to map, crawl and batch-extract whole websites",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
   "types": "./dist/index.d.ts",
+  "bin": {
+    "scavio": "./dist/cli.js"
+  },
   "exports": {
     ".": {
       "import": {
@@ -24,7 +27,7 @@
     "LICENSE"
   ],
   "scripts": {
-    "build": "tsup",
+    "build": "rm -rf dist && tsup",
     "dev": "tsup --watch",
     "test": "vitest run",
     "test:watch": "vitest",