seo 0.15.0 → 0.15.2

package/dist/index.js

@@ -1,5 +1,5 @@
 import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
-import { n as version, t as load } from "./esm-DieyhVZ-.js";
+import { n as version, t as load } from "./esm-Dt4JJZ0P.js";
 import { SEO_PROVIDER_API_VERSION } from "./provider-sdk.js";
 import crypto, { createHash, randomUUID, sign } from "node:crypto";
 import { z } from "zod";
@@ -2709,7 +2709,7 @@ async function readableSites(google, signIn) {
 		permissionLevel: entry.permissionLevel ?? ""
 	})).sort((a, b) => a.siteUrl.localeCompare(b.siteUrl));
 }
-const hostOf$2 = (value) => {
+const hostOf$3 = (value) => {
 	try {
 		return new URL(value.includes("://") ? value : `https://${value.replace(/^sc-domain:/, "")}`).hostname.replace(/^www\./, "");
 	} catch {
@@ -2727,14 +2727,14 @@ async function runSites(input) {
 		};
 	})), input.bing ? listBingSites(input.bing).catch((error) => error instanceof Error ? error : new Error(String(error))) : Promise.resolve(null)]);
 	const bingSites = Array.isArray(bing) ? bing.filter((site) => site.verified) : [];
-	const bingByHost = new Map(bingSites.map((site) => [hostOf$2(site.url), site]));
+	const bingByHost = new Map(bingSites.map((site) => [hostOf$3(site.url), site]));
 	const properties = google.flatMap((account) => account.properties instanceof Error ? [] : account.properties.map((property) => ({
 		...property,
 		...named ? { account: account.email ?? "" } : {},
-		...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$2(property.siteUrl))?.url ?? "" } : {}
+		...Array.isArray(bing) ? { bing: bingByHost.get(hostOf$3(property.siteUrl))?.url ?? "" } : {}
 	})));
-	const googleHosts = new Set(properties.map((property) => hostOf$2(property.siteUrl)));
-	const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$2(site.url)));
+	const googleHosts = new Set(properties.map((property) => hostOf$3(property.siteUrl)));
+	const bingOnly = bingSites.filter((site) => !googleHosts.has(hostOf$3(site.url)));
 	const failed = google.filter((account) => account.properties instanceof Error);
 	const analytics = google.flatMap((account) => (account.analytics ?? []).flatMap((summary) => summary.propertySummaries.map((property) => ({
 		property: property.property.replace("properties/", ""),
@@ -23563,7 +23563,7 @@ function typesafeDecisionModel(input) {
 const MAX_CONCURRENCY = 32;
 const MIN_CONCURRENCY = 2;
 const MAX_ATTEMPTS = 5;
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 var DecisionRunner = class {
 	model;
 	limit = MAX_CONCURRENCY;
@@ -23628,7 +23628,7 @@ var DecisionRunner = class {
 			} finally {
 				this.release();
 			}
-			await sleep(delay);
+			await sleep$1(delay);
 		}
 		this.failed += 1;
 		return null;
@@ -31876,6 +31876,30 @@ async function dataForSeoTask([login, password], path, task) {
 }
 const NO_RESULTS = 40102;
 const PARTIAL = 40106;
+const queued = (task) => ({
+	id: String(task.id ?? ""),
+	status_code: Number(task.status_code ?? 0),
+	status_message: String(task.status_message ?? ""),
+	cost: Number(task.cost ?? 0),
+	data: task.data ?? {},
+	result: task.result?.[0] ?? null
+});
+async function dataForSeoPost([login, password], path, tasks) {
+	return ((await readJson(await call("dataforseo", new Request(`https://api.dataforseo.com/${path}`, {
+		method: "POST",
+		headers: {
+			authorization: `Basic ${btoa(`${login}:${password}`)}`,
+			"content-type": "application/json"
+		},
+		body: JSON.stringify(tasks)
+	})), "DataForSEO")).tasks ?? []).map(queued);
+}
+const DATAFORSEO_NOT_READY = /* @__PURE__ */ new Set([40601, 40602]);
+async function dataForSeoGet([login, password], path) {
+	const task = (await readJson(await call("dataforseo", new Request(`https://api.dataforseo.com/${path}`, { headers: { authorization: `Basic ${btoa(`${login}:${password}`)}` } })), "DataForSEO")).tasks?.[0];
+	if (!task) throw new CommandError("PROVIDER_ERROR", `dataforseo: no task at ${path}`);
+	return queued(task);
+}
 async function dataForSeo(keys, input) {
 	const spent = {
 		cost: 0,
@@ -32175,7 +32199,8 @@ const ENGINES = [
 	{
 		name: "chatgpt",
 		means: "a ChatGPT search, read by DataForSEO",
-		path: "v3/ai_optimization/chat_gpt/llm_scraper/live/advanced",
+		post: "v3/ai_optimization/chat_gpt/llm_scraper/task_post",
+		get: "v3/ai_optimization/chat_gpt/llm_scraper/task_get/advanced/",
 		languages: "v3/ai_optimization/chat_gpt/llm_scraper/languages",
 		task: (question, market) => ({
 			keyword: question,
@@ -32195,7 +32220,8 @@ const ENGINES = [
 	{
 		name: "google_ai_mode",
 		means: "Google AI Mode",
-		path: "v3/serp/google/ai_mode/live/advanced",
+		post: "v3/serp/google/ai_mode/task_post",
+		get: "v3/serp/google/ai_mode/task_get/advanced/",
 		languages: "v3/serp/google/ai_mode/languages",
 		task: (question, market) => ({
 			keyword: question,
@@ -32206,7 +32232,8 @@ const ENGINES = [
 	{
 		name: "google_ai_overview",
 		means: "the AI Overview on a Google search",
-		path: "v3/serp/google/organic/live/advanced",
+		post: "v3/serp/google/organic/task_post",
+		get: "v3/serp/google/organic/task_get/advanced/",
 		languages: "v3/serp/google/languages",
 		task: (question, market) => ({
 			keyword: question,
@@ -32228,56 +32255,55 @@ function onSite(url, domain) {
 		return false;
 	}
 }
-async function ask(keys, engine, question, market, site) {
-	if (typeof market === "string") return {
-		result: "unavailable",
-		model: null,
-		cited_url: null,
-		answered_at: null,
-		cost_usd: 0,
-		error: market
-	};
+const MAX_SOURCES = 8;
+const hostOf$2 = (url) => {
 	try {
-		const done = await dataForSeoTask(keys, engine.path, engine.task(question, market));
-		const answered = {
-			answered_at: String(done.result?.datetime ?? "").slice(0, 10) || null,
-			cost_usd: done.cost
-		};
-		const shown = done.result && engine.shown(done.result);
-		if (!shown) return {
-			result: "no_answer",
-			model: null,
-			cited_url: null,
-			...answered
-		};
-		const base = {
-			model: shown.model,
-			...answered
-		};
-		const linked = shown.markdown.match(/https?:\/\/[^\s)\]]+/g) ?? [];
-		const cited = [...shown.sources, ...linked].find((url) => onSite(url, site.domain));
-		if (cited) return {
-			result: "cited",
-			cited_url: cited,
-			...base
-		};
-		const text = prose(shown.markdown);
-		return {
-			result: text.toLowerCase().includes(site.domain) || isBrandQuery(text, site.terms) ? "mentioned" : "neither",
-			cited_url: null,
-			...base
-		};
-	} catch (error) {
-		if (error instanceof CommandError && error.code === "NO_KEY") throw error;
-		return {
-			result: "unavailable",
-			model: null,
-			cited_url: null,
-			answered_at: null,
-			cost_usd: error.cost ?? 0,
-			error: error instanceof Error ? error.message.slice(0, 200) : "failed"
-		};
+		return new URL(url).hostname.toLowerCase().replace(/^www\./, "");
+	} catch {
+		return null;
 	}
+};
+const empty = {
+	cited_url: null,
+	sources: [],
+	model: null,
+	answered_at: null
+};
+function judge(engine, task, site) {
+	if (task.status_code === 40102) return {
+		result: "no_answer",
+		...empty
+	};
+	if (task.status_code >= 4e4 || !task.result) return {
+		result: "unavailable",
+		...empty,
+		error: `dataforseo: ${task.status_message}`.slice(0, 200)
+	};
+	const answered_at = String(task.result.datetime ?? "").slice(0, 10) || null;
+	const shown = engine.shown(task.result);
+	if (!shown) return {
+		result: "no_answer",
+		...empty,
+		answered_at
+	};
+	const linked = shown.markdown.match(/https?:\/\/[^\s)\]]+/g) ?? [];
+	const all = [...shown.sources, ...linked];
+	const cited = all.find((url) => onSite(url, site.domain)) ?? null;
+	const base = {
+		cited_url: cited,
+		sources: [...new Set(all.filter((url) => !onSite(url, site.domain)).map(hostOf$2).filter((host) => Boolean(host)))].slice(0, MAX_SOURCES),
+		model: shown.model,
+		answered_at
+	};
+	if (cited) return {
+		result: "cited",
+		...base
+	};
+	const text = prose(shown.markdown);
+	return {
+		result: text.toLowerCase().includes(site.domain) || isBrandQuery(text, site.terms) ? "mentioned" : "neither",
+		...base
+	};
 }
 async function marketsFor(keys, country) {
 	const code = country.toUpperCase();
@@ -32295,38 +32321,162 @@ async function marketsFor(keys, country) {
 	}));
 }
 const usd = (value) => Math.round(value * 1e4) / 1e4;
+const FIRST_WAIT_MS = 1e4;
+const ROUND_WAIT_MS = 5e3;
+const sleep = (ms) => new Promise((done) => setTimeout(done, ms));
+const TASK_PATTERN = /^([a-z_]+):([0-9a-f-]{20,64})$/;
+const AI_ANSWER_TASK_PATTERN = TASK_PATTERN;
 async function runAiAnswers(input, headers, deps, more) {
 	const keys = dataForSeoKeys(headers);
 	const domain = registrableDomain(input.site)?.domain;
 	if (!domain) throw new CommandError("INVALID_INPUT", `ai_answers needs a site with a domain, not "${input.site}".`);
-	const range = dateWindow({}, QUESTION_DAYS);
-	const questions = ((await aiQueriesReport({
-		site: input.site,
-		...range
-	}, deps.searchAnalytics)).items ?? []).filter((item) => item.askable).slice(0, 10);
 	const site = {
 		domain,
 		terms: deriveBrandTerms({ siteUrl: input.site })
 	};
-	const markets = questions.length ? await marketsFor(keys, input.country) : /* @__PURE__ */ new Map();
-	const answers = await Promise.all(questions.flatMap((question) => ENGINES.map(async (engine) => ({
-		question: question.query,
-		impressions: question.impressions,
-		engine: engine.name,
-		...await ask(keys, engine, question.query, markets.get(engine.name) ?? "no market", site)
-	}))));
-	const cost = usd(answers.reduce((sum, row) => sum + row.cost_usd, 0));
-	const count = (result) => answers.filter((row) => row.result === result).length;
-	const notes = [questions.length ? `The top ${questions.length} askable questions in Search Console’s AI-shaped queries for ${range.startDate} to ${range.endDate}, by impressions. No calls were made for other questions.` : `No askable questions in Search Console’s AI-shaped queries for ${range.startDate} to ${range.endDate}, so no answers were checked and nothing was spent.`, ...questions.length ? [
-		`Cited ${count("cited")}, mentioned ${count("mentioned")}, neither ${count("neither")}, no answer shown ${count("no_answer")}, unavailable ${count("unavailable")}. DataForSEO cost: $${cost}.`,
+	const given = [...new Set((input.questions ?? []).map((q) => q.trim()).filter(Boolean))];
+	if (given.length > 10) throw new CommandError("INVALID_INPUT", `ai_answers takes up to 10 questions, not ${given.length}.`);
+	const tasks = [...new Set(input.tasks ?? [])];
+	const range = dateWindow({}, QUESTION_DAYS);
+	const rows = [];
+	const waiting = [];
+	const blank = (question, impressions) => ({
+		question,
+		impressions,
+		task: null,
+		cost_usd: 0,
+		...empty
+	});
+	if (tasks.length) for (const value of tasks) {
+		const match = TASK_PATTERN.exec(value);
+		const engine = ENGINES.find((item) => item.name === match?.[1]);
+		if (!match || !engine) throw new CommandError("INVALID_INPUT", `"${value}" is not an ai_answers task. Tasks look like <engine>:<id>, with the engine one of ${ENGINES.map((item) => item.name).join(", ")}.`);
+		const row = {
+			...blank("", null),
+			engine: engine.name,
+			result: "pending",
+			task: value
+		};
+		rows.push(row);
+		waiting.push({
+			row,
+			engine,
+			id: match[2]
+		});
+	}
+	else {
+		let questions;
+		if (given.length) questions = given.map((query) => ({
+			query,
+			impressions: null
+		}));
+		else questions = ((await aiQueriesReport({
+			site: input.site,
+			...range
+		}, deps.searchAnalytics)).items ?? []).filter((item) => item.askable).slice(0, 10);
+		const markets = questions.length ? await marketsFor(keys, input.country) : /* @__PURE__ */ new Map();
+		await Promise.all(ENGINES.map(async (engine) => {
+			const engineRows = questions.map((question) => ({
+				...blank(question.query, question.impressions),
+				engine: engine.name,
+				result: "unavailable"
+			}));
+			rows.push(...engineRows);
+			if (!questions.length) return;
+			const market = markets.get(engine.name) ?? "no market";
+			if (typeof market === "string") {
+				for (const row of engineRows) row.error = market;
+				return;
+			}
+			let queuedTasks;
+			try {
+				queuedTasks = await dataForSeoPost(keys, engine.post, questions.map((question) => ({
+					...engine.task(question.query, market),
+					priority: 2
+				})));
+			} catch (error) {
+				if (error instanceof CommandError && error.code === "NO_KEY") throw error;
+				for (const row of engineRows) row.error = error instanceof Error ? error.message.slice(0, 200) : "failed";
+				return;
+			}
+			engineRows.forEach((row, index) => {
+				const task = queuedTasks[index];
+				row.cost_usd = task?.cost ?? 0;
+				if (!task || task.status_code >= 4e4 || !task.id) {
+					row.error = `dataforseo: ${task?.status_message ?? "no task"}`;
+					return;
+				}
+				row.result = "pending";
+				row.task = `${engine.name}:${task.id}`;
+				waiting.push({
+					row,
+					engine,
+					id: task.id
+				});
+			});
+		}));
+	}
+	const deadline = Date.now() + (input.waitSeconds ?? 40) * 1e3;
+	let pending = waiting;
+	let first = !tasks.length;
+	while (pending.length) {
+		const pause = first ? FIRST_WAIT_MS : ROUND_WAIT_MS;
+		first = false;
+		if (!tasks.length || pending !== waiting) {
+			if (Date.now() + pause > deadline) break;
+			await sleep(pause);
+		}
+		const still = [];
+		await Promise.all(pending.map(async (item) => {
+			try {
+				const task = await dataForSeoGet(keys, `${item.engine.get}${item.id}`);
+				if (!item.row.question) item.row.question = String(task.data.keyword ?? "");
+				if (DATAFORSEO_NOT_READY.has(task.status_code)) {
+					still.push(item);
+					return;
+				}
+				Object.assign(item.row, judge(item.engine, task, site), { task: null });
+			} catch (error) {
+				if (error instanceof CommandError && error.code === "NO_KEY") throw error;
+				still.push(item);
+			}
+		}));
+		pending = still;
+		if (tasks.length && Date.now() >= deadline) break;
+	}
+	rows.sort((a, b) => a.question.localeCompare(b.question) || ENGINES.findIndex((e) => e.name === a.engine) - ENGINES.findIndex((e) => e.name === b.engine));
+	const cost = usd(rows.reduce((sum, row) => sum + row.cost_usd, 0));
+	const count = (result) => rows.filter((row) => row.result === result).length;
+	const left = rows.filter((row) => row.task).map((row) => row.task);
+	const cites = /* @__PURE__ */ new Map();
+	for (const row of rows) for (const host of row.sources) cites.set(host, (cites.get(host) ?? 0) + 1);
+	const topCited = [...cites].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 10);
+	const questionCount = new Set(rows.map((row) => row.question)).size;
+	const notes = [tasks.length ? `${tasks.length === 1 ? "One answer" : `${tasks.length} answers`} queued by an earlier run, read back at no cost.` : given.length ? `The ${given.length} questions you gave, asked as written. Impressions are blank because they don't come from Search Console.` : questionCount ? `The top ${questionCount} askable questions in Search Console’s AI-shaped queries for ${range.startDate} to ${range.endDate}, by impressions. No calls were made for other questions.` : `No askable questions in Search Console’s AI-shaped queries for ${range.startDate} to ${range.endDate}, so no answers were checked and nothing was spent.`, ...rows.length ? [
+		`Cited ${count("cited")}, mentioned ${count("mentioned")}, neither ${count("neither")}, no answer shown ${count("no_answer")}, unavailable ${count("unavailable")}, not ready ${count("pending")}. DataForSEO cost: $${cost}.`,
+		...left.length ? [`${left.length === 1 ? "One answer wasn't" : `${left.length} answers weren't`} ready yet. They're paid for: ask ai_answers again with the same target and tasks ${JSON.stringify(left)} in a minute to read them at no cost (DataForSEO keeps them for 30 days).`] : [],
+		...topCited.length ? [`Other sites these answers cite: ${topCited.map(([host, n]) => `${host} (${n})`).join(", ")}.`] : [],
 		"Each row is one sample of an answer that changes from run to run, not a ranking.",
-		`cited: the answer lists or links a page on ${domain} or a subdomain. mentioned: no such source, but the answer text names ${[domain, ...site.terms].join(" or ")}, a text match. no_answer: the engine showed no AI answer for the question. model: the model DataForSEO reports for the answer, when it reports one.`,
+		`cited: the answer lists or links a page on ${domain} or a subdomain. mentioned: no such source, but the answer text names ${[domain, ...site.terms].join(" or ")}, a text match. no_answer: the engine showed no AI answer for the question. sources: the other sites the answer cites, up to ${MAX_SOURCES}. model: the model DataForSEO reports for the answer, when it reports one.`,
 		`Engines: ${ENGINES.map((engine) => `${engine.name} is ${engine.means}`).join("; ")}.`
 	] : []];
+	const shown = rows.map((row) => ({
+		question: row.question,
+		impressions: row.impressions,
+		engine: row.engine,
+		result: row.result,
+		cited_url: row.cited_url,
+		sources: row.sources.join(", "),
+		model: row.model,
+		answered_at: row.answered_at,
+		cost_usd: row.cost_usd,
+		task: row.task,
+		error: row.error ?? null
+	}));
 	return {
 		text: rowsText({
 			title: `AI answers for ${input.site}, ${input.country}.`,
-			rows: answers,
+			rows: shown,
 			offset: input.offset,
 			limit: 50,
 			detail: input.detail,
@@ -32338,15 +32488,23 @@ async function runAiAnswers(input, headers, deps, more) {
 			action: "ai_answers",
 			site: input.site,
 			country: input.country,
-			questionsFrom: {
+			questionsFrom: tasks.length ? { section: "tasks" } : given.length ? { section: "given" } : {
 				section: "ai-queries",
 				...range
 			},
-			questions: questions.map((question) => question.query),
+			questions: [...new Set(rows.map((row) => row.question))],
 			costUsd: cost,
-			total: answers.length,
+			pendingTasks: left,
+			citedSites: topCited.map(([host, answers]) => ({
+				host,
+				answers
+			})),
+			total: rows.length,
 			offset: input.offset,
-			rows: answers.slice(input.offset, input.offset + 50)
+			rows: rows.slice(input.offset, input.offset + 50).map(({ error, ...row }) => error ? {
+				...row,
+				error
+			} : row)
 		}
 	};
 }
@@ -34928,7 +35086,7 @@ function nextFor(section, call) {
 		"content-decay": `Next: ${call("index")} for pages that lost most of their clicks; ${call("page")} to see which queries moved.`,
 		cannibalization: `Next: ${call("page")} on each competing URL to see Google’s chosen canonical.`,
 		"traffic-drop": `Next: ${call("compare")} with the date the drop started, for the pages and queries that moved; ${call("index")} for pages that lost almost all their clicks and their index status.`,
-		"ai-queries": `Next: ${call("page")} for a page that gets many of these questions, to see whether it answers them; ${call("research")} ai_answers with the site as target checks whether AI answers cite it for the top 10 askable questions here (about $0.12 of DataForSEO credit), so show the person those questions and ask first.`,
+		"ai-queries": `Next: ${call("page")} for a page that gets many of these questions, to see whether it answers them; ${call("research")} ai_answers with the site as target checks whether AI answers cite it for the top 10 askable questions here (about $0.07 of DataForSEO credit), so show the person those questions and ask first.`,
 		bing: `Next: ${call("data", { source: "bing" })} for every Bing row; ${call("page")} for one page.`
 	}[section];
 }
@@ -35432,7 +35590,7 @@ const RESEARCH_ACTIONS = [
 	"ai_answers"
 ];
 const RESEARCH_PROVIDERS = ["dataforseo", "ahrefs"];
-const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), ai_answers (DataForSEO only; target: the site. It takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself, then asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s live prices are $0.012 a question, so up to about $0.12 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
+const RESEARCH_HELP = `keyword_metrics (keywords: volume, difficulty, CPC), ranked_keywords (target: keywords a domain or URL ranks for), serp (keywords, up to 5: the ranking pages), competitors (target: domains that share keywords), backlinks (target: referring domains), ai_answers (DataForSEO only; target: the site. Without keywords it takes the site’s top 10 chat-style and long questions from Search Console’s AI-shaped queries itself. With up to 10 questions in keywords it asks those instead, and the target can be any domain, such as a competitor's. Each answer row also lists the other sites it cites, and the notes rank them. Answers come from DataForSEO's queue: answers not ready after about 40 seconds are returned as tasks, already paid for; call again with the same target and those tasks to read them at no cost. It asks ChatGPT search, Google AI Mode and Google AI Overviews each one and says whether the answer cites or mentions the site. DataForSEO’s queue prices are about $0.0072 a question across the three engines, so about $0.07 a run: tell the person before running it). Rows with a domain, and the target, get a free Ahrefs domain_rating column. Numbers are provider estimates, not Search Console data.`;
 const GENERIC_PLATFORMS = [
 	"youtube.com",
 	"facebook.com",
@@ -36351,6 +36509,7 @@ const SHARED_TOOLS = [
 			action: z.enum(RESEARCH_ACTIONS),
 			keywords: z.array(z.string().min(1)).max(100).default([]),
 			target: z.string().default("").describe("Domain or URL."),
+			tasks: z.array(z.string().regex(AI_ANSWER_TASK_PATTERN)).max(30).default([]).describe("ai_answers only: the tasks an earlier run returned as not ready, to read back at no cost."),
 			country: z.string().regex(/^[a-z]{2}$/).default("us").describe("Two-letter country code."),
 			...detail
 		}),
@@ -36359,10 +36518,12 @@ const SHARED_TOOLS = [
 			if (input.action === "ai_answers") {
 				if (input.provider !== "dataforseo") throw new CommandError("INVALID_INPUT", "ai_answers uses dataforseo.");
 				return runAiAnswers({
-					site: await surface.resolveSite(input.target),
+					site: input.keywords.length || input.tasks.length ? input.target : await surface.resolveSite(input.target),
 					country: input.country,
 					detail: input.detail,
-					offset: input.offset
+					offset: input.offset,
+					questions: input.keywords,
+					tasks: input.tasks
 				}, await surface.keys(), surface.deps, more);
 			}
 			return runResearch(input, await surface.keys(), {