polite-crawler 0.1.2 → 0.1.4

package/build/recipe-product-Dellqudu.mjs

@@ -0,0 +1,892 @@
+import { i as durationMs } from "./version-BPem-XZd.mjs";
+import { S as archivedOf, a as fetchPage, c as isHeldBack, n as HELD_BACK, p as recordOverride } from "./runtime-D_r2SYge.mjs";
+import { o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
+import { c as runtimeOf, r as deadlineOf } from "./recipe-GiFQojmo.mjs";
+import { DAYS, describeWeek, disagreements, fromCompact, fromSchemaOrg, fromText } from "./hours.mjs";
+//#region src/recipe-venue.ts
+/** These are the limits of the venue recipe. */
+const VENUE_LIMITS = {
+	pages: 3,
+	maxPages: 8,
+	menus: 8,
+	readingsPerPage: 20,
+	links: 200,
+	nearMetres: 150
+};
+const STOP_WORDS = /* @__PURE__ */ new Set([
+	"the",
+	"a",
+	"an",
+	"and",
+	"of",
+	"at"
+]);
+/** This function gives the words of a text in one form: lower case, no accents, and `&` as `and`. */
+const wordsOf = (text) => text.toLowerCase().normalize("NFKD").replace(/\p{M}/gu, "").replace(/&/g, " and ").replace(/[^\p{L}\p{N} ]/gu, " ").split(/\s+/).filter((word) => word.length > 0 && !STOP_WORDS.has(word));
+/**
+* This function tells if a page names a venue: the page has the first word of the name and 60
+* percent of its words. A domain that a venue let go can show a different business. It gives
+* undefined for a name with no word.
+*/
+const namesVenue = (name, page) => {
+	const words = wordsOf(name);
+	if (words.length === 0) return void 0;
+	const onPage = new Set(wordsOf(`${page.title} ${page.text}`));
+	const named = new Set(words);
+	const found = [...named].filter((word) => onPage.has(word)).length;
+	return onPage.has(words[0]) && found / named.size >= .6;
+};
+/** The words of a link that tell what its page holds, with their weights. */
+const LINK_WORDS = [
+	[/hours|horario|horaires|opening/, 6],
+	[/menu|food|drink|wine|cocktail/, 4],
+	[/contact|location|visit|find.?us|directions/, 3],
+	[/about|info/, 2],
+	[/lunch|dinner|brunch|breakfast/, 2]
+];
+/** The hosts of review and social sites: a menu there is a different page about the venue. */
+const SOCIAL = /(?:^|\.)(?:yelp|tripadvisor|facebook|instagram|twitter|x|tiktok|google|apple|spotify)\.[a-z.]+$/i;
+/** The files that hold no hours as text. */
+const FILE = /\.(?:pdf|jpe?g|png|gif|webp|svg|mp4)(?:[?#]|$)/i;
+const hostOf = (url) => {
+	try {
+		return new URL(url).hostname.toLowerCase().replace(/^www\./, "");
+	} catch {
+		return;
+	}
+};
+const withoutHash = (url) => url.replace(/#.*$/s, "").replace(/\/$/, "");
+/**
+* This function ranks the links of a page by the words of their text and path. The links to
+* the hours or a menu come first. It keeps only the links on the site of `home`,
+* and it leaves out the files (a PDF, an image) and the home page itself.
+*/
+const rankHoursLinks = (links, home) => {
+	const site = hostOf(home);
+	const self = withoutHash(home);
+	const seen = /* @__PURE__ */ new Set();
+	const scored = [];
+	for (const link of links.slice(0, VENUE_LIMITS.links)) {
+		if (typeof link?.href !== "string" || hostOf(link.href) !== site || FILE.test(link.href)) continue;
+		const key = withoutHash(link.href);
+		if (key === self || seen.has(key)) continue;
+		let path = "";
+		try {
+			path = new URL(link.href).pathname;
+		} catch {
+			continue;
+		}
+		const words = `${typeof link.text === "string" ? link.text : ""} ${path}`.toLowerCase();
+		const score = LINK_WORDS.reduce((sum, [pattern, weight]) => sum + (pattern.test(words) ? weight : 0), 0);
+		if (score <= 0) continue;
+		seen.add(key);
+		scored.push({
+			link,
+			score
+		});
+	}
+	return scored.sort((a, b) => b.score - a.score).map((entry) => entry.link);
+};
+const EARTH_METRES = 6371e3;
+/** This function gives the distance in metres between two points on the earth (the haversine formula). */
+const metresBetween = (a, b) => {
+	const rad = Math.PI / 180;
+	const dLat = (b.lat - a.lat) * rad;
+	const dLon = (b.lon - a.lon) * rad;
+	const h = Math.sin(dLat / 2) ** 2 + Math.cos(a.lat * rad) * Math.cos(b.lat * rad) * Math.sin(dLon / 2) ** 2;
+	return 2 * EARTH_METRES * Math.asin(Math.min(1, Math.sqrt(h)));
+};
+const pointOf = (lat, lon) => {
+	const la = typeof lat === "string" ? Number.parseFloat(lat) : lat;
+	const lo = typeof lon === "string" ? Number.parseFloat(lon) : lon;
+	return typeof la === "number" && typeof lo === "number" && Number.isFinite(la) && Number.isFinite(lo) && Math.abs(la) <= 90 && Math.abs(lo) <= 180 ? {
+		lat: la,
+		lon: lo
+	} : void 0;
+};
+/** The words of a street that tell nothing about which street: directions and kinds of road. */
+const STREET_NOISE = /* @__PURE__ */ new Set([
+	"n",
+	"s",
+	"e",
+	"w",
+	"north",
+	"south",
+	"east",
+	"west",
+	"st",
+	"street",
+	"ave",
+	"avenue",
+	"rd",
+	"road",
+	"blvd",
+	"boulevard",
+	"dr",
+	"drive",
+	"ln",
+	"lane",
+	"ct",
+	"court",
+	"pl",
+	"place",
+	"sq",
+	"square",
+	"hwy",
+	"highway",
+	"pkwy",
+	"parkway",
+	"way",
+	"ter",
+	"terrace",
+	"suite",
+	"ste",
+	"unit"
+]);
+/** This function gives the number of a street address and the words that name its street. */
+const streetOf = (address) => {
+	const words = wordsOf((address.split(",")[0] ?? address).slice(0, 200));
+	const number = words.find((word) => /^\d+[a-z]?$/.test(word));
+	if (number === void 0) return void 0;
+	return {
+		number,
+		words: new Set(words.filter((word) => !/^\d/.test(word) && !STREET_NOISE.has(word)))
+	};
+};
+const addressText = (value) => {
+	if (typeof value === "string") return value;
+	if (Array.isArray(value)) return addressText(value[0]);
+	if (typeof value === "object" && value !== null) {
+		const street = value["streetAddress"];
+		return typeof street === "string" ? street : void 0;
+	}
+};
+/**
+* This function tells if a JSON-LD location record is the venue. The record must have the same
+* number on a street that shares a word, or a position near the venue (`VENUE_LIMITS.nearMetres`).
+*/
+const locationMatches = (record, location) => {
+	const geo = record["geo"];
+	const at = typeof geo === "object" && geo !== null ? pointOf(geo["latitude"], geo["longitude"]) : void 0;
+	const ours = pointOf(location.lat, location.lon);
+	if (at !== void 0 && ours !== void 0 && metresBetween(at, ours) <= VENUE_LIMITS.nearMetres) return true;
+	const theirs = addressText(record["address"]);
+	if (theirs === void 0 || location.address === void 0) return false;
+	const a = streetOf(theirs);
+	const b = streetOf(location.address);
+	return a !== void 0 && b !== void 0 && a.number === b.number && [...a.words].some((word) => b.words.has(word));
+};
+const located = (record) => record["address"] !== void 0 || record["geo"] !== void 0;
+/** The types of JSON-LD records that have the hours of a venue. */
+const HOURS_TYPES = /Restaurant|LocalBusiness|FoodEstablishment|BarOrPub|CafeOrCoffeeShop|Bakery|Store|Place|Winery|Brewery|NightClub|Museum|TouristAttraction|Library|EntertainmentBusiness/;
+/**
+* A label that names a schedule that is not the opening hours. The line above a schedule is often a
+* different thing, for example a phone number, and such a line tells nothing.
+*/
+const OTHER_SCHEDULE = /brunch|lunch|dinner|breakfast|happy|kitchen|bar|deliver|take.?out|pick.?up|patio|holiday|season|summer|winter|special|late|menu/i;
+/**
+* A label of a service beside the opening hours, or of an office. Its hours are a different
+* schedule, also when the label says "hours" ("Happy hour", "Kitchen hours", "Office hours").
+*/
+const SERVICE = /happy|kitchen|deliver|take.?out|pick.?up|brunch|holiday|special|patio|bar|late.?night|office/i;
+const rankOf = (from, label) => {
+	if (from === "jsonld") return 0;
+	if (from === "osm") return 2;
+	const text = label ?? "";
+	if (SERVICE.test(text)) return 4;
+	if (/hour|open/i.test(text)) return 1;
+	return OTHER_SCHEDULE.test(text) ? 4 : 3;
+};
+const reading = (from, hours, label, source) => {
+	const rank = rankOf(from, label);
+	return {
+		from,
+		rank,
+		schedule: rank <= 3 ? "opening" : "other",
+		label,
+		hours: describeWeek(hours.week),
+		week: hours.week,
+		inferred: hours.inferred,
+		partial: hours.partial === true,
+		season: hours.season ?? null,
+		raw: hours.raw,
+		path: source.path,
+		url: source.url,
+		doc: source.doc,
+		capture: source.capture,
+		fetched_at: source.fetched_at,
+		...source.archived === void 0 ? {} : { archived: source.archived }
+	};
+};
+const recordsOf = (meta) => {
+	const jsonld = meta["jsonld"];
+	return Array.isArray(jsonld) ? jsonld.filter((record) => record !== null && typeof record === "object" && !Array.isArray(record)) : [];
+};
+const linksOf = (meta) => {
+	const links = meta["links"];
+	return Array.isArray(links) ? links.filter((link) => link !== null && typeof link === "object" && typeof link.href === "string") : [];
+};
+/**
+* This function reads the hours of one stored document, or of one version of it. It is pure, and
+* it sends no request. Thus it also reads a version at a time (`versionAtTime`) or an archived copy.
+*
+* - It reads each JSON-LD record of a venue type with `fromSchemaOrg`, and the text with `fromText`.
+* - On a page that lists more than one location, it reads only the location that matches
+*   `location`. With no `location`, it reads none of them, and a note says so.
+* - It gives the first `telephone` and the menu links that are not on a review or social site.
+*/
+const venueReadingsOf = (source, options = {}) => {
+	let steps = 0;
+	const notes = [];
+	const archived = archivedOf(source.meta);
+	const where = {
+		url: archived?.original ?? source.url,
+		doc: source.doc ?? null,
+		capture: source.capture ?? null,
+		fetched_at: source.fetched_at ?? null,
+		...archived === void 0 ? {} : { archived }
+	};
+	const all = recordsOf(source.meta);
+	steps += all.length;
+	const venues = all.map((record, index) => ({
+		record,
+		index
+	})).filter(({ record }) => typesOf(record).some((type) => HOURS_TYPES.test(type)));
+	let usable = venues;
+	const placed = venues.filter(({ record }) => located(record));
+	if (placed.length > 1) {
+		const location = options.location;
+		const ours = location === void 0 ? [] : placed.filter(({ record }) => locationMatches(record, location));
+		if (ours.length === 0) notes.push(location === void 0 ? `${where.url ?? "the page"} lists ${String(placed.length)} locations: give location to read the hours of one` : `${where.url ?? "the page"} lists ${String(placed.length)} locations, and none matched the location: their hours were not read`);
+		usable = [...ours, ...venues.filter(({ record }) => !located(record))];
+	}
+	const readings = [];
+	let phone = null;
+	for (const { record, index } of usable) {
+		steps += 1;
+		const hours = fromSchemaOrg(record);
+		if (hours !== void 0 && readings.length < VENUE_LIMITS.readingsPerPage) readings.push(reading("jsonld", hours, null, {
+			...where,
+			path: `jsonld[${String(index)}]`
+		}));
+		const telephone = record["telephone"];
+		if (phone === null && typeof telephone === "string" && telephone.trim().length > 0) phone = telephone.trim().slice(0, 100);
+	}
+	steps += source.text.length;
+	for (const hours of fromText(source.text)) {
+		if (readings.length >= VENUE_LIMITS.readingsPerPage) {
+			notes.push(`${where.url ?? "the page"} gives more than ${String(VENUE_LIMITS.readingsPerPage)} readings: the rest were not read`);
+			break;
+		}
+		readings.push(reading("text", hours, hours.label, {
+			...where,
+			path: null
+		}));
+	}
+	const menus = [];
+	for (const link of linksOf(source.meta).slice(0, VENUE_LIMITS.links)) {
+		steps += 1;
+		if (menus.length >= VENUE_LIMITS.menus) break;
+		const text = typeof link.text === "string" ? link.text.slice(0, 200) : "";
+		const host = hostOf(link.href);
+		if (host === void 0 || SOCIAL.test(host) || !/menu/i.test(`${text} ${link.href}`)) continue;
+		if (menus.some((menu) => menu.url === link.href)) continue;
+		menus.push({
+			url: link.href,
+			text
+		});
+	}
+	return {
+		readings,
+		phone,
+		menus,
+		notes,
+		steps
+	};
+};
+const answered$1 = (page) => page.outcome === "ok" || page.outcome === "not-modified" || page.archived !== void 0;
+/**
+* This function reads the hours of a venue from its site. It fetches the home page and the pages
+* that most likely hold hours or a menu. Each one is a fetch of the crawler. Thus robots.txt, the
+* pace, the stops, the capture log and the rule for a held-back page apply. It renders a page only
+* when the page needs JavaScript, as `fetch` does.
+*
+* It gives each reading with its source, the best reading, and the days where the readings
+* disagree. It writes nothing but the capture log. To choose the true hours is the decision of
+* the application.
+*/
+const readVenue = async (target, url, options = {}) => {
+	const runtime = runtimeOf(target);
+	const extra = options.pages ?? VENUE_LIMITS.pages;
+	if (!Number.isInteger(extra) || extra < 0 || extra > VENUE_LIMITS.maxPages) throw new RangeError(`pages must be a whole number from 0 to ${String(VENUE_LIMITS.maxPages)}, not ${String(extra)}`);
+	let site;
+	try {
+		site = new URL(url);
+	} catch {
+		throw new TypeError(`readVenue needs the URL of a site, not ${JSON.stringify(url)}`);
+	}
+	if (site.protocol !== "http:" && site.protocol !== "https:") throw new TypeError(`readVenue needs an http or https URL, not ${url}`);
+	const maxAge = options.maxAge ?? "24h";
+	const job = options.job ?? "venue";
+	const userProvided = options.userProvided ?? false;
+	const pageOptions = {
+		render: options.render ?? "auto",
+		maxAgeMs: maxAge === "auto" ? "auto" : durationMs(maxAge),
+		onBehalfOf: options.onBehalfOf,
+		job,
+		signal: options.signal,
+		scope: options.scope,
+		labels: options.labels,
+		deadline: deadlineOf(runtime, options.budget),
+		fallback: options.fallback,
+		archiveDespiteRobots: options.archiveDespiteRobots
+	};
+	const notes = [];
+	const readings = [];
+	const pages = [];
+	const menus = [];
+	const captures = /* @__PURE__ */ new Set();
+	let phone = null;
+	let heldBack = false;
+	if (options.osm !== void 0) {
+		const osm = typeof options.osm === "string" ? { value: options.osm } : options.osm;
+		const hours = typeof osm.value === "string" ? fromCompact(osm.value, "osm") : void 0;
+		if (hours === void 0) notes.push(`the OpenStreetMap hours cannot be read: ${JSON.stringify(String(osm.value).slice(0, 200))}`);
+		else {
+			readings.push(reading("osm", hours, null, {
+				url: osm.url ?? null,
+				doc: osm.doc ?? null,
+				capture: osm.capture ?? null,
+				fetched_at: osm.fetched_at ?? null,
+				path: null
+			}));
+			if (typeof osm.capture === "string") captures.add(osm.capture);
+		}
+	}
+	/** This function fetches one page, and gives its document when the recipe can read it. */
+	const readPage = async (pageUrl) => {
+		const page = await fetchPage(runtime, pageUrl, pageOptions);
+		const doc = page.doc === null ? void 0 : runtime.index.get(page.doc);
+		const held = doc !== void 0 && isHeldBack(doc, userProvided);
+		pages.push({
+			url: pageUrl,
+			doc: page.doc,
+			outcome: page.outcome,
+			status: page.status,
+			rendered: page.rendered,
+			capture: page.capture,
+			fetched_at: page.fetched_at,
+			from_cache: page.from_cache,
+			held_back: held,
+			...page.archived === void 0 ? {} : { archived: page.archived },
+			note: page.note
+		});
+		if (!answered$1(page)) notes.push(`${pageUrl}: ${page.outcome}${page.note === null ? "" : ` (${page.note})`}`);
+		if (doc === void 0) return void 0;
+		if (held) {
+			heldBack = true;
+			return;
+		}
+		if (doc.ai_input === "no") await recordOverride(runtime, doc, job);
+		captures.add(doc.capture);
+		return doc;
+	};
+	const readDoc = (doc) => {
+		const read = venueReadingsOf(doc, { location: options.location });
+		readings.push(...read.readings);
+		notes.push(...read.notes);
+		phone ??= read.phone;
+		for (const menu of read.menus) if (menus.length < VENUE_LIMITS.menus && !menus.some((known) => known.url === menu.url)) menus.push(menu);
+	};
+	let homeUrl = null;
+	let namesVenueResult = null;
+	const home = await readPage(site.href);
+	if (home !== void 0) {
+		homeUrl = archivedOf(home.meta)?.original ?? home.url ?? site.href;
+		if (options.name !== void 0) {
+			namesVenueResult = namesVenue(options.name, home) ?? null;
+			if (namesVenueResult === false) notes.push(`${homeUrl} does not name ${options.name}: the domain can belong to a different business now`);
+		}
+		if (namesVenueResult === false && options.requireName === true) notes.push("requireName: the recipe read no hours from a site that does not name the venue");
+		else {
+			readDoc(home);
+			for (const link of rankHoursLinks(linksOf(home.meta), homeUrl).slice(0, extra)) {
+				if (options.signal?.aborted === true) break;
+				const doc = await readPage(link.href);
+				if (doc !== void 0) readDoc(doc);
+			}
+		}
+	}
+	const moved = homeUrl !== null && hostOf(homeUrl) !== hostOf(site.href) ? homeUrl : null;
+	if (moved !== null) notes.push(`${site.href} now goes to ${moved}`);
+	if (heldBack) notes.push(HELD_BACK);
+	const firstOf = /* @__PURE__ */ new Map();
+	for (const one of readings) {
+		const key = `${one.from}|${one.label ?? ""}|${one.hours}|${one.season ?? ""}`;
+		if (!firstOf.has(key)) firstOf.set(key, one);
+	}
+	const unique = [...firstOf.values()].sort((a, b) => a.rank - b.rank || Number(a.inferred) - Number(b.inferred));
+	const disagree = /* @__PURE__ */ new Set();
+	const whole = unique.filter((one) => one.rank <= 2);
+	for (let i = 0; i < whole.length; i += 1) for (let j = i + 1; j < whole.length; j += 1) for (const day of disagreements(whole[i].week, whole[j].week)) disagree.add(day);
+	const best = unique.find((one) => one.schedule === "opening") ?? null;
+	const fromSite = unique.find((one) => one.from !== "osm" && one.schedule === "opening" && !one.inferred) ?? null;
+	return {
+		url: site.href,
+		home: homeUrl,
+		moved,
+		names_venue: namesVenueResult,
+		pages,
+		readings: unique,
+		best,
+		from_site: fromSite,
+		disagree: DAYS.filter((day) => disagree.has(day)),
+		menus,
+		phone,
+		captures: [...captures],
+		held_back: heldBack,
+		note: notes.length > 0 ? notes.join("; ") : null
+	};
+};
+//#endregion
+//#region src/recipe-product.ts
+/** These are the limits of the product reader. A page with more is hostile or broken, and the reader stops there. */
+const PRODUCT_LIMITS = {
+	records: 500,
+	offers: 200,
+	variants: 200,
+	depth: 2,
+	priceChars: 64
+};
+const PRODUCT_TYPES = /^(?:Product|ProductGroup|IndividualProduct|ProductModel|SomeProducts)$/;
+/** A list price or a struck-through price is the old price, not the price. */
+const WAS_PRICE = /ListPrice|StrikethroughPrice|MSRP|SRP/i;
+const GTIN_KEYS = [
+	"gtin13",
+	"gtin12",
+	"gtin14",
+	"gtin8",
+	"gtin"
+];
+/** Two symbols that name one currency only. The other symbols (`$`, `¥`) name more than one. */
+const SYMBOLS = {
+	"€": "EUR",
+	"£": "GBP"
+};
+let knownCurrencies;
+/** This function tells if a code is an ISO 4217 currency that the runtime knows. */
+const isCurrency = (code) => {
+	if (!/^[A-Z]{3}$/.test(code)) return false;
+	if (knownCurrencies === void 0) try {
+		knownCurrencies = new Set(Intl.supportedValuesOf("currency"));
+	} catch {
+		knownCurrencies = /* @__PURE__ */ new Set();
+	}
+	return knownCurrencies.size === 0 || knownCurrencies.has(code);
+};
+/** This function gives the number of digits after the decimal point of a currency: 2 for EUR, 0 for JPY. */
+const fractionDigits = (currency) => {
+	if (currency === void 0) return void 0;
+	try {
+		return new Intl.NumberFormat("en", {
+			style: "currency",
+			currency
+		}).resolvedOptions().maximumFractionDigits;
+	} catch {
+		return;
+	}
+};
+/**
+* This function reads an amount as a shop writes it: `19.99`, `1,299.00`, `1.299,00`, `$19.99` or
+* the number 19.99. The last separator is the decimal point, with one exception. One kind of
+* separator, each followed by three digits, separates thousands: `1.299` is 1299. For a
+* currency with three decimal digits, one point with three digits is the decimal point. A text
+* with two numbers (a range), a negative amount or zero gives undefined.
+*/
+const readAmount = (value, currency) => {
+	if (typeof value === "number") return Number.isFinite(value) && value > 0 ? value : void 0;
+	if (typeof value !== "string" || value.length > PRODUCT_LIMITS.priceChars) return void 0;
+	const match = /^[^\d-]*?(\d(?:[\d.,'   ]*\d)?)[^\d]*$/.exec(value.trim());
+	if (match === null) return void 0;
+	let digits = match[1].replace(/['   ]/g, "");
+	const last = Math.max(digits.lastIndexOf("."), digits.lastIndexOf(","));
+	if (last !== -1) {
+		const groups = new Set(digits.match(/[.,]/g)).size === 1 && /^[1-9]\d{0,2}(?:[.,]\d{3})+$/.test(digits);
+		const decimalThree = groups && fractionDigits(currency) === 3 && /^\d+\.\d{3}$/.test(digits);
+		digits = groups && !decimalThree ? digits.replace(/[.,]/g, "") : `${digits.slice(0, last).replace(/[.,]/g, "")}.${digits.slice(last + 1)}`;
+	}
+	const amount = Number(digits);
+	return Number.isFinite(amount) && amount > 0 ? amount : void 0;
+};
+const list = (value, limit) => (Array.isArray(value) ? value : value === void 0 || value === null ? [] : [value]).slice(0, limit);
+const objects = (value, limit) => list(value, limit).filter((item) => typeof item === "object" && item !== null && !Array.isArray(item));
+/** This function gives a name, written as a text or as an object that has one. */
+const nameOf = (value) => {
+	for (const item of list(value, 20)) {
+		if (typeof item === "string" && item.trim().length > 0) return item.trim().slice(0, 500);
+		if (typeof item === "object" && item !== null) {
+			const named = item["name"];
+			if (typeof named === "string" && named.trim().length > 0) return named.trim().slice(0, 500);
+		}
+	}
+	return null;
+};
+const textOf = (value) => typeof value === "string" && value.trim().length > 0 ? value.trim().slice(0, 200) : typeof value === "number" && Number.isFinite(value) ? String(value) : null;
+const imageOf = (value) => {
+	for (const item of list(value, 20)) {
+		if (typeof item === "string" && /^https?:/i.test(item)) return item;
+		const url = typeof item === "object" && item !== null ? item["url"] : void 0;
+		if (typeof url === "string" && /^https?:/i.test(url)) return url;
+	}
+	return null;
+};
+const emptyOffers = () => ({
+	prices: [],
+	unknownCurrencies: /* @__PURE__ */ new Set(),
+	availability: /* @__PURE__ */ new Set(),
+	sellers: /* @__PURE__ */ new Set(),
+	unstated: false,
+	steps: 0
+});
+/** This function reads the offers of a product into `read`, and the offers of its variants to `depth` levels. */
+const readOffers = (product, read, depth) => {
+	const money = (value, currency) => {
+		const written = typeof currency === "string" ? currency.trim() : "";
+		const code = SYMBOLS[written] ?? written.toUpperCase();
+		const amount = readAmount(value, code.length > 0 ? code : void 0);
+		if (amount === void 0) return;
+		if (code.length === 0) {
+			read.unstated = true;
+			return;
+		}
+		if (!isCurrency(code)) {
+			read.unknownCurrencies.add(written.slice(0, 20));
+			return;
+		}
+		read.prices.push({
+			amount,
+			currency: code,
+			raw: String(value).slice(0, PRODUCT_LIMITS.priceChars)
+		});
+	};
+	const offer = (o, level) => {
+		read.steps += 1;
+		const spec = objects(o["priceSpecification"], PRODUCT_LIMITS.offers).find((entry) => {
+			const kind = entry["priceType"];
+			return !(typeof kind === "string" && WAS_PRICE.test(kind));
+		});
+		const currency = o["priceCurrency"] ?? spec?.["priceCurrency"];
+		money(o["price"] ?? spec?.["price"], currency);
+		money(o["lowPrice"], currency);
+		money(o["highPrice"], currency);
+		const availability = typeof o["availability"] === "string" ? o["availability"].trim().replace(/^.*[/#:]/, "").slice(0, 60) : void 0;
+		if (availability !== void 0 && availability.length > 0) read.availability.add(availability);
+		const seller = nameOf(o["seller"]);
+		if (seller !== null) read.sellers.add(seller);
+		if (level < PRODUCT_LIMITS.depth) for (const inner of objects(o["offers"], PRODUCT_LIMITS.offers)) offer(inner, level + 1);
+	};
+	for (const o of objects(product["offers"], PRODUCT_LIMITS.offers)) offer(o, 0);
+	if (depth <= 0) return;
+	for (const variant of objects(product["hasVariant"], PRODUCT_LIMITS.variants)) {
+		read.steps += 1;
+		readOffers(variant, read, depth - 1);
+	}
+};
+const availabilityOf = (read) => read.availability.size === 0 ? null : read.availability.size === 1 || !read.availability.has("InStock") ? [...read.availability][0] : "InStock";
+const pathKey = (value, page) => {
+	if (value.startsWith("#")) return void 0;
+	try {
+		const url = new URL(value, page);
+		return `${url.hostname.toLowerCase().replace(/^www\./, "")}${url.pathname.replace(/\/$/, "")}`;
+	} catch {
+		return;
+	}
+};
+/** This function gives the size of a record as JSON, or 0 for a record that JSON cannot write (a nesting too deep). */
+const sizeOf = (record) => {
+	try {
+		return JSON.stringify(record).length;
+	} catch {
+		return 0;
+	}
+};
+/**
+* This function picks the product of a page from its JSON-LD records. A page often has more
+* records of products: the related products, the products that others bought. It picks:
+*
+* 1. The product whose `url` or `@id` is the page (the path, with no query).
+* 2. Else the largest product with offers or variants.
+* 3. Else the largest product.
+*
+* It also reads the `mainEntity` of a record (a `WebPage` or an `ItemPage` that names its product).
+*/
+const pickProduct = (records, page) => {
+	let steps = 0;
+	const products = [];
+	for (const record of records.slice(0, PRODUCT_LIMITS.records)) {
+		steps += 1;
+		if (typesOf(record).some((type) => PRODUCT_TYPES.test(type))) products.push(record);
+		for (const inner of objects(record["mainEntity"], 20)) {
+			steps += 1;
+			if (typesOf(inner).some((type) => PRODUCT_TYPES.test(type))) products.push(inner);
+		}
+	}
+	const here = pathKey(page, page);
+	const own = products.find((product) => [product["url"], product["@id"]].some((value) => typeof value === "string" && here !== void 0 && pathKey(value, page) === here));
+	if (own !== void 0) return {
+		product: own,
+		steps
+	};
+	const selling = products.filter((product) => list(product["offers"], 1).length > 0 || list(product["hasVariant"], 1).length > 0);
+	const pool = selling.length > 0 ? selling : products;
+	let best;
+	let size = -1;
+	for (const product of pool) {
+		const length = sizeOf(product);
+		steps += length;
+		if (length > size) {
+			best = product;
+			size = length;
+		}
+	}
+	return {
+		product: best,
+		steps
+	};
+};
+/** This function gives the variant that a URL names: the `variant` parameter of Shopify, or a `sku` or `gtin` parameter. */
+const pickVariant = (product, page) => {
+	let params;
+	try {
+		params = new URL(page).searchParams;
+	} catch {
+		return;
+	}
+	const variantId = params.get("variant");
+	const code = params.get("sku") ?? params.get("gtin");
+	if (variantId === null && code === null) return void 0;
+	const names = (value) => {
+		if (typeof value !== "string" || variantId === null) return false;
+		try {
+			return new URL(value, page).searchParams.get("variant") === variantId || value.endsWith(`/${variantId}`);
+		} catch {
+			return false;
+		}
+	};
+	return objects(product["hasVariant"], PRODUCT_LIMITS.variants).find((variant) => {
+		const offers = objects(variant["offers"], PRODUCT_LIMITS.offers);
+		const codes = [
+			variant["sku"],
+			variant["gtin"],
+			variant["gtin13"],
+			variant["gtin12"],
+			variant["productID"]
+		].map(textOf).filter((one) => one !== null);
+		return code !== null && codes.includes(code) || [
+			variant["@id"],
+			variant["url"],
+			...offers.map((offer) => offer["url"])
+		].some(names) || variantId !== null && codes.includes(variantId);
+	});
+};
+const byAmount = (a, b) => a.amount - b.amount;
+/**
+* This function reads the Product data of a page from its JSON-LD records. It is pure, and it sends
+* no request. Thus it also reads a version at a time, or the stored JSON-LD of a document.
+*
+* - The prices are in the first currency of the offers. A price with no currency is not a price.
+* - A list price or a struck-through price is not the price.
+* - A currency must be an ISO 4217 code (`€` and `£` count too). The notes name each other one.
+*/
+const productOf = (records, page) => {
+	const picked = pickProduct(records, page);
+	const product = picked.product;
+	if (product === void 0) return {
+		found: false,
+		name: null,
+		brand: null,
+		sku: null,
+		gtin: null,
+		price: null,
+		price_to: null,
+		availability: null,
+		availabilities: [],
+		variant: null,
+		seller: null,
+		image: null,
+		notes: ["no schema.org Product data on the page: a list of products, or a shop that does not publish it"],
+		steps: picked.steps
+	};
+	const read = emptyOffers();
+	readOffers(product, read, PRODUCT_LIMITS.depth);
+	const currency = read.prices[0]?.currency;
+	const sorted = read.prices.filter((price) => price.currency === currency).sort(byAmount);
+	const low = sorted[0] ?? null;
+	const high = sorted.at(-1) ?? null;
+	const gtin = GTIN_KEYS.map((key) => textOf(product[key])).find((value) => value !== null);
+	const chosen = pickVariant(product, page);
+	let variant = null;
+	let steps = picked.steps + read.steps;
+	if (chosen !== void 0) {
+		const own = emptyOffers();
+		readOffers(chosen, own, 0);
+		steps += own.steps;
+		const prices = own.prices.filter((price) => price.currency === currency).sort(byAmount);
+		variant = {
+			name: nameOf(chosen["name"]),
+			sku: textOf(chosen["sku"]),
+			price: prices[0] ?? null,
+			availability: availabilityOf(own)
+		};
+	}
+	const notes = [
+		read.unknownCurrencies.size > 0 ? `a currency that is not an ISO 4217 code: ${[...read.unknownCurrencies].join(", ")}` : null,
+		read.unstated && sorted.length === 0 ? "the page gives a price with no currency, so the price is not read as money" : sorted.length === 0 ? "the page names the product and gives no price" : null,
+		new Set(read.prices.map((price) => price.currency)).size > 1 ? `the offers have more than one currency: the prices are in ${currency ?? ""}` : null,
+		/[?&](?:variant|sku|gtin)=/.test(page) && chosen === void 0 ? "the URL names a variant that the Product data does not list: the price is the whole range" : null
+	].filter((note) => note !== null);
+	return {
+		found: true,
+		name: nameOf(product["name"]),
+		brand: nameOf(product["brand"]),
+		sku: textOf(product["sku"]),
+		gtin: gtin ?? null,
+		price: low,
+		price_to: high !== null && low !== null && high.amount > low.amount ? high : null,
+		availability: availabilityOf(read),
+		availabilities: [...read.availability],
+		variant,
+		seller: read.sellers.size === 0 ? null : [...read.sellers].join(", "),
+		image: imageOf(product["image"]),
+		notes,
+		steps
+	};
+};
+const WALLS = /* @__PURE__ */ new Set([
+	"blocked",
+	"challenge",
+	"payment-required"
+]);
+/** This function tells if the site refused the crawler. */
+const wallOf = (page) => WALLS.has(page.outcome) || page.outcome === "skipped" && /^(?:robots\.txt disallows|host stopped)/.test(page.note ?? "") || page.doc === null && /\bbot check\b/.test(page.note ?? "");
+const answered = (page) => page.doc !== null && (page.outcome === "ok" || page.outcome === "not-modified" || page.archived !== void 0);
+/**
+* This function gives the JSON-LD records of a document from its stored body. The index keeps a
+* part of the JSON-LD of a large page only, and a product with many variants can be larger. An
+* archived copy gives the JSON-LD that the index keeps.
+*/
+const recordsOfDoc = async (runtime, doc) => {
+	const kept = Array.isArray(doc.meta["jsonld"]) ? doc.meta["jsonld"] : [];
+	if (doc.blob === null || archivedOf(doc.meta) !== void 0) return kept;
+	const body = await runtime.blobs.get(doc.blob);
+	if (body === void 0) return kept;
+	const at = doc.url ?? "";
+	const type = doc.rendered ? "text/html; charset=utf-8" : runtime.index.latest(at, "plain")?.content_type ?? null;
+	try {
+		return extractPage(decodeBody(body, type), at).jsonld;
+	} catch {
+		return kept;
+	}
+};
+/**
+* This function reads a product page. It fetches the page through the crawler: robots.txt, the
+* pace, the stops, the capture log and the rule for a held-back page apply. When the plain page has
+* no Product data, it renders the page once, because some shops write that data with JavaScript.
+*
+* A shop that refuses the crawler gives `wall: true` and a note. The recipe writes nothing but the
+* capture log. `extract` can check each value against the stored page.
+*/
+const readProduct = async (target, url, options = {}) => {
+	const runtime = runtimeOf(target);
+	try {
+		const parsed = new URL(url);
+		if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new TypeError(url);
+	} catch {
+		throw new TypeError(`readProduct needs an http or https URL, not ${JSON.stringify(url)}`);
+	}
+	const maxAge = options.maxAge ?? "24h";
+	const job = options.job ?? "product";
+	const userProvided = options.userProvided ?? false;
+	const render = options.render ?? "auto";
+	const deadline = deadlineOf(runtime, options.budget);
+	const pageOptions = (mode) => ({
+		render: mode,
+		maxAgeMs: maxAge === "auto" ? "auto" : durationMs(maxAge),
+		onBehalfOf: options.onBehalfOf,
+		job,
+		signal: options.signal,
+		scope: options.scope,
+		labels: options.labels,
+		deadline,
+		fallback: options.fallback,
+		archiveDespiteRobots: options.archiveDespiteRobots
+	});
+	let page = await fetchPage(runtime, url, pageOptions(render));
+	const notes = [];
+	const result = (from, read, more = {}) => {
+		const all = [
+			from.note,
+			...notes,
+			...read?.notes ?? []
+		].filter((part) => part !== null && part !== void 0 && part.length > 0);
+		return {
+			url,
+			found: read?.found ?? false,
+			verified: read?.found === true && read.name !== null,
+			name: read?.name ?? null,
+			brand: read?.brand ?? null,
+			sku: read?.sku ?? null,
+			gtin: read?.gtin ?? null,
+			price: read?.price ?? null,
+			price_to: read?.price_to ?? null,
+			availability: read?.availability ?? null,
+			availabilities: read?.availabilities ?? [],
+			variant: read?.variant ?? null,
+			seller: read?.seller ?? null,
+			image: read?.image ?? null,
+			wall: wallOf(from),
+			outcome: from.outcome,
+			status: from.status,
+			rendered: from.rendered,
+			doc: from.doc,
+			capture: from.capture,
+			fetched_at: from.fetched_at,
+			captures: [...from.captures],
+			from_cache: from.from_cache,
+			held_back: more.held_back ?? false,
+			...from.archived === void 0 ? {} : { archived: from.archived },
+			note: all.length > 0 ? [...new Set(all)].join("; ") : null
+		};
+	};
+	if (!answered(page)) {
+		const wall = wallOf(page);
+		notes.push(wall ? `the shop refused the crawler (${page.outcome}): the recipe read nothing, and it does not try a different identity` : `the page gave no document (${page.outcome})`);
+		return result(page, void 0);
+	}
+	const held = (from) => {
+		const doc = from.doc === null ? void 0 : runtime.index.get(from.doc);
+		return doc !== void 0 && isHeldBack(doc, userProvided) ? doc : void 0;
+	};
+	if (held(page) !== void 0) {
+		notes.push(HELD_BACK);
+		return result(page, void 0, { held_back: true });
+	}
+	const docOf = (from) => from.doc === null ? void 0 : runtime.index.get(from.doc);
+	let doc = docOf(page);
+	let read = doc === void 0 ? void 0 : productOf(await recordsOfDoc(runtime, doc), url);
+	if (read?.found !== true && !page.rendered && render === "auto" && page.archived === void 0) {
+		const rendered = await fetchPage(runtime, url, pageOptions("always"));
+		const shown = docOf(rendered);
+		if (rendered.rendered && answered(rendered) && shown !== void 0 && held(rendered) === void 0) {
+			page = rendered;
+			doc = shown;
+			read = productOf(await recordsOfDoc(runtime, shown), url);
+		} else if (rendered.note !== null && !rendered.rendered) notes.push(`the render gave nothing more: ${rendered.note}`);
+	}
+	if (userProvided && doc?.ai_input === "no") await recordOverride(runtime, doc, job);
+	return result(page, read);
+};
+//#endregion
+export { readAmount as a, locationMatches as c, readVenue as d, venueReadingsOf as f, productOf as i, namesVenue as l, pickProduct as n, readProduct as o, pickVariant as r, VENUE_LIMITS as s, PRODUCT_LIMITS as t, rankHoursLinks as u };
+
+//# sourceMappingURL=recipe-product-Dellqudu.mjs.map