polite-crawler 0.1.4 → 0.2.0

package/build/politeness.d.mts

@@ -1,2 +1,366 @@
-import { A as validatorsToSend, C as backoffMs, D as policyFor, E as nextValidatorMemo, O as realClock, S as ValidatorVerdict, T as equalJitterMs, _ as VALIDATOR_MEMO, a as HostScheduler, b as ValidatorMemo, c as HostStopped, d as Paced, f as PolicyOptions, g as SchedulerOptions, h as RestInput, i as HostResult, k as restAfter, l as MAX_STOP_MS, m as Rest, n as DEFAULT_POLICY, o as HostStateStore, p as PolitenessPolicy, r as HostCounts, s as HostStop, t as Clock, u as MemoryHostStateStore, v as ValidatorAnswer, w as checkPolicy, x as ValidatorReport, y as ValidatorLimits } from "./politeness-yI3xBEx9.mjs";
-export { Clock, DEFAULT_POLICY, HostCounts, HostResult, HostScheduler, HostStateStore, HostStop, HostStopped, MAX_STOP_MS, MemoryHostStateStore, Paced, PolicyOptions, PolitenessPolicy, Rest, RestInput, SchedulerOptions, VALIDATOR_MEMO, ValidatorAnswer, ValidatorLimits, ValidatorMemo, ValidatorReport, ValidatorVerdict, backoffMs, checkPolicy, equalJitterMs, nextValidatorMemo, policyFor, realClock, restAfter, validatorsToSend };
+//#region src/politeness.d.ts
+/**
+ * The scheduler controls the pace of each host. The speed of a polite crawler is the number of
+ * hosts in progress divided by the delay for each host. Thus the scheduler keeps one request in
+ * progress for each host, and it waits the delay of the host between two requests. It uses its
+ * concurrency across many hosts, as the back queues of the Mercator crawler do.
+ *
+ * The caller decides the delay after each request, and `run` gets it from the task. Only the caller
+ * knows the documented rate of the source, the crawl delay of the site and the duration of the
+ * request. A wait holds no global slot, so a host that waits does not block a different host.
+ *
+ * The state of each host (its next time, its counts and its stop) is in a `HostStateStore`. The
+ * default store is in memory. A store that processes share keeps one pace for each host across them.
+ */
+export interface Clock {
+  readonly now: () => number;
+  /** This function waits `ms` milliseconds. When the signal aborts, the wait ends early. */
+  readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
+}
+export declare const realClock: Clock;
+/** The crawler does not ask a host again until the stop ends, after a refusal (a challenge, 403 or 402). */
+export declare class HostStopped extends Error {
+  readonly host: string;
+  readonly reason: string;
+  /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */
+  readonly until: number;
+  constructor(host: string, reason: string, until?: number);
+}
+export interface Paced<T> {
+  readonly value: T;
+  /** This value is the time that the host rests before its next request. */
+  readonly delayMs: number;
+}
+/** A stop keeps the crawler away from a host until a time. */
+export interface HostStop {
+  /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */
+  readonly until: number;
+  readonly reason: string;
+}
+/** These are the counts of a host. Each count gives the results of one kind in sequence. */
+export interface HostCounts {
+  /** This value counts errors and requests with no answer. At the limit of the policy, the host stops. */
+  readonly failures: number;
+  /** This value counts the requests to wait (429, 502, 503, 504). It sets the backoff and does not stop the host. */
+  readonly waits: number;
+}
+/**
+ * This value tells how one result changes the counts of a host:
+ *
+ * - `success` sets the two counts to zero.
+ * - `failure` adds one to `failures`.
+ * - `wait` adds one to `waits`.
+ * - `neutral` changes nothing. A client error (a 400) is neutral, because a different request can succeed.
+ */
+export type HostResult = "success" | "failure" | "wait" | "neutral";
+/**
+ * A host state store keeps the state of each host for the scheduler. Its methods are synchronous,
+ * and each method is one atomic step on the state of one host. Thus a store can be a table of
+ * SQLite that processes share. Times are milliseconds since the epoch.
+ *
+ * The state of a host has five parts:
+ *
+ * - the next time: the earliest time of the next request
+ * - the lease: the end of the slot of the request in progress, when there is one
+ * - the counts (`HostCounts`)
+ * - the stop: an end time and a reason, when there is one
+ */
+export interface HostStateStore {
+  /**
+   * This method asks for the slot of a host at the time `now`. When the host has no rest left
+   * and no lease, the store gives the slot to the caller until `now + leaseMs` and gives 0.
+   * Otherwise it changes nothing and gives the time to wait in milliseconds: the time to the next
+   * time, or to the end of the lease. A store that processes share gives a short wait (one second
+   * or less) during a lease, because the holder usually releases it early. The caller asks again after the wait.
+   */
+  reserve(host: string, now: number, leaseMs: number): number;
+  /** This method changes the counts of a host for one result, and it gives the new counts. */
+  record(host: string, result: HostResult): HostCounts;
+  /**
+   * This method ends the lease of the caller. It sets the next time to `nextAt` when `nextAt` is
+   * later than the stored next time. Only the holder of the lease uses it.
+   */
+  release(host: string, nextAt: number): void;
+  /** This method sets the next time to `until` when `until` is later. It does not touch the lease. */
+  defer(host: string, until: number): void;
+  /** This method stops a host until a time. When the host has a stop that ends later, that stop stays. */
+  stop(host: string, until: number, reason: string): void;
+  /** This method gives the stop of a host at the time `now`. A stop that ended gives no value. */
+  stopped(host: string, now: number): HostStop | undefined;
+  /**
+   * This method removes the stops and the counts of one host, or of each host when there is no
+   * host. It keeps the next times and the leases, so a reset cannot make a host go faster than its pace.
+   */
+  reset(host?: string): void;
+  /**
+   * This method gives the validator memo of a host, when the store has one. A store without this
+   * method and `noteValidators` has no memo. The crawler then sends each validator of a stored copy.
+   */
+  validators?(host: string): ValidatorMemo | undefined;
+  /**
+   * This method adds one conditional request to the memo of a host, with the rules of
+   * `nextValidatorMemo`, in one atomic step. It gives the new memo.
+   */
+  noteValidators?(host: string, report: ValidatorReport, now: number): ValidatorMemo;
+}
+/** This value tells the result that the crawler learned for one validator of a host. */
+export type ValidatorVerdict = "unknown" | "works" | "fails";
+/**
+ * This value tells what one conditional request got:
+ *
+ * - `not-modified`: a 304.
+ * - `same`: a 200 with the body that the store has. Thus the validator did not match.
+ * - `changed`: a 200 with a different body. This answer tells nothing about the validator.
+ */
+export type ValidatorAnswer = "not-modified" | "same" | "changed";
+/** This report tells the validators that one conditional request sent, and the answer that it got. */
+export interface ValidatorReport {
+  /** This value is true when the request sent `If-None-Match`. */
+  readonly etag: boolean;
+  /** This value is true when the request sent `If-Modified-Since`. */
+  readonly lastModified: boolean;
+  readonly answer: ValidatorAnswer;
+}
+/**
+ * A validator memo holds what the crawler learned about the validators of one host. A server
+ * examines `If-None-Match` first, and it ignores `If-Modified-Since` when the request has the two
+ * (RFC 9110, section 13.1.3). Thus a host with unstable ETags does not answer 304 to a request with
+ * the two validators at any time. Its stable `Last-Modified` date does not help. `nextValidatorMemo`
+ * gives the rules.
+ */
+export interface ValidatorMemo {
+  /** This value is true when the crawler leaves out `If-None-Match` for the host and sends only `If-Modified-Since`. */
+  readonly skipEtag: boolean;
+  /** This value tells if a request with `If-None-Match` got a 304 (`works`) or the stored body again (`fails`). */
+  readonly etag: ValidatorVerdict;
+  /** This value tells the same for a request with only `If-Modified-Since`. */
+  readonly lastModified: ValidatorVerdict;
+  /** This value counts the stored bodies in sequence that came back to a request with `If-None-Match`. */
+  readonly etagMisses: number;
+  /** This value counts the stored bodies in sequence that came back to a request with only `If-Modified-Since`. */
+  readonly dateMisses: number;
+  /** This value counts the requests with only `If-Modified-Since` since the crawler stopped the ETag, or since the last probe. */
+  readonly sinceProbe: number;
+  /** This time (milliseconds since the epoch) is the last time that the crawler stopped or started the ETag, or probed it. */
+  readonly changedAt: number;
+}
+/** These are the limits of the validator memo. */
+export interface ValidatorLimits {
+  /** After this number of stored bodies in sequence, a validator fails. The value is 3. */
+  readonly misses: number;
+  /** After this number of requests with only `If-Modified-Since`, the next request sends the ETag again (a probe). The value is 20. */
+  readonly probeEvery: number;
+  /** After this time with no probe, the next request is a probe. The value is 7 days. */
+  readonly probeAgeMs: number;
+}
+/** These are the limits of the validator memo of the crawler. */
+export declare const VALIDATOR_MEMO: ValidatorLimits;
+/**
+ * This function tells which validators the next conditional request to a host sends. It sends
+ * each validator of the stored copy, except when the memo stopped the ETag and the copy has a
+ * `Last-Modified` date. Then it sends only `If-Modified-Since`, and the ETag too for a probe.
+ * A probe comes after `probeEvery` requests or after `probeAgeMs`.
+ */
+export declare const validatorsToSend: (memo: ValidatorMemo | undefined, stored: {
+  readonly etag: boolean;
+  readonly lastModified: boolean;
+}, now: number) => {
+  readonly etag: boolean;
+  readonly lastModified: boolean;
+};
+/**
+ * This function adds one conditional request to the memo of a host, and gives the new memo.
+ * These are the rules, with the limits of `VALIDATOR_MEMO`:
+ *
+ * 1. A 304 to a request with `If-None-Match` marks the ETag `works`. After a probe, the crawler sends the ETag again.
+ * 2. After `misses` stored bodies in sequence with `If-None-Match`, the ETag fails. When the request also sent
+ *    `If-Modified-Since`, the crawler stops the ETag for the host.
+ * 3. A probe that gets a 200 keeps the ETag stopped, and the count to the next probe starts again.
+ * 4. After `misses` stored bodies in sequence with only `If-Modified-Since`, that validator fails too. Then the
+ *    crawler sends the two validators again, because they cost nothing. It tries the date alone again after `probeAgeMs`.
+ * 5. A changed body tells nothing about a validator.
+ */
+export declare const nextValidatorMemo: (memo: ValidatorMemo | undefined, report: ValidatorReport, now: number) => ValidatorMemo;
+/**
+ * This store keeps the state of each host in memory, for one process. It removes the state of an
+ * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.
+ * It keeps the validator memos of the `maxHosts` hosts that it used last.
+ */
+export declare class MemoryHostStateStore implements HostStateStore {
+  private readonly maxHosts;
+  private readonly states;
+  /** The validator memos, the memo used last at the end. */
+  private readonly memos;
+  private sweepAt;
+  constructor(maxHosts?: number);
+  /** This value is the number of hosts that the store holds. */
+  get size(): number;
+  private state;
+  private sweep;
+  reserve(host: string, now: number, leaseMs: number): number;
+  record(host: string, result: HostResult): HostCounts;
+  release(host: string, nextAt: number): void;
+  defer(host: string, until: number): void;
+  stop(host: string, until: number, reason: string): void;
+  stopped(host: string, now: number): HostStop | undefined;
+  reset(host?: string): void;
+  validators(host: string): ValidatorMemo | undefined;
+  noteValidators(host: string, report: ValidatorReport, now: number): ValidatorMemo;
+}
+/** These options change the scheduler. */
+export interface SchedulerOptions {
+  /** This store keeps the state of each host. The default is a `MemoryHostStateStore`. */
+  readonly store?: HostStateStore | undefined;
+  /** This value is the longest slot of one request. A slot that a crashed process holds ends after it. The default is 5 minutes. */
+  readonly leaseMs?: number | undefined;
+  /** A host rests this long after a task that throws an error, or a task that gives no valid delay. The default is 1 second. */
+  readonly fallbackDelayMs?: number | undefined;
+}
+export declare class HostScheduler {
+  private readonly clock;
+  private readonly maxActive;
+  /** This store keeps the state of each host. */
+  readonly store: HostStateStore;
+  private readonly chains;
+  private active;
+  private readonly queue;
+  private readonly leaseMs;
+  private readonly fallbackDelayMs;
+  constructor(clock: Clock, maxActive: number, options?: SchedulerOptions);
+  private acquire;
+  private release;
+  private sleep;
+  /**
+   * This method starts one request on a host, in sequence, after the rest of the host. When the
+   * signal aborts during the wait, the promise rejects with the reason of the signal.
+   */
+  run<T>(host: string, task: () => Promise<Paced<T>>, signal?: AbortSignal): Promise<T>;
+  /** The server asked for a pause (Retry-After), or a backoff chose one. */
+  park(host: string, untilMs: number): void;
+  /** This method stops a host until `untilMs`. With no time, the stop has no end. */
+  stop(host: string, reason: string, untilMs?: number): void;
+  stoppedReason(host: string): string | undefined;
+  /** This method gives the stop of a host, with its end, when the host is stopped. */
+  stopOf(host: string): HostStop | undefined;
+  /** This value counts the failures of a host in sequence, for the backoff. A success sets it to zero. */
+  failed(host: string): number;
+  /** This value counts the requests to wait (429, 503) of a host in sequence. They do not stop the host. */
+  waited(host: string): number;
+  succeeded(host: string): void;
+  /** This method removes the stops and the counts of one host, or of each host. The pace stays. */
+  reset(host?: string): void;
+}
+/**
+ * This function gives an exponential backoff with full jitter, from the analysis of AWS. The wait
+ * is random, up to base × 2^(try − 1), with a maximum. The jitter keeps retries apart.
+ */
+export declare const backoffMs: (attempt: number, baseMs: number, capMs: number, random: () => number) => number;
+/**
+ * This function gives an exponential backoff with equal jitter. The wait is half of
+ * base × 2^(try − 1), with a maximum, and a random part up to the other half. Thus each backoff
+ * is at least half of its step, and it is not lost under the rest of the host.
+ */
+export declare const equalJitterMs: (attempt: number, baseMs: number, capMs: number, random: () => number) => number;
+/** These are the facts of one request that set the rest of its host. */
+export interface RestInput {
+  /** This value is the kind of the request: a web page, a documented API, or a render in the browser. */
+  readonly kind: "page" | "api" | "render";
+  /** This value is the duration of the request, in milliseconds. */
+  readonly elapsedMs: number;
+  /** This value is the `crawl-delay` of robots.txt, in seconds. */
+  readonly crawlDelaySeconds?: number | undefined;
+  /** This value is the documented pace of the source. It replaces the interval of the policy for the kind. */
+  readonly minIntervalMs?: number | undefined;
+  /** This value is the wait that the server asked for (`Retry-After`). */
+  readonly retryAfterMs?: number | undefined;
+}
+/** The rest of a host after a request, and the reason for its length. */
+export interface Rest {
+  readonly ms: number;
+  readonly reason: "pace" | "slow" | "retry-after" | "crawl-delay";
+}
+/**
+ * This function gives the rest of a host after one request, as the crawler calculates it. These
+ * are the parts:
+ *
+ * - The interval of the kind (3 seconds for a page), or the documented pace of the source.
+ * - The `crawl-delay` of robots.txt, up to `crawlDelayCapMs`, when it is longer.
+ * - After a page or a render, `elapsedFactor` times the duration of the request, up to `elapsedCapMs`.
+ * - The `Retry-After` of the server, when it is longer.
+ *
+ * The backoff after an error or a request to wait comes on top, with jitter. The function is
+ * pure, so a simulation and a study can use the pace of the crawler.
+ */
+export declare const restAfter: (policy: PolitenessPolicy, input: RestInput) => Rest;
+/**
+ * The politeness policy holds each limit of the crawler. Each value has a default, and a host can
+ * have values of its own. The times are in milliseconds.
+ */
+export interface PolitenessPolicy {
+  /** This value is the shortest rest of a host between two page requests. The default is 3 seconds. */
+  readonly pageIntervalMs: number;
+  /** This value is the shortest rest of a host between two API requests. The default is 1 second. */
+  readonly apiIntervalMs: number;
+  /** This value is the shortest rest of a host after a render. The default is 5 seconds. */
+  readonly renderIntervalMs: number;
+  /** After a page, the host rests this number of times the duration of the request. The default is 5. */
+  readonly elapsedFactor: number;
+  /** This value is the maximum of the rest that comes from the duration. The default is 30 seconds. */
+  readonly elapsedCapMs: number;
+  /** This value is the maximum `crawl-delay` that the crawler obeys. The default is 60 seconds. */
+  readonly crawlDelayCapMs: number;
+  /** This value is the maximum number of redirects for one fetch. The default is 5. */
+  readonly maxRedirects: number;
+  /** This value is the maximum size of a body after decompression. The default is 5 MiB. */
+  readonly maxBytes: number;
+  /** This value is the deadline of one request, the body included. The default is 90 seconds. */
+  readonly requestTimeoutMs: number;
+  /** This value is the deadline of one render. The default is 20 seconds. */
+  readonly renderTimeoutMs: number;
+  /** After this number of failures in sequence, the host stops. The default is 5. */
+  readonly maxFailures: number;
+  /** This value is the stop after too many failures. The default is 1 hour. */
+  readonly failureStopMs: number;
+  /** This value is the stop after a 401 or a 403. The default is 1 hour. */
+  readonly blockedStopMs: number;
+  /** This value is the stop after a bot challenge. The default is 24 hours. */
+  readonly challengeStopMs: number;
+  /** This value is the stop after a 402. The default is 24 hours. */
+  readonly paymentStopMs: number;
+  /** The crawler obeys a `Retry-After` up to this value. A longer wait stops the host for that time. The default is 1 hour. */
+  readonly maxRetryAfterMs: number;
+  /** This value is the first step of the backoff after an error or a request to wait. The default is 5 seconds. */
+  readonly backoffBaseMs: number;
+  /** This value is the maximum step of the backoff. The default is 300 seconds. */
+  readonly backoffCapMs: number;
+  /** The crawler keeps a robots.txt for this time. The default is 24 hours. */
+  readonly robotsTtlMs: number;
+  /** The crawler keeps an unreachable robots.txt verdict for this time first, with a backoff. The default is 5 minutes. */
+  readonly robotsRetryMs: number;
+  /** This value is the maximum time for an unreachable robots.txt verdict. The default is 1 hour. */
+  readonly robotsRetryCapMs: number;
+  /** A stored robots.txt younger than this age replaces a robots.txt that the crawler cannot read. The default is 30 days. */
+  readonly robotsStaleMs: number;
+  /** This value is the maximum freshness that the crawler gives a response with no freshness of its own. The default is 24 hours. */
+  readonly heuristicFreshnessCapMs: number;
+}
+/**
+ * This value is the longest stop that a `Retry-After` can give: 30 days. A wait longer than
+ * `maxRetryAfterMs` stops the host for the time that the server asks, up to this value.
+ */
+export declare const MAX_STOP_MS: number;
+export declare const DEFAULT_POLICY: PolitenessPolicy;
+/**
+ * These options change the politeness policy. The values at the top apply to each host. The
+ * `hosts` map gives values for one host (`api.example.org`) or for the hosts under a name
+ * (`*.example.org`). The most specific entry wins.
+ */
+export interface PolicyOptions extends Partial<PolitenessPolicy> {
+  readonly hosts?: Readonly<Record<string, Partial<PolitenessPolicy>>> | undefined;
+}
+/** This function examines the options of a policy. It throws a `RangeError` for a value that is not a valid number. */
+export declare const checkPolicy: (options: PolicyOptions | undefined) => void;
+/** This function gives the policy of one host: the defaults, then the general options, then the entries for the host. */
+export declare const policyFor: (options: PolicyOptions | undefined, host: string) => PolitenessPolicy;
+//#endregion
+//# sourceMappingURL=politeness.d.mts.map

package/build/{recipe-GiFQojmo.mjs → recipe-B2SkepDZ.mjs}

@@ -1,5 +1,5 @@
-import { i as durationMs } from "./version-BPem-XZd.mjs";
-import { C as fallbackReason, S as archivedOf, c as isHeldBack, i as fetchAndIndex, n as HELD_BACK, p as recordOverride, s as heldBackMeta, y as archivedCopy } from "./runtime-D_r2SYge.mjs";
+import { i as durationMs } from "./version-Djva7JkN.mjs";
+import { b as fallbackReason, c as isHeldBack, d as recordOverride, g as archivedCopy, i as fetchAndIndex, n as HELD_BACK, s as heldBackMeta, y as archivedOf } from "./runtime-C_2BQibH.mjs";
 import { o as decodeBody } from "./html-DkQUAE_d.mjs";
 //#region src/recipe.ts
 /**
@@ -27,7 +27,11 @@ const RESERVED_RECIPES = [
 	"sitemap",
 	"feed"
 ];
-/** These are the limits of a recipe. */
+/**
+* These are the limits of a recipe.
+*
+* @beta
+*/
 const RECIPE_LIMITS = {
 	nameChars: 64,
 	docIdChars: 2048
@@ -144,6 +148,8 @@ const defineRecipe = (definition) => {
 * This function gives the extractor of a recipe to the extractors of a runtime. A second
 * registration of the same recipe does nothing. A different extractor with the same name throws
 * an error, because `reindex` must read each capture with one extractor.
+*
+* @beta
 */
 const registerRecipe = (extractors, recipe) => {
 	const known = extractors.get(recipe.name);
@@ -276,4 +282,4 @@ const runRecipe = async (target, recipe, input, options = {}) => {
 //#endregion
 export { readJson as a, runtimeOf as c, defineRecipe as i, RESERVED_RECIPES as n, registerRecipe as o, deadlineOf as r, runRecipe as s, RECIPE_LIMITS as t };
 
-//# sourceMappingURL=recipe-GiFQojmo.mjs.map
+//# sourceMappingURL=recipe-B2SkepDZ.mjs.map

package/build/recipe-B2SkepDZ.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"recipe-B2SkepDZ.mjs","names":[],"sources":["../src/recipe.ts"],"sourcesContent":["/*\n * The recipe SDK. A recipe tells the crawler what to fetch and how to read it into documents.\n *\n * - `defineRecipe` declares a recipe: its name, the request for an input, and its extractor.\n * - `runRecipe` fetches one input through the crawler and gives the documents of the capture.\n * - `registerRecipe` gives the extractor to the extractors of a runtime. `openCrawler` and\n *   `createRuntime` take `recipes` and register them at open, so `reindex` reads their captures.\n *\n * Each fetch of a recipe is a fetch of the crawler. Thus robots.txt (for a page), the pace of each\n * host, the stops, the egress policy and the capture log apply. The documents are versions in the\n * index, so `find`, `read`, `verify`, `extract`, `diff` and the time travel use them.\n */\nimport type { FetchRequest } from \"./crawler.ts\";\nimport type { Doc, ExtractedDoc, VersionText } from \"./docs.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport type { Archived } from \"./fallback.ts\";\nimport { archivedCopy, archivedOf, fallbackReason } from \"./fallback.ts\";\nimport type { CrawlRuntime, Extractor, Extractors, Indexed, PageOptions } from \"./runtime.ts\";\nimport { HELD_BACK, fetchAndIndex, heldBackMeta, isHeldBack, recordOverride } from \"./runtime.ts\";\nimport type { AiInput } from \"./signals.ts\";\nimport { decodeBody } from \"./text.ts\";\n\n/** The capture that an extractor reads: the final URL of the response and its content type. */\nexport interface RecipeCapture {\n  readonly url: string;\n  readonly contentType: string | null;\n}\n\n/**\n * This function reads one captured body into the documents that it holds. It is pure: the same\n * body and capture always give the same documents. It sends no request. It can answer later.\n */\nexport type RecipeExtract = (\n  body: Uint8Array,\n  capture: RecipeCapture,\n) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;\n\n/** The request of a recipe for one input: the URL, and the parts that change with the input. */\nexport interface RecipeRequest {\n  readonly url: string;\n  /** This value replaces the `accept` of the recipe for this request. */\n  readonly accept?: string | undefined;\n  /** These headers go with the request. The crawler does not write them into the capture. */\n  readonly headers?: Readonly<Record<string, string>> | undefined;\n  /** These query parameters hold secrets. The capture shows `REDACTED` in their place. */\n  readonly secretParams?: readonly string[] | undefined;\n}\n\n/** These values declare a recipe. Only `name` and `extract` are necessary. */\nexport interface RecipeDefinition<Input = string> {\n  /**\n   * This name goes into each capture record of the recipe, so it is part of each capture id.\n   * `reindex` finds the extractor by it. Do not change it after the first capture.\n   */\n  readonly name: string;\n  /** This text tells what the recipe reads. */\n  readonly description?: string | undefined;\n  /**\n   * The value `page` (the default) is a web page: robots.txt applies. The value `api` is a\n   * documented API with terms of its own: robots.txt does not apply, and the pace still does.\n   */\n  readonly mode?: \"api\" | \"page\" | undefined;\n  /** This value is the `Accept` header. The defaults are HTML for a page and JSON for an API. */\n  readonly accept?: string | undefined;\n  /** This value is the documented pace of the source: the shortest time between two of its requests. */\n  readonly minInterval?: Duration | undefined;\n  /** A stored copy younger than this age answers with no request. The default is `24h`. */\n  readonly maxAge?: Duration | undefined;\n  /** This value is the most bytes of a body that the crawler keeps. */\n  readonly maxBytes?: number | undefined;\n  /** This function gives the request for an input. With no function, the input is the URL. */\n  readonly fetch?: ((input: Input) => string | RecipeRequest) | undefined;\n  /** This function reads a captured body into documents. It is pure, and it sends no request. */\n  readonly extract: RecipeExtract;\n}\n\n/** These options build the request of a recipe for one input (`Recipe.request`). */\nexport interface RecipeRequestOptions {\n  /** This value replaces the `maxAge` of the recipe. The value `0` asks the source again. */\n  readonly maxAge?: Duration | undefined;\n  /** These agents get their robots.txt rules applied too, for a recipe of the mode `page`. */\n  readonly onBehalfOf?: readonly string[] | undefined;\n  /** This name connects the captures to the job that asked for them. */\n  readonly job?: string | undefined;\n  /** When this signal aborts, the crawler stops the fetch. */\n  readonly signal?: AbortSignal | undefined;\n  /** This label goes into each capture record of the request. */\n  readonly scope?: string | undefined;\n  /** These labels go into each capture record of the request. They are not part of the capture id. */\n  readonly labels?: Readonly<Record<string, string>> | undefined;\n  /** No request starts at or after this time (milliseconds since the epoch, on the clock of the crawler). */\n  readonly deadline?: number | undefined;\n}\n\n/** A recipe that `defineRecipe` made. Give it to `runRecipe`, to `registerRecipe` or to `openCrawler`. */\nexport interface Recipe<Input = string> {\n  readonly name: string;\n  readonly description: string | null;\n  readonly mode: \"api\" | \"page\";\n  readonly accept: string;\n  /**\n   * This extractor reads the captures of the recipe. It checks the documents of the definition.\n   *\n   * @beta\n   */\n  readonly extractor: Extractor;\n  /**\n   * This method gives the request of the crawler for one input. A value that is not a URL throws a `TypeError`.\n   *\n   * @beta\n   */\n  request(input: Input, options?: RecipeRequestOptions): FetchRequest;\n}\n\n/**\n * This function reads a body as JSON, in the encoding that the response declares. A body that is\n * not JSON (a page about maintenance, or a login page) throws an error that names the URL and the\n * content type. In an extractor, the runtime then reports that error.\n */\nexport const readJson = (body: Uint8Array, capture: RecipeCapture): unknown => {\n  const text = decodeBody(body, capture.contentType);\n  try {\n    return JSON.parse(text) as unknown;\n  } catch {\n    throw new Error(\n      `${capture.url} answered ${capture.contentType ?? \"a body of no stated type\"}, not JSON`,\n    );\n  }\n};\n\n/**\n * These recipe names belong to the crawler: the page reader, the archived fallback, robots.txt,\n * and the sitemaps and feeds of `discover`. A recipe cannot use them.\n */\nexport const RESERVED_RECIPES: readonly string[] = [\n  \"page\",\n  \"archive-copy\",\n  \"archive-index\",\n  \"robots.txt\",\n  \"sitemap\",\n  \"feed\",\n];\n\n/**\n * These are the limits of a recipe.\n *\n * @beta\n */\nexport const RECIPE_LIMITS: {\n  /** The longest recipe name. */\n  readonly nameChars: number;\n  /** The longest document id. */\n  readonly docIdChars: number;\n} = { nameChars: 64, docIdChars: 2048 };\n\nconst NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;\nconst PAGE_ACCEPT = \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.5\";\nconst API_ACCEPT = \"application/json\";\nconst AI_INPUTS: readonly AiInput[] = [\"yes\", \"no\", \"unset\"];\n\n/** This function tells what is wrong with one document of an extractor, or gives undefined. */\nconst faultOf = (value: unknown, seen: Set<string>): string | undefined => {\n  if (value === null || typeof value !== \"object\" || Array.isArray(value))\n    return \"a document must be an object\";\n  const doc = value as Record<string, unknown>;\n  const id = doc[\"doc\"];\n  if (typeof id !== \"string\" || id.length === 0 || id.length > RECIPE_LIMITS.docIdChars)\n    return `doc must be a text of 1 to ${String(RECIPE_LIMITS.docIdChars)} characters`;\n  // the archived fallback labels the copies of third parties, and only it makes these\n  if (id.startsWith(\"archive:\"))\n    return `the id ${id} starts with archive:, which is for the archived copies`;\n  if (seen.has(id)) return `two documents have the id ${id}`;\n  seen.add(id);\n  const kind = doc[\"kind\"];\n  if (typeof kind !== \"string\" || kind.length === 0) return \"kind must be a text that is not empty\";\n  if (kind === \"archived\") return \"the kind archived is for the archived copies\";\n  if (typeof doc[\"title\"] !== \"string\") return \"title must be a text\";\n  if (typeof doc[\"text\"] !== \"string\") return \"text must be a text\";\n  if (doc[\"url\"] !== null && typeof doc[\"url\"] !== \"string\") return \"url must be a text or null\";\n  const meta = doc[\"meta\"];\n  if (meta === null || typeof meta !== \"object\" || Array.isArray(meta))\n    return \"meta must be an object\";\n  try {\n    JSON.stringify(meta);\n  } catch {\n    return \"meta must be JSON: no cycle and no BigInt\";\n  }\n  const merge = doc[\"merge\"];\n  if (merge !== undefined && merge !== \"latest\" && merge !== \"richest\")\n    return 'merge must be \"latest\" or \"richest\"';\n  if (doc[\"full_text\"] !== undefined && typeof doc[\"full_text\"] !== \"string\")\n    return \"full_text must be a text\";\n  const aiInput = doc[\"ai_input\"];\n  if (aiInput !== undefined && !AI_INPUTS.includes(aiInput as AiInput))\n    return 'ai_input must be \"yes\", \"no\" or \"unset\"';\n  return undefined;\n};\n\n/** This function gives the documents of an extractor, or throws an error that names the first fault. */\nconst checkedDocs = (name: string, docs: unknown): ReadonlyArray<ExtractedDoc> => {\n  if (!Array.isArray(docs))\n    throw new TypeError(`the recipe ${name} gave ${typeof docs}, not an array of documents`);\n  const seen = new Set<string>();\n  for (const [index, doc] of docs.entries()) {\n    const fault = faultOf(doc, seen);\n    if (fault !== undefined)\n      throw new TypeError(\n        `the recipe ${name} gave a document that is not valid (${String(index)}): ${fault}`,\n      );\n  }\n  return docs as ReadonlyArray<ExtractedDoc>;\n};\n\n/**\n * This function declares a recipe. It checks the definition at once: a name that is not valid, a\n * reserved name or a duration that it cannot read throws an error. The extractor of the recipe\n * checks each document of `extract`. A document that is not valid makes the extractor throw, and\n * the runtime reports it as an error of the extractor.\n */\nexport const defineRecipe = <Input = string>(\n  definition: RecipeDefinition<Input>,\n): Recipe<Input> => {\n  const name = definition.name;\n  if (typeof name !== \"string\" || name.length > RECIPE_LIMITS.nameChars || !NAME.test(name)) {\n    throw new RangeError(\n      `a recipe name is 1 to ${String(RECIPE_LIMITS.nameChars)} letters, digits and . _ : -, and it starts with a letter or a digit, not ${JSON.stringify(name) ?? \"undefined\"}`,\n    );\n  }\n  if (RESERVED_RECIPES.includes(name)) {\n    throw new RangeError(\n      `the recipe name ${name} belongs to the crawler: give the recipe a name of its own`,\n    );\n  }\n  if (typeof definition.extract !== \"function\") {\n    throw new TypeError(\n      `the recipe ${name} needs extract, a function that reads a body into documents`,\n    );\n  }\n  const mode = definition.mode ?? \"page\";\n  if (mode !== \"page\" && mode !== \"api\") {\n    throw new RangeError(\n      `the mode of the recipe ${name} must be \"page\" or \"api\", not ${String(mode)}`,\n    );\n  }\n  const accept = definition.accept ?? (mode === \"api\" ? API_ACCEPT : PAGE_ACCEPT);\n  const minIntervalMs =\n    definition.minInterval === undefined ? undefined : durationMs(definition.minInterval);\n  const maxAgeOf = (value: Duration): number | \"auto\" =>\n    value === \"auto\" ? \"auto\" : durationMs(value);\n  const maxAge = maxAgeOf(definition.maxAge ?? \"24h\");\n  const maxBytes = definition.maxBytes;\n  if (maxBytes !== undefined && (!Number.isSafeInteger(maxBytes) || maxBytes < 1)) {\n    throw new RangeError(`maxBytes of the recipe ${name} must be a whole number of 1 or more`);\n  }\n  const extract = definition.extract;\n  const extractor: Extractor = async (body, capture) =>\n    checkedDocs(name, await extract(body, { url: capture.url, contentType: capture.contentType }));\n  const fetch = definition.fetch;\n  return {\n    name,\n    description: definition.description ?? null,\n    mode,\n    accept,\n    extractor,\n    request(input, options = {}) {\n      const asked: unknown = fetch === undefined ? input : fetch(input);\n      const target = typeof asked === \"string\" ? { url: asked } : (asked as RecipeRequest | null);\n      const url = target?.url;\n      let parsed: URL | undefined;\n      try {\n        parsed = typeof url === \"string\" ? new URL(url) : undefined;\n      } catch {\n        parsed = undefined;\n      }\n      if (parsed === undefined || (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\")) {\n        throw new TypeError(\n          `the recipe ${name} needs an http or https URL for its input, not ${JSON.stringify(url) ?? String(url)}`,\n        );\n      }\n      return {\n        url: parsed.href,\n        mode,\n        recipe: name,\n        accept: target?.accept ?? accept,\n        headers: target?.headers,\n        secretParams: target?.secretParams,\n        minIntervalMs,\n        maxAgeMs: options.maxAge === undefined ? maxAge : maxAgeOf(options.maxAge),\n        maxBytes,\n        onBehalfOf: options.onBehalfOf,\n        job: options.job,\n        signal: options.signal,\n        scope: options.scope,\n        labels: options.labels,\n        deadline: options.deadline,\n      };\n    },\n  };\n};\n\n/**\n * This function gives the extractor of a recipe to the extractors of a runtime. A second\n * registration of the same recipe does nothing. A different extractor with the same name throws\n * an error, because `reindex` must read each capture with one extractor.\n *\n * @beta\n */\nexport const registerRecipe = <Input>(extractors: Extractors, recipe: Recipe<Input>): void => {\n  const known = extractors.get(recipe.name);\n  if (known === recipe.extractor) return;\n  if (known !== undefined) {\n    throw new Error(\n      `the recipe ${recipe.name} has a different extractor in this runtime: give each recipe a name of its own`,\n    );\n  }\n  extractors.register(recipe.name, recipe.extractor);\n};\n\n/** These options control one run of a recipe. */\nexport interface RecipeRunOptions extends Omit<RecipeRequestOptions, \"deadline\"> {\n  /** No request starts after this time from the start of the run, for example `\"30s\"`. */\n  readonly budget?: Duration | undefined;\n  /** This value is true when a person gave this input directly. It is the only way to see a held-back document. The crawler logs it. */\n  readonly userProvided?: boolean | undefined;\n  /**\n   * The value `archive` gives a copy from a public web archive when the site refuses the page or is\n   * down. Only a recipe of the mode `page` takes it. The page reader reads the copy.\n   */\n  readonly fallback?: \"archive\" | \"none\" | undefined;\n  /** The value true also gives an archived copy of a page that the current robots.txt disallows. */\n  readonly archiveDespiteRobots?: boolean | undefined;\n}\n\n/** One document of a run of a recipe: the version of the capture of the run. */\nexport interface RecipeDoc {\n  readonly doc: string;\n  readonly kind: string;\n  /** This value is the title. A held-back document shows no title. */\n  readonly title: string;\n  readonly url: string | null;\n  /** This value is the metadata. A held-back document shows only the structural fields. */\n  readonly meta: Readonly<Record<string, unknown>>;\n  /** This value is the length of the text. `read` gives the text. */\n  readonly chars: number;\n  /** This id is the capture of this version. A citation names it. */\n  readonly capture: string;\n  readonly fetched_at: string;\n  readonly held_back: boolean;\n  /** This value tells that the document is an archived copy, with the archive and the date of the copy. */\n  readonly archived?: Archived | undefined;\n}\n\n/** The result of `runRecipe`. */\nexport interface RecipeResult {\n  readonly recipe: string;\n  /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */\n  readonly url: string;\n  readonly outcome: string;\n  readonly status: number;\n  /** These documents came from the capture of the run, in the order of the extractor. */\n  readonly docs: readonly RecipeDoc[];\n  /** This id is the capture of the documents. */\n  readonly capture: string | null;\n  readonly fetched_at: string | null;\n  /** This value is true when the stored copy answered, and the crawler sent no request. */\n  readonly from_cache: boolean;\n  /** This value is true when a document is held back. */\n  readonly held_back: boolean;\n  /** This text is the error of the extractor, when it threw for the capture. */\n  readonly error: string | null;\n  /** This value tells that the documents are an archived copy, not a fresh fetch. */\n  readonly archived?: Archived | undefined;\n  readonly note: string | null;\n}\n\n/** A recipe operates on a runtime, or on a crawler that has one (`crawler.runtime`). */\nexport type RecipeTarget = CrawlRuntime | { readonly runtime: CrawlRuntime };\n\n/** This function gives the runtime of a target. */\nexport const runtimeOf = (target: RecipeTarget): CrawlRuntime =>\n  \"crawler\" in target ? target : target.runtime;\n\n/** This function gives the deadline of a budget, on the clock of the runtime. */\nexport const deadlineOf = (\n  runtime: CrawlRuntime,\n  budget: Duration | undefined,\n): number | undefined =>\n  budget === undefined ? undefined : Date.parse(runtime.now()) + durationMs(budget);\n\n/** This function gives one document of a run, with the rule for a held-back document. */\nconst recipeDoc = async (\n  runtime: CrawlRuntime,\n  doc: Doc,\n  capture: string | undefined,\n  userProvided: boolean,\n  action: string,\n): Promise<RecipeDoc> => {\n  const current = runtime.index.get(doc.doc) ?? doc;\n  const version: VersionText | Doc =\n    (capture === undefined ? undefined : runtime.index.versionText(doc.doc, capture)) ?? current;\n  const aiInput: AiInput =\n    current.ai_input === \"no\" || version.ai_input === \"no\" ? \"no\" : version.ai_input;\n  const held = isHeldBack({ ai_input: aiInput }, userProvided);\n  if (aiInput === \"no\" && userProvided) await recordOverride(runtime, current, action);\n  const archived = archivedOf(version.meta);\n  return {\n    doc: doc.doc,\n    kind: version.kind,\n    title: held ? \"\" : version.title,\n    url: version.url,\n    meta: held ? heldBackMeta(version.meta) : version.meta,\n    chars: version.text.length,\n    capture: version.capture,\n    fetched_at: version.fetched_at,\n    held_back: held,\n    ...(archived === undefined ? {} : { archived }),\n  };\n};\n\nconst noteOf = (parts: ReadonlyArray<string | null | undefined>): string | null => {\n  const kept = parts.filter((part): part is string => typeof part === \"string\" && part.length > 0);\n  return kept.length > 0 ? kept.join(\"; \") : null;\n};\n\n/**\n * This function starts a recipe for one input. It fetches the request through the crawler, and it\n * reads the capture with the extractor of the recipe. It gives the documents of that capture.\n * These are its rules:\n *\n * - The runtime registers the recipe the first time that it gets it. Give the recipe to\n *   `openCrawler` too, so that `reindex` reads its captures before the first fetch.\n * - A stored copy younger than `maxAge` answers with no request. When the extractor cannot read a\n *   stored copy, the crawler asks the source one more time.\n * - An extractor that throws gives no document, and `error` tells why. The fetch still succeeds.\n * - A document that asked not to be AI input shows no title and only its structural metadata,\n *   unless `userProvided`. The crawler logs each `userProvided` read of a held-back document.\n * - With `fallback: \"archive\"`, a page that the site refuses gets an archived copy, with\n *   `archived`. The page reader reads the copy. The extractor of the recipe reads only live\n *   captures. Thus its documents do not mix a live answer and an old copy of a third party.\n */\nexport const runRecipe = async <Input>(\n  target: RecipeTarget,\n  recipe: Recipe<Input>,\n  input: Input,\n  options: RecipeRunOptions = {},\n): Promise<RecipeResult> => {\n  const runtime = runtimeOf(target);\n  if (options.fallback === \"archive\" && recipe.mode !== \"page\") {\n    throw new RangeError(\n      `the recipe ${recipe.name} reads an API: an archived copy is for a page, so give fallback only to a recipe of the mode page`,\n    );\n  }\n  registerRecipe(runtime.extractors, recipe);\n  const deadline = deadlineOf(runtime, options.budget);\n  const request = recipe.request(input, { ...options, deadline });\n  let indexed: Indexed = await fetchAndIndex(runtime, request);\n  // a stored answer that the extractor cannot read is no answer: it would give the same error until maxAge\n  if (indexed.error !== undefined && indexed.blocked === undefined && indexed.fetched.fromCache) {\n    indexed = await fetchAndIndex(runtime, { ...request, maxAgeMs: 0 });\n  }\n  const userProvided = options.userProvided ?? false;\n  const action = options.job ?? `recipe ${recipe.name}`;\n  const fetched = indexed.fetched;\n  const notes: Array<string | undefined> = [fetched.note, indexed.error];\n  if (indexed.docs.length === 0 && options.fallback === \"archive\") {\n    const reason = fallbackReason(indexed);\n    if (reason !== undefined) {\n      const page: PageOptions = {\n        render: \"never\",\n        maxAgeMs: request.maxAgeMs,\n        onBehalfOf: request.onBehalfOf,\n        job: request.job,\n        signal: request.signal,\n        scope: request.scope,\n        labels: request.labels,\n        deadline,\n        fallback: \"archive\",\n        archiveDespiteRobots: options.archiveDespiteRobots,\n      };\n      const answer = await archivedCopy(runtime, request.url, reason, page);\n      if (answer.doc !== undefined && answer.archived !== undefined) {\n        const doc = await recipeDoc(runtime, answer.doc, answer.doc.capture, userProvided, action);\n        return {\n          recipe: recipe.name,\n          url: fetched.url,\n          outcome: fetched.outcome,\n          status: fetched.status,\n          docs: [doc],\n          capture: doc.capture,\n          fetched_at: doc.fetched_at,\n          from_cache: answer.fromCache,\n          held_back: doc.held_back,\n          error: indexed.error ?? null,\n          archived: answer.archived,\n          note: noteOf([\n            ...notes,\n            ...answer.notes,\n            \"the page reader read the archived copy, not the extractor of the recipe\",\n            doc.held_back ? HELD_BACK : undefined,\n          ]),\n        };\n      }\n      notes.push(...answer.notes);\n    }\n  }\n  const capture = fetched.capture;\n  const docs: RecipeDoc[] = [];\n  for (const doc of indexed.docs)\n    docs.push(await recipeDoc(runtime, doc, capture, userProvided, action));\n  const held = docs.some((doc) => doc.held_back);\n  return {\n    recipe: recipe.name,\n    url: fetched.url,\n    outcome: fetched.outcome,\n    status: fetched.status,\n    docs,\n    capture: docs[0]?.capture ?? capture ?? null,\n    fetched_at: docs[0]?.fetched_at ?? fetched.fetchedAt ?? null,\n    from_cache: fetched.fromCache,\n    held_back: held,\n    error: indexed.error ?? null,\n    note: noteOf([...notes, held ? HELD_BACK : undefined]),\n  };\n};\n"],"mappings":";;;;;;;;;AAwHA,MAAa,YAAY,MAAkB,YAAoC;CAC7E,MAAM,OAAO,WAAW,MAAM,QAAQ,WAAW;CACjD,IAAI;EACF,OAAO,KAAK,MAAM,IAAI;CACxB,QAAQ;EACN,MAAM,IAAI,MACR,GAAG,QAAQ,IAAI,YAAY,QAAQ,eAAe,2BAA2B,WAC/E;CACF;AACF;;;;;AAMA,MAAa,mBAAsC;CACjD;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;AAOA,MAAa,gBAKT;CAAE,WAAW;CAAI,YAAY;AAAK;AAEtC,MAAM,OAAO;AACb,MAAM,cAAc;AACpB,MAAM,aAAa;AACnB,MAAM,YAAgC;CAAC;CAAO;CAAM;AAAO;;AAG3D,MAAM,WAAW,OAAgB,SAA0C;CACzE,IAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GACpE,OAAO;CACT,MAAM,MAAM;CACZ,MAAM,KAAK,IAAI;CACf,IAAI,OAAO,OAAO,YAAY,GAAG,WAAW,KAAK,GAAG,SAAS,cAAc,YACzE,OAAO,8BAA8B,OAAO,cAAc,UAAU,EAAE;CAExE,IAAI,GAAG,WAAW,UAAU,GAC1B,OAAO,UAAU,GAAG;CACtB,IAAI,KAAK,IAAI,EAAE,GAAG,OAAO,6BAA6B;CACtD,KAAK,IAAI,EAAE;CACX,MAAM,OAAO,IAAI;CACjB,IAAI,OAAO,SAAS,YAAY,KAAK,WAAW,GAAG,OAAO;CAC1D,IAAI,SAAS,YAAY,OAAO;CAChC,IAAI,OAAO,IAAI,aAAa,UAAU,OAAO;CAC7C,IAAI,OAAO,IAAI,YAAY,UAAU,OAAO;CAC5C,IAAI,IAAI,WAAW,QAAQ,OAAO,IAAI,WAAW,UAAU,OAAO;CAClE,MAAM,OAAO,IAAI;CACjB,IAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GACjE,OAAO;CACT,IAAI;EACF,KAAK,UAAU,IAAI;CACrB,QAAQ;EACN,OAAO;CACT;CACA,MAAM,QAAQ,IAAI;CAClB,IAAI,UAAU,KAAA,KAAa,UAAU,YAAY,UAAU,WACzD,OAAO;CACT,IAAI,IAAI,iBAAiB,KAAA,KAAa,OAAO,IAAI,iBAAiB,UAChE,OAAO;CACT,MAAM,UAAU,IAAI;CACpB,IAAI,YAAY,KAAA,KAAa,CAAC,UAAU,SAAS,OAAkB,GACjE,OAAO;AAEX;;AAGA,MAAM,eAAe,MAAc,SAA+C;CAChF,IAAI,CAAC,MAAM,QAAQ,IAAI,GACrB,MAAM,IAAI,UAAU,cAAc,KAAK,QAAQ,OAAO,KAAK,4BAA4B;CACzF,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,CAAC,OAAO,QAAQ,KAAK,QAAQ,GAAG;EACzC,MAAM,QAAQ,QAAQ,KAAK,IAAI;EAC/B,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,UACR,cAAc,KAAK,sCAAsC,OAAO,KAAK,EAAE,KAAK,OAC9E;CACJ;CACA,OAAO;AACT;;;;;;;AAQA,MAAa,gBACX,eACkB;CAClB,MAAM,OAAO,WAAW;CACxB,IAAI,OAAO,SAAS,YAAY,KAAK,SAAS,cAAc,aAAa,CAAC,KAAK,KAAK,IAAI,GACtF,MAAM,IAAI,WACR,yBAAyB,OAAO,cAAc,SAAS,EAAE,4EAA4E,KAAK,UAAU,IAAI,KAAK,aAC/J;CAEF,IAAI,iBAAiB,SAAS,IAAI,GAChC,MAAM,IAAI,WACR,mBAAmB,KAAK,2DAC1B;CAEF,IAAI,OAAO,WAAW,YAAY,YAChC,MAAM,IAAI,UACR,cAAc,KAAK,4DACrB;CAEF,MAAM,OAAO,WAAW,QAAQ;CAChC,IAAI,SAAS,UAAU,SAAS,OAC9B,MAAM,IAAI,WACR,0BAA0B,KAAK,gCAAgC,OAAO,IAAI,GAC5E;CAEF,MAAM,SAAS,WAAW,WAAW,SAAS,QAAQ,aAAa;CACnE,MAAM,gBACJ,WAAW,gBAAgB,KAAA,IAAY,KAAA,IAAY,WAAW,WAAW,WAAW;CACtF,MAAM,YAAY,UAChB,UAAU,SAAS,SAAS,WAAW,KAAK;CAC9C,MAAM,SAAS,SAAS,WAAW,UAAU,KAAK;CAClD,MAAM,WAAW,WAAW;CAC5B,IAAI,aAAa,KAAA,MAAc,CAAC,OAAO,cAAc,QAAQ,KAAK,WAAW,IAC3E,MAAM,IAAI,WAAW,0BAA0B,KAAK,qCAAqC;CAE3F,MAAM,UAAU,WAAW;CAC3B,MAAM,YAAuB,OAAO,MAAM,YACxC,YAAY,MAAM,MAAM,QAAQ,MAAM;EAAE,KAAK,QAAQ;EAAK,aAAa,QAAQ;CAAY,CAAC,CAAC;CAC/F,MAAM,QAAQ,WAAW;CACzB,OAAO;EACL;EACA,aAAa,WAAW,eAAe;EACvC;EACA;EACA;EACA,QAAQ,OAAO,UAAU,CAAC,GAAG;GAC3B,MAAM,QAAiB,UAAU,KAAA,IAAY,QAAQ,MAAM,KAAK;GAChE,MAAM,SAAS,OAAO,UAAU,WAAW,EAAE,KAAK,MAAM,IAAK;GAC7D,MAAM,MAAM,QAAQ;GACpB,IAAI;GACJ,IAAI;IACF,SAAS,OAAO,QAAQ,WAAW,IAAI,IAAI,GAAG,IAAI,KAAA;GACpD,QAAQ;IACN,SAAS,KAAA;GACX;GACA,IAAI,WAAW,KAAA,KAAc,OAAO,aAAa,WAAW,OAAO,aAAa,UAC9E,MAAM,IAAI,UACR,cAAc,KAAK,iDAAiD,KAAK,UAAU,GAAG,KAAK,OAAO,GAAG,GACvG;GAEF,OAAO;IACL,KAAK,OAAO;IACZ;IACA,QAAQ;IACR,QAAQ,QAAQ,UAAU;IAC1B,SAAS,QAAQ;IACjB,cAAc,QAAQ;IACtB;IACA,UAAU,QAAQ,WAAW,KAAA,IAAY,SAAS,SAAS,QAAQ,MAAM;IACzE;IACA,YAAY,QAAQ;IACpB,KAAK,QAAQ;IACb,QAAQ,QAAQ;IAChB,OAAO,QAAQ;IACf,QAAQ,QAAQ;IAChB,UAAU,QAAQ;GACpB;EACF;CACF;AACF;;;;;;;;AASA,MAAa,kBAAyB,YAAwB,WAAgC;CAC5F,MAAM,QAAQ,WAAW,IAAI,OAAO,IAAI;CACxC,IAAI,UAAU,OAAO,WAAW;CAChC,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,cAAc,OAAO,KAAK,+EAC5B;CAEF,WAAW,SAAS,OAAO,MAAM,OAAO,SAAS;AACnD;;AA+DA,MAAa,aAAa,WACxB,aAAa,SAAS,SAAS,OAAO;;AAGxC,MAAa,cACX,SACA,WAEA,WAAW,KAAA,IAAY,KAAA,IAAY,KAAK,MAAM,QAAQ,IAAI,CAAC,IAAI,WAAW,MAAM;;AAGlF,MAAM,YAAY,OAChB,SACA,KACA,SACA,cACA,WACuB;CACvB,MAAM,UAAU,QAAQ,MAAM,IAAI,IAAI,GAAG,KAAK;CAC9C,MAAM,WACH,YAAY,KAAA,IAAY,KAAA,IAAY,QAAQ,MAAM,YAAY,IAAI,KAAK,OAAO,MAAM;CACvF,MAAM,UACJ,QAAQ,aAAa,QAAQ,QAAQ,aAAa,OAAO,OAAO,QAAQ;CAC1E,MAAM,OAAO,WAAW,EAAE,UAAU,QAAQ,GAAG,YAAY;CAC3D,IAAI,YAAY,QAAQ,cAAc,MAAM,eAAe,SAAS,SAAS,MAAM;CACnF,MAAM,WAAW,WAAW,QAAQ,IAAI;CACxC,OAAO;EACL,KAAK,IAAI;EACT,MAAM,QAAQ;EACd,OAAO,OAAO,KAAK,QAAQ;EAC3B,KAAK,QAAQ;EACb,MAAM,OAAO,aAAa,QAAQ,IAAI,IAAI,QAAQ;EAClD,OAAO,QAAQ,KAAK;EACpB,SAAS,QAAQ;EACjB,YAAY,QAAQ;EACpB,WAAW;EACX,GAAI,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS;CAC/C;AACF;AAEA,MAAM,UAAU,UAAmE;CACjF,MAAM,OAAO,MAAM,QAAQ,SAAyB,OAAO,SAAS,YAAY,KAAK,SAAS,CAAC;CAC/F,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI;AAC7C;;;;;;;;;;;;;;;;;AAkBA,MAAa,YAAY,OACvB,QACA,QACA,OACA,UAA4B,CAAC,MACH;CAC1B,MAAM,UAAU,UAAU,MAAM;CAChC,IAAI,QAAQ,aAAa,aAAa,OAAO,SAAS,QACpD,MAAM,IAAI,WACR,cAAc,OAAO,KAAK,kGAC5B;CAEF,eAAe,QAAQ,YAAY,MAAM;CACzC,MAAM,WAAW,WAAW,SAAS,QAAQ,MAAM;CACnD,MAAM,UAAU,OAAO,QAAQ,OAAO;EAAE,GAAG;EAAS;CAAS,CAAC;CAC9D,IAAI,UAAmB,MAAM,cAAc,SAAS,OAAO;CAE3D,IAAI,QAAQ,UAAU,KAAA,KAAa,QAAQ,YAAY,KAAA,KAAa,QAAQ,QAAQ,WAClF,UAAU,MAAM,cAAc,SAAS;EAAE,GAAG;EAAS,UAAU;CAAE,CAAC;CAEpE,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,SAAS,QAAQ,OAAO,UAAU,OAAO;CAC/C,MAAM,UAAU,QAAQ;CACxB,MAAM,QAAmC,CAAC,QAAQ,MAAM,QAAQ,KAAK;CACrE,IAAI,QAAQ,KAAK,WAAW,KAAK,QAAQ,aAAa,WAAW;EAC/D,MAAM,SAAS,eAAe,OAAO;EACrC,IAAI,WAAW,KAAA,GAAW;GACxB,MAAM,OAAoB;IACxB,QAAQ;IACR,UAAU,QAAQ;IAClB,YAAY,QAAQ;IACpB,KAAK,QAAQ;IACb,QAAQ,QAAQ;IAChB,OAAO,QAAQ;IACf,QAAQ,QAAQ;IAChB;IACA,UAAU;IACV,sBAAsB,QAAQ;GAChC;GACA,MAAM,SAAS,MAAM,aAAa,SAAS,QAAQ,KAAK,QAAQ,IAAI;GACpE,IAAI,OAAO,QAAQ,KAAA,KAAa,OAAO,aAAa,KAAA,GAAW;IAC7D,MAAM,MAAM,MAAM,UAAU,SAAS,OAAO,KAAK,OAAO,IAAI,SAAS,cAAc,MAAM;IACzF,OAAO;KACL,QAAQ,OAAO;KACf,KAAK,QAAQ;KACb,SAAS,QAAQ;KACjB,QAAQ,QAAQ;KAChB,MAAM,CAAC,GAAG;KACV,SAAS,IAAI;KACb,YAAY,IAAI;KAChB,YAAY,OAAO;KACnB,WAAW,IAAI;KACf,OAAO,QAAQ,SAAS;KACxB,UAAU,OAAO;KACjB,MAAM,OAAO;MACX,GAAG;MACH,GAAG,OAAO;MACV;MACA,IAAI,YAAY,YAAY,KAAA;KAC9B,CAAC;IACH;GACF;GACA,MAAM,KAAK,GAAG,OAAO,KAAK;EAC5B;CACF;CACA,MAAM,UAAU,QAAQ;CACxB,MAAM,OAAoB,CAAC;CAC3B,KAAK,MAAM,OAAO,QAAQ,MACxB,KAAK,KAAK,MAAM,UAAU,SAAS,KAAK,SAAS,cAAc,MAAM,CAAC;CACxE,MAAM,OAAO,KAAK,MAAM,QAAQ,IAAI,SAAS;CAC7C,OAAO;EACL,QAAQ,OAAO;EACf,KAAK,QAAQ;EACb,SAAS,QAAQ;EACjB,QAAQ,QAAQ;EAChB;EACA,SAAS,KAAK,EAAE,EAAE,WAAW,WAAW;EACxC,YAAY,KAAK,EAAE,EAAE,cAAc,QAAQ,aAAa;EACxD,YAAY,QAAQ;EACpB,WAAW;EACX,OAAO,QAAQ,SAAS;EACxB,MAAM,OAAO,CAAC,GAAG,OAAO,OAAO,YAAY,KAAA,CAAS,CAAC;CACvD;AACF"}

package/build/{recipe-product-Dellqudu.mjs → recipe-product-CyZE3Ukd.mjs}

@@ -1,10 +1,14 @@
-import { i as durationMs } from "./version-BPem-XZd.mjs";
-import { S as archivedOf, a as fetchPage, c as isHeldBack, n as HELD_BACK, p as recordOverride } from "./runtime-D_r2SYge.mjs";
+import { i as durationMs } from "./version-Djva7JkN.mjs";
+import { a as fetchPage, c as isHeldBack, d as recordOverride, n as HELD_BACK, y as archivedOf } from "./runtime-C_2BQibH.mjs";
 import { o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
-import { c as runtimeOf, r as deadlineOf } from "./recipe-GiFQojmo.mjs";
+import { c as runtimeOf, r as deadlineOf } from "./recipe-B2SkepDZ.mjs";
 import { DAYS, describeWeek, disagreements, fromCompact, fromSchemaOrg, fromText } from "./hours.mjs";
 //#region src/recipe-venue.ts
-/** These are the limits of the venue recipe. */
+/**
+* These are the limits of the venue recipe.
+*
+* @beta
+*/
 const VENUE_LIMITS = {
 	pages: 3,
 	maxPages: 8,
@@ -27,6 +31,8 @@ const wordsOf = (text) => text.toLowerCase().normalize("NFKD").replace(/\p{M}/gu
 * This function tells if a page names a venue: the page has the first word of the name and 60
 * percent of its words. A domain that a venue let go can show a different business. It gives
 * undefined for a name with no word.
+*
+* @beta
 */
 const namesVenue = (name, page) => {
 	const words = wordsOf(name);
@@ -60,6 +66,8 @@ const withoutHash = (url) => url.replace(/#.*$/s, "").replace(/\/$/, "");
 * This function ranks the links of a page by the words of their text and path. The links to
 * the hours or a menu come first. It keeps only the links on the site of `home`,
 * and it leaves out the files (a PDF, an image) and the home page itself.
+*
+* @beta
 */
 const rankHoursLinks = (links, home) => {
 	const site = hostOf(home);
@@ -164,6 +172,8 @@ const addressText = (value) => {
 /**
 * This function tells if a JSON-LD location record is the venue. The record must have the same
 * number on a street that shares a word, or a position near the venue (`VENUE_LIMITS.nearMetres`).
+*
+* @beta
 */
 const locationMatches = (record, location) => {
 	const geo = record["geo"];
@@ -450,7 +460,11 @@ const readVenue = async (target, url, options = {}) => {
 };
 //#endregion
 //#region src/recipe-product.ts
-/** These are the limits of the product reader. A page with more is hostile or broken, and the reader stops there. */
+/**
+* These are the limits of the product reader. A page with more is hostile or broken, and the reader stops there.
+*
+* @beta
+*/
 const PRODUCT_LIMITS = {
 	records: 500,
 	offers: 200,
@@ -502,6 +516,8 @@ const fractionDigits = (currency) => {
 * separator, each followed by three digits, separates thousands: `1.299` is 1299. For a
 * currency with three decimal digits, one point with three digits is the decimal point. A text
 * with two numbers (a range), a negative amount or zero gives undefined.
+*
+* @beta
 */
 const readAmount = (value, currency) => {
 	if (typeof value === "number") return Number.isFinite(value) && value > 0 ? value : void 0;
@@ -619,6 +635,8 @@ const sizeOf = (record) => {
 * 3. Else the largest product.
 *
 * It also reads the `mainEntity` of a record (a `WebPage` or an `ItemPage` that names its product).
+*
+* @beta
 */
 const pickProduct = (records, page) => {
 	let steps = 0;
@@ -654,7 +672,11 @@ const pickProduct = (records, page) => {
 		steps
 	};
 };
-/** This function gives the variant that a URL names: the `variant` parameter of Shopify, or a `sku` or `gtin` parameter. */
+/**
+* This function gives the variant that a URL names: the `variant` parameter of Shopify, or a `sku` or `gtin` parameter.
+*
+* @beta
+*/
 const pickVariant = (product, page) => {
 	let params;
 	try {
@@ -889,4 +911,4 @@ const readProduct = async (target, url, options = {}) => {
 //#endregion
 export { readAmount as a, locationMatches as c, readVenue as d, venueReadingsOf as f, productOf as i, namesVenue as l, pickProduct as n, readProduct as o, pickVariant as r, VENUE_LIMITS as s, PRODUCT_LIMITS as t, rankHoursLinks as u };
 
-//# sourceMappingURL=recipe-product-Dellqudu.mjs.map
+//# sourceMappingURL=recipe-product-CyZE3Ukd.mjs.map