polite-crawler 0.1.0 → 0.1.1

package/build/{open-CJR4b44v.mjs → open-DYZ7KL2A.mjs}

@@ -1,9 +1,9 @@
 import { a as RenderRefused, o as playwrightRenderer, r as checkedRenderer, s as __exportAll } from "./browser-gJ_h4AhB.mjs";
 import { classify, isClientError, parseHttpDate } from "./classify.mjs";
-import { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, MemoryHostStateStore, checkPolicy, equalJitterMs, policyFor, realClock, restAfter } from "./politeness.mjs";
+import { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, MemoryHostStateStore, checkPolicy, equalJitterMs, nextValidatorMemo, policyFor, realClock, restAfter, validatorsToSend } from "./politeness.mjs";
 import { decide, isProductToken, robotsFromResponse, robotsFromStatus, signalsFor, unreachableRobots } from "./robots.mjs";
 import { NO_SIGNALS, metaSignals, readSignals } from "./signals.mjs";
-import { n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-rVx8_y1_.mjs";
+import { n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-no12qgos.mjs";
 import { c as canonicalUrl, o as decodeBody, r as typesOf, s as cacheKeyOf, t as extractPage } from "./html-oNpGAHwi.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { mkdir, open, readFile, readdir, rename, rm, stat } from "node:fs/promises";
@@ -1069,6 +1069,14 @@ var Crawler = class {
 		work.then(() => this.running.delete(work), () => this.running.delete(work));
 		return work;
 	}
+	/** This method gives the validator memo of a host. The memo is a hint, so an error of its store gives no memo. */
+	memoOf(host) {
+		try {
+			return this.scheduler.store.validators?.(host);
+		} catch {
+			return;
+		}
+	}
 	stopHost(host, until, reason) {
 		this.scheduler.store.stop(host, until, reason);
 		this.emit({
@@ -1636,8 +1644,16 @@ var Crawler = class {
 				let stored;
 				if (usable?.blob && (usable.etag || usable.last_modified)) stored = await this.options.blobs.get(usable.blob);
 				const headers = { ...sent };
-				if (stored !== void 0 && usable?.etag) headers["if-none-match"] = usable.etag;
-				if (stored !== void 0 && usable?.last_modified) headers["if-modified-since"] = usable.last_modified;
+				const send = stored === void 0 || usable === void 0 ? {
+					etag: false,
+					lastModified: false
+				} : validatorsToSend(robotsFile ? void 0 : this.memoOf(host), {
+					etag: Boolean(usable.etag),
+					lastModified: Boolean(usable.last_modified)
+				}, this.clock.now());
+				if (send.etag && usable?.etag) headers["if-none-match"] = usable.etag;
+				if (send.lastModified && usable?.last_modified) headers["if-modified-since"] = usable.last_modified;
+				const validators = ["if-none-match", "if-modified-since"].filter((name) => headers[name] !== void 0);
 				const started = this.clock.now();
 				const timeout = AbortSignal.timeout(policy.requestTimeoutMs);
 				const abort = AbortSignal.any([timeout, signal]);
@@ -1824,6 +1840,7 @@ var Crawler = class {
 					blob: blob ?? null,
 					etag,
 					last_modified: lastModified,
+					...validators.length > 0 ? { validators } : {},
 					...revisit ? { revisit_of: usable.capture } : {},
 					robots: request_.mode === "page" ? "allowed" : "not-checked",
 					...check !== void 0 ? { robots_check: check } : {},
@@ -1866,6 +1883,14 @@ var Crawler = class {
 					fresh_until: freshness,
 					vary_key: varyKey
 				}, next);
+				const answer = validators.length === 0 || capture === void 0 || robotsFile ? void 0 : outcome === "not-modified" ? "not-modified" : outcome === "ok" && blob !== void 0 && !cut ? blob === usable?.blob ? "same" : "changed" : void 0;
+				if (answer !== void 0) try {
+					this.scheduler.store.noteValidators?.(host, {
+						etag: headers["if-none-match"] !== void 0,
+						lastModified: headers["if-modified-since"] !== void 0,
+						answer
+					}, fetchedAt);
+				} catch {}
 				const delayMs = this.settle(host, policy, {
 					outcome,
 					status: response.statusCode,
@@ -2593,6 +2618,7 @@ const parseQuery = (q) => {
 	return {
 		all: parts.map((part) => part.text).join(" AND "),
 		any: any.map((part) => part.text).join(" OR "),
+		anyParts: any.map((part) => part.text),
 		terms: any.map((part) => part.term),
 		parts: parts.length
 	};
@@ -2637,6 +2663,14 @@ const snippetOf = (text, terms, width = 160) => {
 };
 /** The search index reads this many characters of each text, at most. */
 const MAX_INDEX_CHARS = 2e5;
+/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
+const FAST_RANK_SHARE = .2;
+/** This function gives the rank mode of an option. A value that is not a rank mode throws a `RangeError`. */
+const rankModeOf = (value) => {
+	if (value === void 0 || value === "exact") return "exact";
+	if (value === "fast") return "fast";
+	throw new RangeError(`rank must be "exact" or "fast", not ${JSON.stringify(value) ?? "undefined"}`);
+};
 const namesOf = (generation) => ({
 	docs: `docs_${String(generation)}`,
 	fts: `fts_${String(generation)}`,
@@ -3121,9 +3155,10 @@ var DocIndex = class {
 	* This method searches the documents with BM25, the best first. The syntax of the query is the
 	* syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it
 	* adds the documents that match one part. A document that asked not to be AI input is not in
-	* the results, so a search cannot show its text.
+	* the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).
 	*/
 	find(q, options) {
+		const rank = rankModeOf(options.rank);
 		const query = parseQuery(q);
 		if (query === void 0) return [];
 		const limit = Math.max(1, Math.min(1e3, Math.floor(options.limit) || 10));
@@ -3142,7 +3177,7 @@ var DocIndex = class {
 			const rows = run(query.all);
 			if (rows.length < limit && query.parts > 1) {
 				const seen = new Set(rows.map((row) => row.doc));
-				for (const row of run(query.any)) {
+				for (const row of run(rank === "fast" ? this.rareParts(n, query.anyParts) : query.any)) {
 					if (rows.length >= limit) break;
 					if (!seen.has(row.doc)) rows.push(row);
 				}
@@ -3164,18 +3199,36 @@ var DocIndex = class {
 	* This method scores a set of documents against a query, with the best first. It leaves out a
 	* document that matches no term, and a document that asked not to be AI input. It ranks the
 	* results of a source again, because the order of the source is only as good as its search engine.
+	* In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).
 	*/
-	rankWithin(q, docs) {
+	rankWithin(q, docs, options = {}) {
+		const rank = rankModeOf(options.rank);
 		const query = parseQuery(q);
 		if (query === void 0 || docs.length === 0) return [];
 		return this.live((n) => this.stmt(`SELECT d.doc AS doc, bm25(${n.fts}, 3.0, 1.0) AS rank
              FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid
             WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' AND d.doc IN (SELECT value FROM json_each(?))
-            ORDER BY rank`).all(query.any, JSON.stringify(docs)).map((row) => ({
+            ORDER BY rank`).all(rank === "fast" ? this.rareParts(n, query.anyParts) : query.any, JSON.stringify(docs)).map((row) => ({
 			doc: row.doc,
 			score: Math.round(-row.rank * 1e3) / 1e3
 		})));
 	}
+	/**
+	* This method gives the form of a query that needs one part, without the common parts (the mode
+	* `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,
+	* it keeps the rarest part.
+	*/
+	rareParts(n, parts) {
+		if (parts.length < 2) return parts.join(" OR ");
+		const total = this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get().n;
+		const counted = parts.map((part) => ({
+			part,
+			count: this.stmt(`SELECT count(*) AS n FROM ${n.fts} WHERE ${n.fts} MATCH ?`).get(part).n
+		}));
+		const rare = counted.filter((one) => one.count <= FAST_RANK_SHARE * total);
+		if (rare.length > 0) return rare.map((one) => one.part).join(" OR ");
+		return counted.reduce((best, one) => one.count < best.count ? one : best).part;
+	}
 	/** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */
 	latest(url, variant) {
 		const key = keyOf(url, variant);
@@ -3353,6 +3406,16 @@ const NO_END = 9e15;
 const SHARED_WAIT_MS = 1e3;
 /** The store removes the rows of idle hosts after this many reservations. */
 const SWEEP_EVERY = 1e3;
+const verdictOf = (value) => value === "works" || value === "fails" ? value : "unknown";
+const memoOf = (row) => ({
+	skipEtag: row.skip_etag === 1,
+	etag: verdictOf(row.etag),
+	lastModified: verdictOf(row.last_modified),
+	etagMisses: row.etag_misses,
+	dateMisses: row.date_misses,
+	sinceProbe: row.since_probe,
+	changedAt: row.changed_at
+});
 /**
 * This store keeps the state of each host in a SQLite file. Two or more processes on one store
 * share it, so they share the pace and the stops of each host. Thus a host gets one request at a
@@ -3363,8 +3426,11 @@ const SWEEP_EVERY = 1e3;
 * the lease gets a wait of one second or less, and asks again. A process that stops during a
 * request holds its lease until the lease ends (5 minutes by default).
 *
+* The table `validators` keeps the validator memo of each host (`ValidatorMemo`), so the processes
+* share it and it stays after a restart. A file of version 0.1.0 gets the table when it opens.
+*
 * The state is not a projection of the capture log. A deletion of the file loses only the pace,
-* the counts and the stops, and the crawler starts them again.
+* the counts, the stops and the memos, and the crawler starts them again.
 */
 var SqliteHostStateStore = class {
 	path;
@@ -3387,16 +3453,37 @@ var SqliteHostStateStore = class {
 		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
 		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
 		this.db.exec("PRAGMA synchronous = NORMAL;");
-		this.db.exec(`CREATE TABLE IF NOT EXISTS hosts (
-      host TEXT PRIMARY KEY,
-      next_at REAL NOT NULL DEFAULT 0,
-      leased_until REAL NOT NULL DEFAULT 0,
-      owner TEXT,
-      failures INTEGER NOT NULL DEFAULT 0,
-      waits INTEGER NOT NULL DEFAULT 0,
-      stop_until REAL,
-      stop_reason TEXT
-    )`);
+		this.migrate();
+	}
+	/**
+	* This method makes the tables, or adds the tables of a later version to an older file. It is one
+	* transaction, so two processes that open one old file at the same time do not collide.
+	*/
+	migrate() {
+		this.step(() => {
+			this.db.exec(`CREATE TABLE IF NOT EXISTS hosts (
+        host TEXT PRIMARY KEY,
+        next_at REAL NOT NULL DEFAULT 0,
+        leased_until REAL NOT NULL DEFAULT 0,
+        owner TEXT,
+        failures INTEGER NOT NULL DEFAULT 0,
+        waits INTEGER NOT NULL DEFAULT 0,
+        stop_until REAL,
+        stop_reason TEXT
+      )`);
+			this.db.exec(`CREATE TABLE IF NOT EXISTS validators (
+        host TEXT PRIMARY KEY,
+        skip_etag INTEGER NOT NULL DEFAULT 0,
+        etag TEXT NOT NULL DEFAULT 'unknown',
+        last_modified TEXT NOT NULL DEFAULT 'unknown',
+        etag_misses INTEGER NOT NULL DEFAULT 0,
+        date_misses INTEGER NOT NULL DEFAULT 0,
+        since_probe INTEGER NOT NULL DEFAULT 0,
+        changed_at REAL NOT NULL DEFAULT 0
+      )`);
+			const { user_version: version } = this.db.prepare("PRAGMA user_version").get();
+			if (version < 1) this.db.exec(`PRAGMA user_version = ${String(1)}`);
+		});
 	}
 	stmt(sql) {
 		let statement = this.statements.get(sql);
@@ -3493,6 +3580,24 @@ var SqliteHostStateStore = class {
 			else this.stmt("UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL WHERE host = ?").run(host);
 		});
 	}
+	validators(host) {
+		if (this.closed) throw new Error("the host state store is closed");
+		const row = this.stmt(`SELECT skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at
+         FROM validators WHERE host = ?`).get(host);
+		return row === void 0 ? void 0 : memoOf(row);
+	}
+	noteValidators(host, report, now) {
+		return this.step(() => {
+			const memo = nextValidatorMemo(this.validators(host), report, now);
+			this.stmt(`INSERT INTO validators (host, skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(host) DO UPDATE SET skip_etag = excluded.skip_etag, etag = excluded.etag,
+           last_modified = excluded.last_modified, etag_misses = excluded.etag_misses,
+           date_misses = excluded.date_misses, since_probe = excluded.since_probe,
+           changed_at = excluded.changed_at`).run(host, memo.skipEtag ? 1 : 0, memo.etag, memo.lastModified, memo.etagMisses, memo.dateMisses, memo.sinceProbe, memo.changedAt);
+			return memo;
+		});
+	}
 	/** This value is the number of hosts that the store holds. */
 	get size() {
 		return this.stmt("SELECT count(*) AS n FROM hosts").get().n;
@@ -4434,13 +4539,15 @@ const verifyQuote = async (runtime, id, quote, options = {}) => {
 };
 /**
 * This function searches the stored documents. The syntax of the query is the syntax of
-* `parseQuery`. A held-back document is not in the results.
+* `parseQuery`. A held-back document is not in the results. The option `rank` chooses the
+* ranking (`RankMode`), and the default is `exact`.
 */
 const findDocuments = async (runtime, q, options = {}) => {
 	await Promise.resolve();
 	return runtime.index.find(q, {
 		limit: options.limit ?? 10,
-		kind: options.kind ?? null
+		kind: options.kind ?? null,
+		...options.rank !== void 0 ? { rank: options.rank } : {}
 	});
 };
 const RENDERED_MARK = "#rendered";
@@ -4883,6 +4990,6 @@ const openCrawler = (options) => {
 	};
 };
 //#endregion
-export { DEFAULT_SECRET_PARAMS as A, readPdf as C, locate as D, DocIndex as E, FileBlobStore as F, makeRecord as M, EgressRefused as N, CAPTURE_SCHEMA as O, egressLookup as P, isPdf as S, SqliteHostStateStore as T, readDocument as _, openCrawler as a, renderAndIndex as b, HELD_BACK as c, fetchPage as d, findDocuments as f, pageExtractor as g, pageDocId as h, isMailbox as i, renderedKey as j, Crawler as k, catchUpIndex as l, isHeldBack as m, defaultUserAgent as n, open_exports as o, heldBackMeta as p, durationMs as r, Extractors as s, createRuntime as t, fetchAndIndex as u, recordOverride as v, CaptureLog as w, verifyQuote as x, reindex as y };
+export { Crawler as A, readPdf as C, FAST_RANK_SHARE as D, DocIndex as E, egressLookup as F, FileBlobStore as I, renderedKey as M, makeRecord as N, locate as O, EgressRefused as P, isPdf as S, SqliteHostStateStore as T, readDocument as _, openCrawler as a, renderAndIndex as b, HELD_BACK as c, fetchPage as d, findDocuments as f, pageExtractor as g, pageDocId as h, isMailbox as i, DEFAULT_SECRET_PARAMS as j, CAPTURE_SCHEMA as k, catchUpIndex as l, isHeldBack as m, defaultUserAgent as n, open_exports as o, heldBackMeta as p, durationMs as r, Extractors as s, createRuntime as t, fetchAndIndex as u, recordOverride as v, CaptureLog as w, verifyQuote as x, reindex as y };
 
-//# sourceMappingURL=open-CJR4b44v.mjs.map
+//# sourceMappingURL=open-DYZ7KL2A.mjs.map