polite-crawler 0.3.1 → 0.4.0

package/build/index.d.mts

@@ -3615,7 +3615,7 @@ export declare const runWatch: (runtime: CrawlRuntime, source: WatchStore | (()
 //#endregion
 //#region src/version.d.ts
 /** This is the version of the package. A test makes sure that it agrees with `package.json`. */
-export declare const VERSION = "0.3.1";
+export declare const VERSION = "0.4.0";
 /** This is the product token that robots.txt rules match when an application gives no token. */
 export declare const DEFAULT_TOKEN = "PoliteCrawler";
 /** This page tells a site owner what the crawler is and how to control it. */

package/build/index.mjs

@@ -1,21 +1,21 @@
 import { n as FileBlobStore, t as CaptureLog } from "./log-CmBWcyQm.mjs";
 import { n as RenderRefused, r as playwrightRenderer, t as NO_PLAYWRIGHT } from "./render-CKE2LILH.mjs";
 import { t as browserMissing } from "./browser-56dBBLec.mjs";
-import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-BVrf4OJP.mjs";
-import { A as DEADLINE_NOTE, F as EgressRefused, I as egressLookup, M as makeRecord, O as CAPTURE_SCHEMA, S as readPdf, T as locate, _ as archivedDocId, a as fetchPage, c as isHeldBack, d as recordOverride, f as reindex, h as archiveExtractor, i as fetchAndIndex, j as DEFAULT_SECRET_PARAMS, k as Crawler, l as isoTime, m as versionAtTime, n as HELD_BACK, o as findDocuments, p as verifyQuote, r as catchUpIndex, s as heldBackMeta, t as Extractors, u as readDocument, x as isPdf } from "./runtime-p5WN2UAA.mjs";
+import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-B9CUZfx5.mjs";
+import { A as DEADLINE_NOTE, F as EgressRefused, I as egressLookup, M as makeRecord, O as CAPTURE_SCHEMA, S as readPdf, T as locate, _ as archivedDocId, a as fetchPage, c as isHeldBack, d as recordOverride, f as reindex, h as archiveExtractor, i as fetchAndIndex, j as DEFAULT_SECRET_PARAMS, k as Crawler, l as isoTime, m as versionAtTime, n as HELD_BACK, o as findDocuments, p as verifyQuote, r as catchUpIndex, s as heldBackMeta, t as Extractors, u as readDocument, x as isPdf } from "./runtime-C2VRY89U.mjs";
 import { DEFAULT_POLICY, MemoryHostStateStore, VALIDATOR_MEMO, nextValidatorMemo, realClock } from "./politeness.mjs";
 import { ALLOW_ALL, decide, parseRobots } from "./robots.mjs";
 import { i as keyDirectory, n as KEY_DIRECTORY_TYPE, o as verifySignature, r as createWebBotAuthSigner, t as KEY_DIRECTORY_PATH } from "./webbotauth-Dq_dA_u3.mjs";
-import { A as NO_EMBEDDER, C as DISCOVER_LIMITS, D as readSitemap, E as feedDate, O as DocIndex, S as crawlSite, T as SITEMAP_LIMITS, _ as estimateTokens, b as SqliteHostStateStore, c as WatchStore, d as VectorStore, f as createHybridSearch, g as chunkTokensOf, h as chunkText, i as openCrawler, k as FAST_RANK_SHARE, l as runWatch, m as HYBRID_DEFAULTS, n as defaultUserAgent, o as WATCH_LIMITS, p as CHUNK_LIMITS, r as isMailbox, s as WATCH_SCHEMA, t as createRuntime, u as VECTORS_SCHEMA, v as fuseRanks, w as discoverSite, x as CRAWL_LIMITS, y as unitVector } from "./open-CDFgV-FC.mjs";
+import { A as NO_EMBEDDER, C as DISCOVER_LIMITS, D as readSitemap, E as feedDate, O as DocIndex, S as crawlSite, T as SITEMAP_LIMITS, _ as estimateTokens, b as SqliteHostStateStore, c as WatchStore, d as VectorStore, f as createHybridSearch, g as chunkTokensOf, h as chunkText, i as openCrawler, k as FAST_RANK_SHARE, l as runWatch, m as HYBRID_DEFAULTS, n as defaultUserAgent, o as WATCH_LIMITS, p as CHUNK_LIMITS, r as isMailbox, s as WATCH_SCHEMA, t as createRuntime, u as VECTORS_SCHEMA, v as fuseRanks, w as discoverSite, x as CRAWL_LIMITS, y as unitVector } from "./open-CsFRWOjf.mjs";
 import { a as bodyEncoding, o as decodeBody, r as typesOf, t as extractPage } from "./html-Dir5QEi0.mjs";
-import { a as formatDiff, n as diffDocument, r as diffTexts, t as DIFF_LIMITS } from "./diff-DLyQ2mkg.mjs";
-import { n as checkFields, r as extractDocument, t as FIELD_KINDS } from "./extract-mnKsYeaM.mjs";
-import { a as readJson, i as defineRecipe, n as RESERVED_RECIPES, o as registerRecipe, s as runRecipe, t as RECIPE_LIMITS } from "./recipe-BNjpjJ_8.mjs";
-import { c as startDaemon, i as daemonStateFile, n as DAEMON_STATE_SCHEMA, o as readDaemonState, r as changeCommand, s as scheduleWatches, t as DAEMON_DEFAULTS } from "./daemon-1RNbvE5p.mjs";
+import { a as formatDiff, n as diffDocument, r as diffTexts, t as DIFF_LIMITS } from "./diff-DuD9pinS.mjs";
+import { n as checkFields, r as extractDocument, t as FIELD_KINDS } from "./extract-BISJyKWN.mjs";
+import { a as readJson, i as defineRecipe, n as RESERVED_RECIPES, o as registerRecipe, s as runRecipe, t as RECIPE_LIMITS } from "./recipe-30dN3orz.mjs";
+import { c as startDaemon, i as daemonStateFile, n as DAEMON_STATE_SCHEMA, o as readDaemonState, r as changeCommand, s as scheduleWatches, t as DAEMON_DEFAULTS } from "./daemon-BPswUOgu.mjs";
 import { i as ollamaEmbedder, n as OLLAMA_MODELS, r as isLoopback, t as NO_OLLAMA } from "./ollama-D1xRh3hY.mjs";
 import { n as TRANSFORMERS_MODELS, r as transformersEmbedder, t as NO_TRANSFORMERS } from "./transformers-BNiuiLxT.mjs";
 import { DAYS, compareHours, describeWeek, disagreements, fromCompact, fromSchemaOrg, fromText, toHoursWeek } from "./hours.mjs";
-import { a as readAmount, c as locationMatches, d as readVenue, f as venueReadingsOf, i as productOf, l as namesVenue, n as pickProduct, o as readProduct, r as pickVariant, s as VENUE_LIMITS, t as PRODUCT_LIMITS, u as rankHoursLinks } from "./recipe-product-BqDdxZxL.mjs";
-import { t as exportWacz } from "./wacz-BXMxWyfg.mjs";
-import { n as exportWarc } from "./warc-DmzB-FkC.mjs";
+import { a as readAmount, c as locationMatches, d as readVenue, f as venueReadingsOf, i as productOf, l as namesVenue, n as pickProduct, o as readProduct, r as pickVariant, s as VENUE_LIMITS, t as PRODUCT_LIMITS, u as rankHoursLinks } from "./recipe-product-BuJ_gEwn.mjs";
+import { t as exportWacz } from "./wacz-D5bvxsiM.mjs";
+import { n as exportWarc } from "./warc-DsB3oP78.mjs";
 export { ALLOW_ALL, BOT_PAGE, CAPTURE_SCHEMA, CHUNK_LIMITS, CRAWL_LIMITS, CaptureLog, Crawler, DAEMON_DEFAULTS, DAEMON_STATE_SCHEMA, DAYS, DEADLINE_NOTE, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, DEFAULT_TOKEN, DIFF_LIMITS, DISCOVER_LIMITS, DocIndex, EgressRefused, Extractors, FAST_RANK_SHARE, FIELD_KINDS, FileBlobStore, HELD_BACK, HYBRID_DEFAULTS, KEY_DIRECTORY_PATH, KEY_DIRECTORY_TYPE, MemoryHostStateStore, NO_EMBEDDER, NO_OLLAMA, NO_PLAYWRIGHT, NO_TRANSFORMERS, OLLAMA_MODELS, PRODUCT_LIMITS, RECIPE_LIMITS, RESERVED_RECIPES, RenderRefused, SITEMAP_LIMITS, SqliteHostStateStore, TRANSFORMERS_MODELS, VALIDATOR_MEMO, VECTORS_SCHEMA, VENUE_LIMITS, VERSION, VectorStore, WATCH_LIMITS, WATCH_SCHEMA, WatchStore, archiveExtractor, archivedDocId, bodyEncoding, browserMissing, catchUpIndex, changeCommand, checkFields, chunkText, chunkTokensOf, compareHours, crawlSite, createHybridSearch, createRuntime, createWebBotAuthSigner, daemonStateFile, decide, decodeBody, defaultUserAgent, defineRecipe, describeWeek, diffDocument, diffTexts, disagreements, discoverSite, durationMs, egressLookup, estimateTokens, exportWacz, exportWarc, extractDocument, extractPage, feedDate, fetchAndIndex, fetchPage, findDocuments, formatDiff, fromCompact, fromSchemaOrg, fromText, fuseRanks, heldBackMeta, isHeldBack, isLoopback, isMailbox, isPdf, isoTime, keyDirectory, locate, locationMatches, makeRecord, namesVenue, nextValidatorMemo, ollamaEmbedder, openCrawler, parseRobots, pickProduct, pickVariant, playwrightRenderer, productOf, rankHoursLinks, readAmount, readDaemonState, readDocument, readJson, readPdf, readProduct, readSitemap, readVenue, realClock, recordOverride, registerRecipe, reindex, runRecipe, runWatch, scheduleWatches, startDaemon, toHoursWeek, transformersEmbedder, typesOf, unitVector, venueReadingsOf, verifyQuote, verifySignature, versionAtTime };

package/build/{open-CDFgV-FC.mjs → open-CsFRWOjf.mjs}

@@ -2,15 +2,15 @@ import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
 import { n as FileBlobStore, t as CaptureLog } from "./log-CmBWcyQm.mjs";
 import { r as playwrightRenderer } from "./render-CKE2LILH.mjs";
 import { r as checkedRenderer } from "./browser-56dBBLec.mjs";
-import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-BVrf4OJP.mjs";
-import { A as DEADLINE_NOTE, C as fold, D as snippetOf, E as parseQuery, N as sha256$1, P as stableStringify, a as fetchPage, c as isHeldBack, f as reindex, i as fetchAndIndex, k as Crawler, l as isoTime, o as findDocuments, p as verifyQuote, r as catchUpIndex, t as Extractors, u as readDocument, y as archivedOf } from "./runtime-p5WN2UAA.mjs";
+import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-B9CUZfx5.mjs";
+import { A as DEADLINE_NOTE, C as fold, D as snippetOf, E as parseQuery, N as sha256$1, P as stableStringify, a as fetchPage, c as isHeldBack, f as reindex, i as fetchAndIndex, k as Crawler, l as isoTime, o as findDocuments, p as verifyQuote, r as catchUpIndex, t as Extractors, u as readDocument, y as archivedOf } from "./runtime-C2VRY89U.mjs";
 import { MemoryHostStateStore, nextValidatorMemo, realClock } from "./politeness.mjs";
 import { matchesPattern } from "./robots.mjs";
 import { n as canonicalUrl, r as scopeTest, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
 import { o as decodeBody, t as extractPage } from "./html-Dir5QEi0.mjs";
-import { n as diffDocument, r as diffTexts } from "./diff-DLyQ2mkg.mjs";
-import { r as extractDocument } from "./extract-mnKsYeaM.mjs";
-import { o as registerRecipe } from "./recipe-BNjpjJ_8.mjs";
+import { n as diffDocument, r as diffTexts } from "./diff-DuD9pinS.mjs";
+import { r as extractDocument } from "./extract-BISJyKWN.mjs";
+import { o as registerRecipe } from "./recipe-30dN3orz.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { dirname, join } from "node:path";
 import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
@@ -4356,4 +4356,4 @@ const openCrawler = (options) => {
 //#endregion
 export { NO_EMBEDDER as A, DISCOVER_LIMITS as C, readSitemap as D, feedDate as E, DocIndex as O, crawlSite as S, SITEMAP_LIMITS as T, estimateTokens as _, open_exports as a, SqliteHostStateStore as b, WatchStore as c, VectorStore as d, createHybridSearch as f, chunkTokensOf as g, chunkText as h, openCrawler as i, FAST_RANK_SHARE as k, runWatch as l, HYBRID_DEFAULTS as m, defaultUserAgent as n, WATCH_LIMITS as o, CHUNK_LIMITS as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, VECTORS_SCHEMA as u, fuseRanks as v, discoverSite as w, CRAWL_LIMITS as x, unitVector as y };
 
-//# sourceMappingURL=open-CDFgV-FC.mjs.map
+//# sourceMappingURL=open-CsFRWOjf.mjs.map