polite-crawler 0.1.0 → 0.1.2

package/build/html.d.mts

@@ -1,2 +1,3 @@
-import { a as typesOf, c as decodeBody, i as readJsonLd, l as canonicalUrl, n as PageLink, o as visibleText, r as extractPage, s as bodyEncoding, t as PageExtract } from "./html-nWYFVsRU.mjs";
+import { n as canonicalUrl } from "./urls-BTDSgDLC.mjs";
+import { a as typesOf, c as decodeBody, i as readJsonLd, n as PageLink, o as visibleText, r as extractPage, s as bodyEncoding, t as PageExtract } from "./html-BACljlkf.mjs";
 export { PageExtract, PageLink, bodyEncoding, canonicalUrl, decodeBody, extractPage, readJsonLd, typesOf, visibleText };

package/build/html.mjs

@@ -1 +1,3 @@
-export { a as bodyEncoding, c as canonicalUrl, o as decodeBody, t as extractPage, n as readJsonLd, r as typesOf, i as visibleText } from "./html-oNpGAHwi.mjs";
+import { n as canonicalUrl } from "./urls-BHFeSyEK.mjs";
+import { a as bodyEncoding, i as visibleText, n as readJsonLd, o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
+export { bodyEncoding, canonicalUrl, decodeBody, extractPage, readJsonLd, typesOf, visibleText };

package/build/index.d.mts

@@ -1,8 +1,9 @@
 import { DAYS, Day, Hours, Interval, Week, describeWeek, disagreements, fromCompact, fromSchemaOrg, fromText, toHoursWeek } from "./hours.mjs";
-import { a as typesOf, c as decodeBody, n as PageLink, r as extractPage, s as bodyEncoding, t as PageExtract } from "./html-nWYFVsRU.mjs";
-import { $ as FetchOutcome, A as findDocuments, B as CaptureFilter, C as ReadResult, Ct as RenderOptions, D as catchUpIndex, Dt as playwrightRenderer, E as VerifyResult, Et as Renderer, F as readDocument, G as CAPTURE_SCHEMA, H as CaptureReader, I as recordOverride, J as CaptureWriter, K as CacheEntry, L as reindex, M as isHeldBack, N as pageDocId, O as fetchAndIndex, Ot as BlobStore, P as pageExtractor, Q as DEFAULT_SECRET_PARAMS, R as renderAndIndex, S as ReadOptions, St as RefusedReason, T as VerifyOptions, Tt as Rendered, U as DamagedLine, V as CaptureLog, W as ScanOptions, X as Crawler, Y as CrawlEvent, Z as CrawlerOptions, _ as Extractors, _t as ReadingIndex, a as PoliteCrawler, at as RecordInput, b as PageOptions, bt as NO_PLAYWRIGHT, c as createRuntime, ct as EgressRefused, d as isMailbox, dt as DocIndex, et as FetchRequest, f as openCrawler, ft as DocInput, g as Extractor, gt as Latest, h as CrawlRuntime, ht as Found, i as OpenOptions, it as CaptureRecord, j as heldBackMeta, k as fetchPage, kt as FileBlobStore, l as defaultUserAgent, lt as egressLookup, m as CatchUpResult, mt as ExtractedDoc, n as FetchOptions, nt as RenderRequest, o as RuntimeOptions, ot as makeRecord, p as CapturePayload, pt as DocVersion, q as CacheIndex, r as FindOptions, rt as renderedKey, s as StoreRuntime, st as EgressOptions, t as Duration, tt as Fetched, u as durationMs, ut as Doc, v as HELD_BACK, vt as Variant, w as ReindexResult, wt as RenderRefused, x as PageResult, xt as PlaywrightRenderer, y as Indexed, yt as VersionText, z as verifyQuote } from "./open-CZqwY7GH.mjs";
+import { t as ScopeOptions } from "./urls-BTDSgDLC.mjs";
+import { a as typesOf, c as decodeBody, n as PageLink, r as extractPage, s as bodyEncoding, t as PageExtract } from "./html-BACljlkf.mjs";
+import { $ as DiscoverOptions, $n as verifyQuote, $t as SignedRequest, A as CrawlPage, An as Extractors, At as diffTexts, B as ExtractSummary, Bn as catchUpIndex, Bt as FetchOutcome, C as WatchStop, Cn as WebArchive, Ct as DiffSide, D as runWatch, Dn as CatchUpResult, Dt as VersionChoice, E as WatchSummary, En as CapturePayload, Et as TextDiff, F as UrlPattern, Fn as ReadOptions, Ft as CrawlEvent, G as FieldKind, Gn as isHeldBack, Gt as DirectoryKey, H as FieldCandidate, Hn as fetchPage, Ht as Fetched, I as crawlSite, In as ReadResult, It as Crawler, J as FieldStatus, Jn as pageExtractor, Jt as KeyDirectory, K as FieldResult, Kn as isoTime, Kt as KEY_DIRECTORY_PATH, L as ExtractOptions, Ln as ReindexResult, Lt as CrawlerOptions, M as CrawlRunOptions, Mn as Indexed, Mt as CacheEntry, N as CrawlStop, Nn as PageOptions, Nt as CacheIndex, O as CRAWL_LIMITS, On as CrawlRuntime, Ot as WordPart, P as CrawlSummary, Pn as PageResult, Pt as CaptureWriter, Q as DISCOVER_LIMITS, Qn as renderAndIndex, Qt as SignatureCheck, R as ExtractResult, Rn as VerifyOptions, Rt as DEADLINE_NOTE, S as WatchRunnerOptions, Sn as Archived, St as DiffResult, T as WatchStoreOptions, Tn as archivedDocId, Tt as MetaChange, U as FieldCheck, Un as findDocuments, Ut as RenderRequest, V as FIELD_KINDS, Vn as fetchAndIndex, Vt as FetchRequest, W as FieldClaim, Wn as heldBackMeta, Wt as renderedKey, X as checkFields, Xn as recordOverride, Xt as RequestSigner, Y as FieldsChecked, Yn as readDocument, Yt as KeyInput, Z as extractDocument, Zn as reindex, Zt as SignatureAgentType, _ as WatchList, _n as RankMode, _r as BlobStore, _t as DiffHunk, a as PoliteCrawler, an as EgressOptions, ar as ScanOptions, at as discoverSite, b as WatchRun, bn as VersionAt, bt as DiffLine, c as createRuntime, cn as Doc, cr as makeRecord, ct as SitemapEntry, d as openCrawler, dn as DocVersion, dr as RefusedReason, dt as SitemapRead, en as WebBotAuthOptions, er as versionAtTime, et as DiscoverResult, f as WATCH_LIMITS, fn as ExtractedDoc, fr as RenderOptions, ft as feedDate, g as WatchKind, gn as Latest, gr as playwrightRenderer, gt as DIFF_LIMITS, h as WatchEvent, hn as Found, hr as Renderer, ht as durationMs, i as OpenedCrawler, in as verifySignature, ir as DamagedLine, it as DiscoveredUrl, j as CrawlRun, jn as HELD_BACK, jt as CAPTURE_SCHEMA, k as CrawlOptions, kn as Extractor, kt as diffDocument, l as defaultUserAgent, ln as DocIndex, lr as NO_PLAYWRIGHT, lt as SitemapKind, m as WatchEntry, mn as FindIndexOptions, mr as Rendered, mt as Duration, n as FindOptions, nn as createWebBotAuthSigner, nr as CaptureLog, nt as DiscoverSource, o as RuntimeOptions, on as EgressRefused, or as CaptureRecord, ot as Changefreq, p as WATCH_SCHEMA, pn as FAST_RANK_SHARE, pr as RenderRefused, pt as readSitemap, q as FieldSpan, qn as pageDocId, qt as KEY_DIRECTORY_TYPE, r as OpenOptions, rn as keyDirectory, rr as CaptureReader, rt as DiscoverStop, s as StoreRuntime, sn as egressLookup, sr as RecordInput, st as SITEMAP_LIMITS, t as FetchOptions, tn as WebBotAuthSigner, tr as CaptureFilter, tt as DiscoverRunOptions, u as isMailbox, un as DocInput, ur as PlaywrightRenderer, ut as SitemapLimits, v as WatchOptions, vn as ReadingIndex, vr as FileBlobStore, vt as DiffInput, w as WatchStore, wn as archiveExtractor, wt as DiffSummary, x as WatchRunOptions, xn as VersionText, xt as DiffOptions, y as WatchPending, yn as Variant, yt as DiffLimits, z as ExtractSource, zn as VerifyResult, zt as DEFAULT_SECRET_PARAMS } from "./open-Ca1lcWR3.mjs";
 import { r as Signals, t as AiInput } from "./signals-BHpy2xb2.mjs";
-import { f as PolicyOptions, i as HostResult, n as DEFAULT_POLICY, o as HostStateStore, p as PolitenessPolicy, r as HostCounts, s as HostStop, t as Clock, u as MemoryHostStateStore, x as realClock } from "./politeness-C5bZ1OKM.mjs";
+import { E as nextValidatorMemo, O as realClock, S as ValidatorVerdict, _ as VALIDATOR_MEMO, b as ValidatorMemo, f as PolicyOptions, i as HostResult, n as DEFAULT_POLICY, o as HostStateStore, p as PolitenessPolicy, r as HostCounts, s as HostStop, t as Clock, u as MemoryHostStateStore, v as ValidatorAnswer, x as ValidatorReport, y as ValidatorLimits } from "./politeness-yI3xBEx9.mjs";
 import { a as Robots, c as RobotsRule, h as parseRobots, l as RobotsSignal, o as RobotsDecision, s as RobotsGroup, t as ALLOW_ALL, u as decide } from "./robots-R-qaEBOb.mjs";
 //#region src/browser.d.ts
 /**
@@ -49,8 +50,11 @@ interface SqliteHostStateOptions {
  * the lease gets a wait of one second or less, and asks again. A process that stops during a
  * request holds its lease until the lease ends (5 minutes by default).
  *
+ * The table `validators` keeps the validator memo of each host (`ValidatorMemo`), so the processes
+ * share it and it stays after a restart. A file of version 0.1.0 gets the table when it opens.
+ *
  * The state is not a projection of the capture log. A deletion of the file loses only the pace,
- * the counts and the stops, and the crawler starts them again.
+ * the counts, the stops and the memos, and the crawler starts them again.
  */
 export declare class SqliteHostStateStore implements HostStateStore {
   readonly path: string;
@@ -62,6 +66,11 @@ export declare class SqliteHostStateStore implements HostStateStore {
   private closed;
   private readonly pollMs;
   constructor(path: string, options?: SqliteHostStateOptions);
+  /**
+   * This method makes the tables, or adds the tables of a later version to an older file. It is one
+   * transaction, so two processes that open one old file at the same time do not collide.
+   */
+  private migrate;
   private stmt;
   /** This method does one step in one transaction. */
   private step;
@@ -75,6 +84,8 @@ export declare class SqliteHostStateStore implements HostStateStore {
   stop(host: string, until: number, reason: string): void;
   stopped(host: string, now: number): HostStop | undefined;
   reset(host?: string): void;
+  validators(host: string): ValidatorMemo | undefined;
+  noteValidators(host: string, report: ValidatorReport, now: number): ValidatorMemo;
   /** This value is the number of hosts that the store holds. */
   get size(): number;
   /** This method closes the file. A second close does nothing. */
@@ -83,7 +94,7 @@ export declare class SqliteHostStateStore implements HostStateStore {
 //#endregion
 //#region src/version.d.ts
 /** This is the version of the package. A test makes sure that it agrees with `package.json`. */
-export declare const VERSION = "0.1.0";
+export declare const VERSION = "0.1.2";
 /** This is the product token that robots.txt rules match when an application gives no token. */
 export declare const DEFAULT_TOKEN = "PoliteCrawler";
 /** This page tells a site owner what the crawler is and how to control it. */
@@ -127,5 +138,94 @@ interface PdfOptions {
 export declare const isPdf: (body: Uint8Array, contentType: string | null) => boolean;
 export declare const readPdf: (bytes: Uint8Array, options?: PdfOptions) => Promise<PdfText>;
 //#endregion
-export { ALLOW_ALL, type AiInput, type BlobStore, CAPTURE_SCHEMA, type CacheEntry, type CacheIndex, type CaptureFilter, CaptureLog, type CapturePayload, type CaptureReader, type CaptureRecord, type CaptureWriter, type CatchUpResult, type Clock, type CrawlEvent, type CrawlRuntime, Crawler, type CrawlerOptions, DAYS, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, type DamagedLine, type Day, type Doc, DocIndex, type DocInput, type DocVersion, type Duration, type EgressOptions, EgressRefused, type ExtractedDoc, type Extractor, Extractors, type FetchOptions, type FetchOutcome, type FetchRequest, type Fetched, FileBlobStore, type FindOptions, type Found, HELD_BACK, type HostCounts, type HostResult, type HostStateStore, type HostStop, type Hours, type Indexed, type Interval, type Latest, MemoryHostStateStore, NO_PLAYWRIGHT, type OpenOptions, type PageExtract, type PageLink, type PageOptions, type PageResult, type PdfOptions, type PdfText, type PlaywrightRenderer, type PolicyOptions, type PoliteCrawler, type PolitenessPolicy, type ReadOptions, type ReadResult, type ReadingIndex, type RecordInput, type RefusedReason, type ReindexResult, type RenderOptions, RenderRefused, type RenderRequest, type Rendered, type Renderer, type Robots, type RobotsDecision, type RobotsGroup, type RobotsRule, type RobotsSignal, type RuntimeOptions, type ScanOptions, type Signals, type StoreRuntime, type Variant, type Verified, type VerifyOptions, type VerifyResult, type VersionText, type Week, bodyEncoding, catchUpIndex, createRuntime, decide, decodeBody, defaultUserAgent, describeWeek, disagreements, durationMs, egressLookup, extractPage, fetchAndIndex, fetchPage, findDocuments, fromCompact, fromSchemaOrg, fromText, heldBackMeta, isHeldBack, isMailbox, makeRecord, openCrawler, pageDocId, pageExtractor, parseRobots, playwrightRenderer, readDocument, realClock, recordOverride, reindex, renderAndIndex, renderedKey, toHoursWeek, typesOf, verifyQuote };
+//#region src/warc.d.ts
+/** The source of an export: the folder of a store, or the capture log and the blob store of a runtime. */
+type ArchiveSource = string | {
+  readonly captures: CaptureReader;
+  readonly blobs: BlobStore;
+};
+/** These filters select the captures of an export. Each filter that you give makes the selection smaller. */
+interface ArchiveFilter {
+  /** The export holds the captures received at this time or later. */
+  readonly since?: string | Date | undefined;
+  /** The export holds the captures received before this time. */
+  readonly until?: string | Date | undefined;
+  /** The export holds the captures of this host and of its subdomains, for example `example.com`. */
+  readonly host?: string | undefined;
+  /** The export holds the captures whose URL starts with this text. */
+  readonly urlPrefix?: string | undefined;
+  /** The export holds only the captures with these ids. */
+  readonly ids?: readonly string[] | undefined;
+}
+/** These options control an export. Only `out` is necessary. */
+interface ArchiveOptions extends ArchiveFilter {
+  /** This file gets the archive. The export writes a temporary file and gives it this name at the end. */
+  readonly out: string;
+  /**
+   * When this value is true, each capture with no body (a refusal, a skip of robots.txt, an error, a
+   * redirect) gets a `metadata` record. The default is false: the export leaves them out.
+   */
+  readonly includeRefusals?: boolean | undefined;
+  /** When this value is true, the export replaces a file that exists. The default is false: the export fails. */
+  readonly overwrite?: boolean | undefined;
+}
+/** These counts tell what an export wrote. */
+interface ArchiveCounts {
+  /** This value is the count of capture records that the filters selected. */
+  readonly captures: number;
+  readonly responses: number;
+  readonly revisits: number;
+  /** This value is the count of rendered pages (`conversion` records). */
+  readonly conversions: number;
+  readonly metadata: number;
+  /** This value is the count of captures with no body: refusals, skips, errors and redirects. */
+  readonly bodiless: number;
+  /** This value is the count of captures whose body is not in the blob store, or does not agree with its hash. */
+  readonly missing: number;
+  /** This value is the count of damaged lines of the capture log. */
+  readonly damaged: number;
+}
+/** This is the result of `exportWarc`. */
+interface WarcResult extends ArchiveCounts {
+  readonly format: "warc";
+  /** This value is the absolute path of the archive. */
+  readonly out: string;
+  readonly bytes: number;
+}
+/**
+ * This function writes the captures of a store as a WARC/1.1 file, with one gzip member for each
+ * record (`.warc.gz`). It reads the capture log as a stream, so its memory does not grow with the
+ * store. It writes a temporary file and gives it the name `out` at the end, so a failed export
+ * leaves no part of a file.
+ *
+ * The source is the folder of a store, or the capture log and the blob store of a runtime
+ * (`crawler.runtime`). The export only reads the store.
+ */
+export declare const exportWarc: (source: ArchiveSource, options: ArchiveOptions) => Promise<WarcResult>;
+//#endregion
+//#region src/wacz.d.ts
+/** This is the result of `exportWacz`. */
+interface WaczResult extends ArchiveCounts {
+  readonly format: "wacz";
+  /** This value is the absolute path of the archive. */
+  readonly out: string;
+  readonly bytes: number;
+  /** This value is the count of pages in `pages/pages.jsonl`: one for each document. */
+  readonly pages: number;
+}
+/**
+ * This function writes the captures of a store as a WACZ 1.1.1 file. The ZIP holds these parts:
+ *
+ * - `archive/data.warc.gz`: the WARC file of `exportWarc`, stored with no more compression, so a
+ *   replay tool reads a record at its offset.
+ * - `indexes/index.cdxj`: the index of the `response` and `revisit` records, sorted by SURT key.
+ * - `pages/pages.jsonl`: one page for each document, at its latest capture.
+ * - `datapackage.json` and `datapackage-digest.json`: the SHA-256 of each part, and of the package.
+ *
+ * The index and the page list are sorted on the disk, so the memory does not grow with the store.
+ * The export writes temporary files beside `out` and removes them at the end.
+ */
+export declare const exportWacz: (source: ArchiveSource, options: ArchiveOptions) => Promise<WaczResult>;
+//#endregion
+export { ALLOW_ALL, type AiInput, type ArchiveCounts, type ArchiveFilter, type ArchiveOptions, type ArchiveSource, type Archived, type BlobStore, CAPTURE_SCHEMA, CRAWL_LIMITS, type CacheEntry, type CacheIndex, type CaptureFilter, CaptureLog, type CapturePayload, type CaptureReader, type CaptureRecord, type CaptureWriter, type CatchUpResult, type Changefreq, type Clock, type CrawlEvent, type CrawlOptions, type CrawlPage, type CrawlRun, type CrawlRunOptions, type CrawlRuntime, type CrawlStop, type CrawlSummary, Crawler, type CrawlerOptions, DAYS, DEADLINE_NOTE, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, DIFF_LIMITS, DISCOVER_LIMITS, type DamagedLine, type Day, type DiffHunk, type DiffInput, type DiffLimits, type DiffLine, type DiffOptions, type DiffResult, type DiffSide, type DiffSummary, type DirectoryKey, type DiscoverOptions, type DiscoverResult, type DiscoverRunOptions, type DiscoverSource, type DiscoverStop, type DiscoveredUrl, type Doc, DocIndex, type DocInput, type DocVersion, type Duration, type EgressOptions, EgressRefused, type ExtractOptions, type ExtractResult, type ExtractSource, type ExtractSummary, type ExtractedDoc, type Extractor, Extractors, FAST_RANK_SHARE, FIELD_KINDS, type FetchOptions, type FetchOutcome, type FetchRequest, type Fetched, type FieldCandidate, type FieldCheck, type FieldClaim, type FieldKind, type FieldResult, type FieldSpan, type FieldStatus, type FieldsChecked, FileBlobStore, type FindIndexOptions, type FindOptions, type Found, HELD_BACK, type HostCounts, type HostResult, type HostStateStore, type HostStop, type Hours, type Indexed, type Interval, KEY_DIRECTORY_PATH, KEY_DIRECTORY_TYPE, type KeyDirectory, type KeyInput, type Latest, MemoryHostStateStore, type MetaChange, NO_PLAYWRIGHT, type OpenOptions, type OpenedCrawler, type PageExtract, type PageLink, type PageOptions, type PageResult, type PdfOptions, type PdfText, type PlaywrightRenderer, type PolicyOptions, type PoliteCrawler, type PolitenessPolicy, type RankMode, type ReadOptions, type ReadResult, type ReadingIndex, type RecordInput, type RefusedReason, type ReindexResult, type RenderOptions, RenderRefused, type RenderRequest, type Rendered, type Renderer, type RequestSigner, type Robots, type RobotsDecision, type RobotsGroup, type RobotsRule, type RobotsSignal, type RuntimeOptions, SITEMAP_LIMITS, type ScanOptions, type ScopeOptions, type Signals, type SignatureAgentType, type SignatureCheck, type SignedRequest, type SitemapEntry, type SitemapKind, type SitemapLimits, type SitemapRead, type StoreRuntime, type TextDiff, type UrlPattern, VALIDATOR_MEMO, type ValidatorAnswer, type ValidatorLimits, type ValidatorMemo, type ValidatorReport, type ValidatorVerdict, type Variant, type Verified, type VerifyOptions, type VerifyResult, type VersionAt, type VersionChoice, type VersionText, WATCH_LIMITS, WATCH_SCHEMA, type WaczResult, type WarcResult, type WatchEntry, type WatchEvent, type WatchKind, type WatchList, type WatchOptions, type WatchPending, type WatchRun, type WatchRunOptions, type WatchRunnerOptions, type WatchStop, WatchStore, type WatchStoreOptions, type WatchSummary, type WebArchive, type WebBotAuthOptions, type WebBotAuthSigner, type Week, type WordPart, archiveExtractor, archivedDocId, bodyEncoding, catchUpIndex, checkFields, crawlSite, createRuntime, createWebBotAuthSigner, decide, decodeBody, defaultUserAgent, describeWeek, diffDocument, diffTexts, disagreements, discoverSite, durationMs, egressLookup, extractDocument, extractPage, feedDate, fetchAndIndex, fetchPage, findDocuments, fromCompact, fromSchemaOrg, fromText, heldBackMeta, isHeldBack, isMailbox, isoTime, keyDirectory, makeRecord, nextValidatorMemo, openCrawler, pageDocId, pageExtractor, parseRobots, playwrightRenderer, readDocument, readSitemap, realClock, recordOverride, reindex, renderAndIndex, renderedKey, runWatch, toHoursWeek, typesOf, verifyQuote, verifySignature, versionAtTime };
 //# sourceMappingURL=index.d.mts.map

package/build/index.mjs

@@ -1,8 +1,16 @@
-import { a as RenderRefused, i as NO_PLAYWRIGHT, o as playwrightRenderer, r as checkedRenderer } from "./browser-gJ_h4AhB.mjs";
-import { A as DEFAULT_SECRET_PARAMS, C as readPdf, D as locate, E as DocIndex, F as FileBlobStore, M as makeRecord, N as EgressRefused, O as CAPTURE_SCHEMA, P as egressLookup, S as isPdf, T as SqliteHostStateStore, _ as readDocument, a as openCrawler, b as renderAndIndex, c as HELD_BACK, d as fetchPage, f as findDocuments, g as pageExtractor, h as pageDocId, i as isMailbox, j as renderedKey, k as Crawler, l as catchUpIndex, m as isHeldBack, n as defaultUserAgent, p as heldBackMeta, r as durationMs, s as Extractors, t as createRuntime, u as fetchAndIndex, v as recordOverride, w as CaptureLog, x as verifyQuote, y as reindex } from "./open-CJR4b44v.mjs";
-import { DEFAULT_POLICY, MemoryHostStateStore, realClock } from "./politeness.mjs";
+import { n as FileBlobStore, t as CaptureLog } from "./log-Dx-wXjJl.mjs";
+import { n as RenderRefused, r as playwrightRenderer, t as NO_PLAYWRIGHT } from "./render-C1CUraWF.mjs";
+import { r as checkedRenderer } from "./browser-CKRNbIDT.mjs";
+import { A as Crawler, C as readPdf, E as locate, L as EgressRefused, M as DEFAULT_SECRET_PARAMS, N as renderedKey, P as makeRecord, R as egressLookup, S as isPdf, _ as versionAtTime, a as fetchPage, c as isHeldBack, d as pageExtractor, f as readDocument, g as verifyQuote, h as renderAndIndex, i as fetchAndIndex, j as DEADLINE_NOTE, k as CAPTURE_SCHEMA, l as isoTime, m as reindex, n as HELD_BACK, o as findDocuments, p as recordOverride, r as catchUpIndex, s as heldBackMeta, t as Extractors, u as pageDocId, v as archiveExtractor, y as archivedDocId } from "./runtime-k8mo1_7U.mjs";
+import { DEFAULT_POLICY, MemoryHostStateStore, VALIDATOR_MEMO, nextValidatorMemo, realClock } from "./politeness.mjs";
 import { ALLOW_ALL, decide, parseRobots } from "./robots.mjs";
-import { n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-rVx8_y1_.mjs";
-import { a as bodyEncoding, o as decodeBody, r as typesOf, t as extractPage } from "./html-oNpGAHwi.mjs";
+import { i as VERSION, n as BOT_PAGE, r as DEFAULT_TOKEN, t as durationMs } from "./duration-iIuMMofi.mjs";
+import { i as keyDirectory, n as KEY_DIRECTORY_TYPE, o as verifySignature, r as createWebBotAuthSigner, t as KEY_DIRECTORY_PATH } from "./webbotauth-CjJdTAfE.mjs";
+import { _ as readSitemap, c as WatchStore, d as CRAWL_LIMITS, f as crawlSite, g as feedDate, h as SITEMAP_LIMITS, i as openCrawler, l as runWatch, m as discoverSite, n as defaultUserAgent, o as WATCH_LIMITS, p as DISCOVER_LIMITS, r as isMailbox, s as WATCH_SCHEMA, t as createRuntime, u as SqliteHostStateStore, v as DocIndex, y as FAST_RANK_SHARE } from "./open-DnrWgkC0.mjs";
+import { a as bodyEncoding, o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
+import { n as diffDocument, r as diffTexts, t as DIFF_LIMITS } from "./diff-Dk2eSvi-.mjs";
+import { n as checkFields, r as extractDocument, t as FIELD_KINDS } from "./extract-DcwNrTK7.mjs";
 import { DAYS, describeWeek, disagreements, fromCompact, fromSchemaOrg, fromText, toHoursWeek } from "./hours.mjs";
-export { ALLOW_ALL, BOT_PAGE, CAPTURE_SCHEMA, CaptureLog, Crawler, DAYS, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, DEFAULT_TOKEN, DocIndex, EgressRefused, Extractors, FileBlobStore, HELD_BACK, MemoryHostStateStore, NO_PLAYWRIGHT, RenderRefused, SqliteHostStateStore, VERSION, bodyEncoding, catchUpIndex, checkedRenderer, createRuntime, decide, decodeBody, defaultUserAgent, describeWeek, disagreements, durationMs, egressLookup, extractPage, fetchAndIndex, fetchPage, findDocuments, fromCompact, fromSchemaOrg, fromText, heldBackMeta, isHeldBack, isMailbox, isPdf, locate, makeRecord, openCrawler, pageDocId, pageExtractor, parseRobots, playwrightRenderer, readDocument, readPdf, realClock, recordOverride, reindex, renderAndIndex, renderedKey, toHoursWeek, typesOf, verifyQuote };
+import { t as exportWacz } from "./wacz-sfOLT8e9.mjs";
+import { n as exportWarc } from "./warc-VIPscWxY.mjs";
+export { ALLOW_ALL, BOT_PAGE, CAPTURE_SCHEMA, CRAWL_LIMITS, CaptureLog, Crawler, DAYS, DEADLINE_NOTE, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, DEFAULT_TOKEN, DIFF_LIMITS, DISCOVER_LIMITS, DocIndex, EgressRefused, Extractors, FAST_RANK_SHARE, FIELD_KINDS, FileBlobStore, HELD_BACK, KEY_DIRECTORY_PATH, KEY_DIRECTORY_TYPE, MemoryHostStateStore, NO_PLAYWRIGHT, RenderRefused, SITEMAP_LIMITS, SqliteHostStateStore, VALIDATOR_MEMO, VERSION, WATCH_LIMITS, WATCH_SCHEMA, WatchStore, archiveExtractor, archivedDocId, bodyEncoding, catchUpIndex, checkFields, checkedRenderer, crawlSite, createRuntime, createWebBotAuthSigner, decide, decodeBody, defaultUserAgent, describeWeek, diffDocument, diffTexts, disagreements, discoverSite, durationMs, egressLookup, exportWacz, exportWarc, extractDocument, extractPage, feedDate, fetchAndIndex, fetchPage, findDocuments, fromCompact, fromSchemaOrg, fromText, heldBackMeta, isHeldBack, isMailbox, isPdf, isoTime, keyDirectory, locate, makeRecord, nextValidatorMemo, openCrawler, pageDocId, pageExtractor, parseRobots, playwrightRenderer, readDocument, readPdf, readSitemap, realClock, recordOverride, reindex, renderAndIndex, renderedKey, runWatch, toHoursWeek, typesOf, verifyQuote, verifySignature, versionAtTime };

package/build/log-Dx-wXjJl.mjs

@@ -0,0 +1,342 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, open, readFile, readdir, rename, rm, stat } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { createReadStream, existsSync } from "node:fs";
+//#region src/blobs.ts
+const HASH = /^[0-9a-f]{64}$/;
+/** A rename can fail with these codes when a different writer puts the same blob at the same time. */
+const RACE_CODES = /* @__PURE__ */ new Set([
+	"EPERM",
+	"EEXIST",
+	"EBUSY",
+	"EACCES",
+	"ENOTEMPTY"
+]);
+const blobPath = (root, hash) => join(root, hash.slice(0, 2), hash.slice(2, 4), hash);
+const hashOf = (bytes) => createHash("sha256").update(bytes).digest("hex");
+const codeOf = (error) => typeof error === "object" && error !== null && "code" in error ? String(error.code) : "";
+const statOf = (path) => stat(path).catch(() => void 0);
+/** This function writes a folder to the disk, so a rename in it stays after a power cut. Windows cannot open a folder, so it skips this step. */
+const syncFolder = async (path) => {
+	if (process.platform === "win32") return;
+	try {
+		const handle = await open(path, "r");
+		try {
+			await handle.sync();
+		} finally {
+			await handle.close();
+		}
+	} catch {}
+};
+/**
+* This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash
+* of the bytes. These are its rules:
+*
+* - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is
+*   written again.
+* - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.
+*   A crash thus cannot leave half a blob under its name.
+* - Two writers can put the same bytes at the same time. When the rename of one fails because the
+*   other was first, the put succeeds if the stored file has the same hash.
+* - The method `get` examines the hash of the bytes. A damaged file gets a new name
+*   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.
+*/
+var FileBlobStore = class FileBlobStore {
+	dir;
+	writing = /* @__PURE__ */ new Map();
+	sync;
+	constructor(dir, options = {}) {
+		this.dir = dir;
+		this.sync = options.sync ?? true;
+	}
+	put(bytes) {
+		const hash = hashOf(bytes);
+		const running = this.writing.get(hash);
+		if (running !== void 0) return running;
+		const work = this.write(hash, bytes).finally(() => this.writing.delete(hash));
+		this.writing.set(hash, work);
+		return work;
+	}
+	/** This method tells if a file holds exactly the bytes of a hash. */
+	static async holds(path, hash) {
+		try {
+			return hashOf(await readFile(path)) === hash;
+		} catch {
+			return false;
+		}
+	}
+	async write(hash, bytes) {
+		const target = blobPath(this.dir, hash);
+		const existing = await statOf(target);
+		if (existing?.isFile() === true && existing.size === bytes.length) return {
+			hash,
+			existed: true
+		};
+		const folder = dirname(target);
+		await mkdir(folder, {
+			recursive: true,
+			mode: 448
+		});
+		const temporary = `${target}.${randomUUID()}.tmp`;
+		try {
+			const handle = await open(temporary, "wx", 384);
+			try {
+				await handle.writeFile(bytes);
+				if (this.sync) await handle.sync();
+			} finally {
+				await handle.close();
+			}
+			try {
+				await rename(temporary, target);
+			} catch (error) {
+				if (!RACE_CODES.has(codeOf(error)) || !await FileBlobStore.holds(target, hash)) throw error;
+				return {
+					hash,
+					existed: true
+				};
+			}
+			if (this.sync) await syncFolder(folder);
+			return {
+				hash,
+				existed: false
+			};
+		} finally {
+			await rm(temporary, { force: true }).catch(() => void 0);
+		}
+	}
+	async get(hash) {
+		if (!HASH.test(hash)) return void 0;
+		const target = blobPath(this.dir, hash);
+		let bytes;
+		try {
+			bytes = await readFile(target);
+		} catch {
+			return;
+		}
+		if (hashOf(bytes) !== hash) {
+			await rename(target, `${target}.${randomUUID()}.damaged`).catch(() => void 0);
+			return;
+		}
+		return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	}
+};
+//#endregion
+//#region src/log.ts
+const MONTH_FILE = /^(\d{2})(?:\.[^.]+)?\.ndjson$/;
+const YEAR_DIR = /^\d{4}$/;
+const NEWLINE = 10;
+/** This function writes the whole buffer. A write to a file can write less than the buffer, and the loop writes the rest. */
+const writeAll = async (handle, bytes) => {
+	let at = 0;
+	while (at < bytes.length) {
+		const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);
+		at += bytesWritten;
+	}
+};
+/**
+* This function reads a file line by line, as bytes, so a long file does not become one string. It
+* gives each line and tells if a line break ends it.
+*/
+async function* linesOf(path) {
+	const decoder = new TextDecoder("utf-8");
+	let rest;
+	for await (const chunk of createReadStream(path)) {
+		const data = rest === void 0 ? chunk : Buffer.concat([rest, chunk]);
+		rest = void 0;
+		let start = 0;
+		for (;;) {
+			const end = data.indexOf(NEWLINE, start);
+			if (end === -1) break;
+			yield {
+				text: decoder.decode(data.subarray(start, end)),
+				complete: true
+			};
+			start = end + 1;
+		}
+		if (start < data.length) rest = Buffer.from(data.subarray(start));
+	}
+	if (rest !== void 0 && rest.length > 0) yield {
+		text: decoder.decode(rest),
+		complete: false
+	};
+}
+const monthKey = (year, file) => `${year}-${MONTH_FILE.exec(file)?.[1] ?? "00"}`;
+/**
+* The capture log keeps each fetch decision as one line of JSON in
+* `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.
+*
+* These are the rules of the log:
+*
+* - The log appends a batch of records with one write, and it writes the batch to the disk before
+*   the appends finish.
+* - An append always starts on a new line. After a crash leaves a partial line, the next append
+*   starts after a line break. The partial line stays as a damaged line, and the record is good.
+* - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.
+* - The log does not remove or change a line.
+*
+* Two processes can append to one log, because each batch is one write to a file in append mode.
+*/
+var CaptureLog = class CaptureLog {
+	dir;
+	handles = /* @__PURE__ */ new Map();
+	pending = [];
+	draining;
+	sync;
+	constructor(dir, options = {}) {
+		this.dir = dir;
+		this.sync = options.sync ?? true;
+	}
+	pathOf(record) {
+		return join(this.dir, record.source, record.received_at.slice(0, 4), `${record.received_at.slice(5, 7)}.ndjson`);
+	}
+	async handleFor(path) {
+		const known = this.handles.get(path);
+		if (known !== void 0) return known;
+		await mkdir(dirname(path), {
+			recursive: true,
+			mode: 448
+		});
+		const handle = await open(path, "a+", 384);
+		this.handles.set(path, handle);
+		return handle;
+	}
+	/** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */
+	static async needsBreak(handle) {
+		const { size } = await handle.stat();
+		if (size === 0) return false;
+		const last = Buffer.alloc(1);
+		await handle.read(last, 0, 1, size - 1);
+		return last[0] !== NEWLINE;
+	}
+	append(record) {
+		let line;
+		try {
+			line = `${JSON.stringify(record)}\n`;
+		} catch (error) {
+			return Promise.reject(error instanceof Error ? error : new Error(String(error)));
+		}
+		return new Promise((resolve, reject) => {
+			this.pending.push({
+				path: this.pathOf(record),
+				line,
+				resolve,
+				reject
+			});
+			this.kick();
+		});
+	}
+	/** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */
+	kick() {
+		if (this.draining !== void 0) return;
+		this.draining = this.drain().finally(() => {
+			this.draining = void 0;
+			if (this.pending.length > 0) this.kick();
+		});
+	}
+	/** This method writes the waiting records in batches: one write and one sync for each file. */
+	async drain() {
+		while (this.pending.length > 0) {
+			const batch = this.pending.splice(0);
+			const byPath = /* @__PURE__ */ new Map();
+			for (const item of batch) {
+				const list = byPath.get(item.path);
+				if (list === void 0) byPath.set(item.path, [item]);
+				else list.push(item);
+			}
+			for (const [path, items] of byPath) try {
+				const handle = await this.handleFor(path);
+				const lead = await CaptureLog.needsBreak(handle) ? "\n" : "";
+				await writeAll(handle, Buffer.from(`${lead}${items.map((item) => item.line).join("")}`, "utf8"));
+				if (this.sync) await handle.datasync();
+				for (const item of items) item.resolve();
+			} catch (error) {
+				const handle = this.handles.get(path);
+				this.handles.delete(path);
+				await handle?.close().catch(() => void 0);
+				for (const item of items) item.reject(error);
+			}
+		}
+	}
+	/** This method waits until each append in progress is on the disk. */
+	async flush() {
+		while (this.draining !== void 0) await this.draining;
+	}
+	/** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */
+	async filesOf(source, since) {
+		const sourceDir = join(this.dir, source);
+		if (!existsSync(sourceDir)) return [];
+		const from = since?.slice(0, 7);
+		const years = (await readdir(sourceDir, { withFileTypes: true })).filter((entry) => entry.isDirectory() && YEAR_DIR.test(entry.name)).map((entry) => entry.name).sort();
+		const files = [];
+		for (const year of years) {
+			if (from !== void 0 && year < from.slice(0, 4)) continue;
+			const months = (await readdir(join(sourceDir, year))).filter((name) => MONTH_FILE.test(name)).sort();
+			for (const month of months) {
+				if (from !== void 0 && monthKey(year, month) < from) continue;
+				files.push(join(sourceDir, year, month));
+			}
+		}
+		return files;
+	}
+	/** This method reads the records of one source in the sequence of the files. */
+	async *readSource(source, filter, options) {
+		for (const path of await this.filesOf(source, filter?.since)) {
+			let number = 0;
+			for await (const { text, complete } of linesOf(path)) {
+				number += 1;
+				const trimmed = text.trim();
+				if (trimmed.length === 0) continue;
+				let row;
+				try {
+					row = JSON.parse(trimmed);
+					if (typeof row !== "object" || row === null || typeof row.received_at !== "string") throw new Error("not a record");
+				} catch {
+					if (complete) options?.onDamaged?.({
+						path,
+						line: number
+					});
+					continue;
+				}
+				if (filter?.kind !== void 0 && row.kind !== filter.kind) continue;
+				if (filter?.since !== void 0 && row.received_at < filter.since) continue;
+				yield row;
+			}
+		}
+	}
+	/**
+	* This method reads the records that agree with the filter. It first waits for the appends that
+	* are in progress. It reads each file as a stream, so its memory does not grow with the log.
+	*
+	* The records of one source come in the sequence of the files. The records of two or more sources
+	* merge in the sequence of `received_at`.
+	*/
+	async *scan(filter, options) {
+		await this.flush();
+		const sources = filter?.source !== void 0 ? [filter.source] : existsSync(this.dir) ? (await readdir(this.dir, { withFileTypes: true })).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort() : [];
+		if (sources.length === 1) {
+			yield* this.readSource(sources[0], filter, options);
+			return;
+		}
+		const streams = sources.map((source) => this.readSource(source, filter, options)[Symbol.asyncIterator]());
+		const heads = [];
+		for (const stream of streams) heads.push((await stream.next()).value);
+		for (;;) {
+			let pick = -1;
+			for (const [index, head] of heads.entries()) if (head !== void 0 && (pick === -1 || head.received_at < heads[pick].received_at)) pick = index;
+			if (pick === -1) return;
+			yield heads[pick];
+			heads[pick] = (await streams[pick].next()).value;
+		}
+	}
+	/** This method writes the appends in progress, then closes the files. A later append opens them again. */
+	async close() {
+		await this.flush();
+		const handles = [...this.handles.values()];
+		this.handles.clear();
+		for (const handle of handles) await handle.close();
+	}
+};
+//#endregion
+export { FileBlobStore as n, CaptureLog as t };
+
+//# sourceMappingURL=log-Dx-wXjJl.mjs.map

package/build/log-Dx-wXjJl.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"log-Dx-wXjJl.mjs","names":[],"sources":["../src/blobs.ts","../src/log.ts"],"sourcesContent":["import { createHash, randomUUID } from \"node:crypto\";\nimport type { Stats } from \"node:fs\";\nimport { mkdir, open, readFile, rename, rm, stat } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\n\n/**\n * A blob store keeps the bytes of each response, and the hash of the bytes is the key. The crawler\n * needs only these two operations, so an application can give its own store.\n */\nexport interface BlobStore {\n  put(bytes: Uint8Array): Promise<{ readonly hash: string }>;\n  get(hash: string): Promise<Uint8Array | undefined>;\n}\n\n/** These options change a file blob store. */\nexport interface FileBlobStoreOptions {\n  /** When this value is true (the default), a blob is on the disk (`fsync`) before `put` finishes. */\n  readonly sync?: boolean | undefined;\n}\n\nconst HASH = /^[0-9a-f]{64}$/;\n\n/** A rename can fail with these codes when a different writer puts the same blob at the same time. */\nconst RACE_CODES: ReadonlySet<string> = new Set([\n  \"EPERM\",\n  \"EEXIST\",\n  \"EBUSY\",\n  \"EACCES\",\n  \"ENOTEMPTY\",\n]);\n\nconst blobPath = (root: string, hash: string): string =>\n  join(root, hash.slice(0, 2), hash.slice(2, 4), hash);\n\nconst hashOf = (bytes: Uint8Array): string => createHash(\"sha256\").update(bytes).digest(\"hex\");\n\nconst codeOf = (error: unknown): string =>\n  typeof error === \"object\" && error !== null && \"code\" in error ? String(error.code) : \"\";\n\nconst statOf = (path: string): Promise<Stats | undefined> => stat(path).catch(() => undefined);\n\n/** This function writes a folder to the disk, so a rename in it stays after a power cut. Windows cannot open a folder, so it skips this step. */\nconst syncFolder = async (path: string): Promise<void> => {\n  if (process.platform === \"win32\") return;\n  try {\n    const handle = await open(path, \"r\");\n    try {\n      await handle.sync();\n    } finally {\n      await handle.close();\n    }\n  } catch {\n    // a file system that cannot sync a folder keeps the blob in any case\n  }\n};\n\n/**\n * This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash\n * of the bytes. These are its rules:\n *\n * - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is\n *   written again.\n * - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.\n *   A crash thus cannot leave half a blob under its name.\n * - Two writers can put the same bytes at the same time. When the rename of one fails because the\n *   other was first, the put succeeds if the stored file has the same hash.\n * - The method `get` examines the hash of the bytes. A damaged file gets a new name\n *   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.\n */\nexport class FileBlobStore implements BlobStore {\n  private readonly writing = new Map<\n    string,\n    Promise<{ readonly hash: string; readonly existed: boolean }>\n  >();\n  private readonly sync: boolean;\n\n  constructor(\n    readonly dir: string,\n    options: FileBlobStoreOptions = {},\n  ) {\n    this.sync = options.sync ?? true;\n  }\n\n  put(bytes: Uint8Array): Promise<{ readonly hash: string; readonly existed: boolean }> {\n    const hash = hashOf(bytes);\n    const running = this.writing.get(hash);\n    if (running !== undefined) return running;\n    const work = this.write(hash, bytes).finally(() => this.writing.delete(hash));\n    this.writing.set(hash, work);\n    return work;\n  }\n\n  /** This method tells if a file holds exactly the bytes of a hash. */\n  private static async holds(path: string, hash: string): Promise<boolean> {\n    try {\n      return hashOf(await readFile(path)) === hash;\n    } catch {\n      return false;\n    }\n  }\n\n  private async write(\n    hash: string,\n    bytes: Uint8Array,\n  ): Promise<{ readonly hash: string; readonly existed: boolean }> {\n    const target = blobPath(this.dir, hash);\n    const existing = await statOf(target);\n    if (existing?.isFile() === true && existing.size === bytes.length)\n      return { hash, existed: true };\n    const folder = dirname(target);\n    await mkdir(folder, { recursive: true, mode: 0o700 });\n    const temporary = `${target}.${randomUUID()}.tmp`;\n    try {\n      const handle = await open(temporary, \"wx\", 0o600);\n      try {\n        await handle.writeFile(bytes);\n        if (this.sync) await handle.sync();\n      } finally {\n        await handle.close();\n      }\n      try {\n        await rename(temporary, target);\n      } catch (error) {\n        // a different writer put the same bytes first, and Windows does not replace a file in use\n        if (!RACE_CODES.has(codeOf(error)) || !(await FileBlobStore.holds(target, hash)))\n          throw error;\n        return { hash, existed: true };\n      }\n      if (this.sync) await syncFolder(folder);\n      return { hash, existed: false };\n    } finally {\n      await rm(temporary, { force: true }).catch(() => undefined);\n    }\n  }\n\n  async get(hash: string): Promise<Uint8Array | undefined> {\n    if (!HASH.test(hash)) return undefined;\n    const target = blobPath(this.dir, hash);\n    let bytes: Buffer;\n    try {\n      bytes = await readFile(target);\n    } catch {\n      return undefined;\n    }\n    if (hashOf(bytes) !== hash) {\n      // a torn or damaged file is never the body: it moves away, and the next put writes it again\n      await rename(target, `${target}.${randomUUID()}.damaged`).catch(() => undefined);\n      return undefined;\n    }\n    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n  }\n}\n","import { createReadStream, existsSync } from \"node:fs\";\nimport type { FileHandle } from \"node:fs/promises\";\nimport { mkdir, open, readdir } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\nimport type { CaptureRecord } from \"./record.ts\";\n\n/** A scan filter selects the records of one source, one kind, or the records after one time. */\nexport interface CaptureFilter {\n  readonly source?: string | undefined;\n  readonly kind?: string | undefined;\n  /** This value is an ISO 8601 time. The scan gives the records received at this time or later. */\n  readonly since?: string | undefined;\n}\n\n/** A damaged line is a line of the log that is not a record: a torn append, or bytes that a crash left. */\nexport interface DamagedLine {\n  /** This value is the path of the file. */\n  readonly path: string;\n  /** This value is the number of the line in the file, from 1. */\n  readonly line: number;\n}\n\n/** These options change a scan. */\nexport interface ScanOptions {\n  /** The scan skips a damaged line, and it gives the line to this function. */\n  readonly onDamaged?: ((damaged: DamagedLine) => void) | undefined;\n}\n\n/** The read side of the capture log is what `reindex` replays. */\nexport interface CaptureReader {\n  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;\n}\n\n/** These options change a capture log. */\nexport interface CaptureLogOptions {\n  /**\n   * When this value is true (the default), the log writes each batch of appends to the disk\n   * (`fdatasync`) before the appends finish. Thus a power cut does not lose a record that the\n   * crawler reported.\n   */\n  readonly sync?: boolean | undefined;\n}\n\ninterface Pending {\n  readonly path: string;\n  readonly line: string;\n  readonly resolve: () => void;\n  readonly reject: (error: unknown) => void;\n}\n\nconst MONTH_FILE = /^(\\d{2})(?:\\.[^.]+)?\\.ndjson$/;\nconst YEAR_DIR = /^\\d{4}$/;\nconst NEWLINE = 0x0a;\n\n/** This function writes the whole buffer. A write to a file can write less than the buffer, and the loop writes the rest. */\nconst writeAll = async (handle: FileHandle, bytes: Buffer): Promise<void> => {\n  let at = 0;\n  while (at < bytes.length) {\n    const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);\n    at += bytesWritten;\n  }\n};\n\n/**\n * This function reads a file line by line, as bytes, so a long file does not become one string. It\n * gives each line and tells if a line break ends it.\n */\nasync function* linesOf(\n  path: string,\n): AsyncGenerator<{ readonly text: string; readonly complete: boolean }> {\n  const decoder = new TextDecoder(\"utf-8\");\n  let rest: Buffer | undefined;\n  for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {\n    const data = rest === undefined ? chunk : Buffer.concat([rest, chunk]);\n    rest = undefined;\n    let start = 0;\n    for (;;) {\n      const end = data.indexOf(NEWLINE, start);\n      if (end === -1) break;\n      yield { text: decoder.decode(data.subarray(start, end)), complete: true };\n      start = end + 1;\n    }\n    if (start < data.length) rest = Buffer.from(data.subarray(start));\n  }\n  if (rest !== undefined && rest.length > 0) yield { text: decoder.decode(rest), complete: false };\n}\n\nconst monthKey = (year: string, file: string): string =>\n  `${year}-${MONTH_FILE.exec(file)?.[1] ?? \"00\"}`;\n\n/**\n * The capture log keeps each fetch decision as one line of JSON in\n * `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.\n *\n * These are the rules of the log:\n *\n * - The log appends a batch of records with one write, and it writes the batch to the disk before\n *   the appends finish.\n * - An append always starts on a new line. After a crash leaves a partial line, the next append\n *   starts after a line break. The partial line stays as a damaged line, and the record is good.\n * - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.\n * - The log does not remove or change a line.\n *\n * Two processes can append to one log, because each batch is one write to a file in append mode.\n */\nexport class CaptureLog implements CaptureReader {\n  private readonly handles = new Map<string, FileHandle>();\n  private readonly pending: Pending[] = [];\n  private draining: Promise<void> | undefined;\n  private readonly sync: boolean;\n\n  constructor(\n    readonly dir: string,\n    options: CaptureLogOptions = {},\n  ) {\n    this.sync = options.sync ?? true;\n  }\n\n  private pathOf(record: CaptureRecord): string {\n    return join(\n      this.dir,\n      record.source,\n      record.received_at.slice(0, 4),\n      `${record.received_at.slice(5, 7)}.ndjson`,\n    );\n  }\n\n  private async handleFor(path: string): Promise<FileHandle> {\n    const known = this.handles.get(path);\n    if (known !== undefined) return known;\n    // the folders and the files are for the owner only, where the system has modes\n    await mkdir(dirname(path), { recursive: true, mode: 0o700 });\n    const handle = await open(path, \"a+\", 0o600);\n    this.handles.set(path, handle);\n    return handle;\n  }\n\n  /** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */\n  private static async needsBreak(handle: FileHandle): Promise<boolean> {\n    const { size } = await handle.stat();\n    if (size === 0) return false;\n    const last = Buffer.alloc(1);\n    await handle.read(last, 0, 1, size - 1);\n    return last[0] !== NEWLINE;\n  }\n\n  append(record: CaptureRecord): Promise<void> {\n    let line: string;\n    try {\n      line = `${JSON.stringify(record)}\\n`;\n    } catch (error) {\n      return Promise.reject(error instanceof Error ? error : new Error(String(error)));\n    }\n    return new Promise<void>((resolve, reject) => {\n      this.pending.push({ path: this.pathOf(record), line, resolve, reject });\n      this.kick();\n    });\n  }\n\n  /** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */\n  private kick(): void {\n    if (this.draining !== undefined) return;\n    this.draining = this.drain().finally(() => {\n      this.draining = undefined;\n      if (this.pending.length > 0) this.kick();\n    });\n  }\n\n  /** This method writes the waiting records in batches: one write and one sync for each file. */\n  private async drain(): Promise<void> {\n    while (this.pending.length > 0) {\n      const batch = this.pending.splice(0);\n      const byPath = new Map<string, Pending[]>();\n      for (const item of batch) {\n        const list = byPath.get(item.path);\n        if (list === undefined) byPath.set(item.path, [item]);\n        else list.push(item);\n      }\n      for (const [path, items] of byPath) {\n        try {\n          const handle = await this.handleFor(path);\n          const lead = (await CaptureLog.needsBreak(handle)) ? \"\\n\" : \"\";\n          await writeAll(\n            handle,\n            Buffer.from(`${lead}${items.map((item) => item.line).join(\"\")}`, \"utf8\"),\n          );\n          if (this.sync) await handle.datasync();\n          for (const item of items) item.resolve();\n        } catch (error) {\n          // a handle that failed is opened again for the next append\n          const handle = this.handles.get(path);\n          this.handles.delete(path);\n          await handle?.close().catch(() => undefined);\n          for (const item of items) item.reject(error);\n        }\n      }\n    }\n  }\n\n  /** This method waits until each append in progress is on the disk. */\n  async flush(): Promise<void> {\n    while (this.draining !== undefined) await this.draining;\n  }\n\n  /** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */\n  private async filesOf(source: string, since: string | undefined): Promise<string[]> {\n    const sourceDir = join(this.dir, source);\n    if (!existsSync(sourceDir)) return [];\n    const from = since?.slice(0, 7);\n    const years = (await readdir(sourceDir, { withFileTypes: true }))\n      .filter((entry) => entry.isDirectory() && YEAR_DIR.test(entry.name))\n      .map((entry) => entry.name)\n      .sort();\n    const files: string[] = [];\n    for (const year of years) {\n      if (from !== undefined && year < from.slice(0, 4)) continue;\n      const months = (await readdir(join(sourceDir, year)))\n        .filter((name) => MONTH_FILE.test(name))\n        .sort();\n      for (const month of months) {\n        if (from !== undefined && monthKey(year, month) < from) continue;\n        files.push(join(sourceDir, year, month));\n      }\n    }\n    return files;\n  }\n\n  /** This method reads the records of one source in the sequence of the files. */\n  private async *readSource(\n    source: string,\n    filter: CaptureFilter | undefined,\n    options: ScanOptions | undefined,\n  ): AsyncGenerator<CaptureRecord> {\n    for (const path of await this.filesOf(source, filter?.since)) {\n      let number = 0;\n      for await (const { text, complete } of linesOf(path)) {\n        number += 1;\n        const trimmed = text.trim();\n        if (trimmed.length === 0) continue;\n        let row: CaptureRecord;\n        try {\n          row = JSON.parse(trimmed) as CaptureRecord;\n          if (typeof row !== \"object\" || row === null || typeof row.received_at !== \"string\") {\n            throw new Error(\"not a record\");\n          }\n        } catch {\n          // the append of a different process can be in progress at the end of the file\n          if (complete) options?.onDamaged?.({ path, line: number });\n          continue;\n        }\n        if (filter?.kind !== undefined && row.kind !== filter.kind) continue;\n        if (filter?.since !== undefined && row.received_at < filter.since) continue;\n        yield row;\n      }\n    }\n  }\n\n  /**\n   * This method reads the records that agree with the filter. It first waits for the appends that\n   * are in progress. It reads each file as a stream, so its memory does not grow with the log.\n   *\n   * The records of one source come in the sequence of the files. The records of two or more sources\n   * merge in the sequence of `received_at`.\n   */\n  async *scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord> {\n    await this.flush();\n    const sources =\n      filter?.source !== undefined\n        ? [filter.source]\n        : existsSync(this.dir)\n          ? (await readdir(this.dir, { withFileTypes: true }))\n              .filter((entry) => entry.isDirectory())\n              .map((entry) => entry.name)\n              .sort()\n          : [];\n    if (sources.length === 1) {\n      yield* this.readSource(sources[0]!, filter, options);\n      return;\n    }\n    // a merge of the sources: the next record is the earliest head\n    const streams = sources.map((source) =>\n      this.readSource(source, filter, options)[Symbol.asyncIterator](),\n    );\n    const heads: Array<CaptureRecord | undefined> = [];\n    for (const stream of streams)\n      heads.push((await stream.next()).value as CaptureRecord | undefined);\n    for (;;) {\n      let pick = -1;\n      for (const [index, head] of heads.entries()) {\n        if (head !== undefined && (pick === -1 || head.received_at < heads[pick]!.received_at)) {\n          pick = index;\n        }\n      }\n      if (pick === -1) return;\n      yield heads[pick]!;\n      heads[pick] = (await streams[pick]!.next()).value as CaptureRecord | undefined;\n    }\n  }\n\n  /** This method writes the appends in progress, then closes the files. A later append opens them again. */\n  async close(): Promise<void> {\n    await this.flush();\n    const handles = [...this.handles.values()];\n    this.handles.clear();\n    for (const handle of handles) await handle.close();\n  }\n}\n"],"mappings":";;;;;AAoBA,MAAM,OAAO;;AAGb,MAAM,6BAAkC,IAAI,IAAI;CAC9C;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,YAAY,MAAc,SAC9B,KAAK,MAAM,KAAK,MAAM,GAAG,CAAC,GAAG,KAAK,MAAM,GAAG,CAAC,GAAG,IAAI;AAErD,MAAM,UAAU,UAA8B,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK;AAE7F,MAAM,UAAU,UACd,OAAO,UAAU,YAAY,UAAU,QAAQ,UAAU,QAAQ,OAAO,MAAM,IAAI,IAAI;AAExF,MAAM,UAAU,SAA6C,KAAK,IAAI,CAAC,CAAC,YAAY,KAAA,CAAS;;AAG7F,MAAM,aAAa,OAAO,SAAgC;CACxD,IAAI,QAAQ,aAAa,SAAS;CAClC,IAAI;EACF,MAAM,SAAS,MAAM,KAAK,MAAM,GAAG;EACnC,IAAI;GACF,MAAM,OAAO,KAAK;EACpB,UAAU;GACR,MAAM,OAAO,MAAM;EACrB;CACF,QAAQ,CAER;AACF;;;;;;;;;;;;;;AAeA,IAAa,gBAAb,MAAa,cAAmC;CAQnC;CAPX,0BAA2B,IAAI,IAG7B;CACF;CAEA,YACE,KACA,UAAgC,CAAC,GACjC;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,IAAI,OAAkF;EACpF,MAAM,OAAO,OAAO,KAAK;EACzB,MAAM,UAAU,KAAK,QAAQ,IAAI,IAAI;EACrC,IAAI,YAAY,KAAA,GAAW,OAAO;EAClC,MAAM,OAAO,KAAK,MAAM,MAAM,KAAK,CAAC,CAAC,cAAc,KAAK,QAAQ,OAAO,IAAI,CAAC;EAC5E,KAAK,QAAQ,IAAI,MAAM,IAAI;EAC3B,OAAO;CACT;;CAGA,aAAqB,MAAM,MAAc,MAAgC;EACvE,IAAI;GACF,OAAO,OAAO,MAAM,SAAS,IAAI,CAAC,MAAM;EAC1C,QAAQ;GACN,OAAO;EACT;CACF;CAEA,MAAc,MACZ,MACA,OAC+D;EAC/D,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,MAAM,WAAW,MAAM,OAAO,MAAM;EACpC,IAAI,UAAU,OAAO,MAAM,QAAQ,SAAS,SAAS,MAAM,QACzD,OAAO;GAAE;GAAM,SAAS;EAAK;EAC/B,MAAM,SAAS,QAAQ,MAAM;EAC7B,MAAM,MAAM,QAAQ;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EACpD,MAAM,YAAY,GAAG,OAAO,GAAG,WAAW,EAAE;EAC5C,IAAI;GACF,MAAM,SAAS,MAAM,KAAK,WAAW,MAAM,GAAK;GAChD,IAAI;IACF,MAAM,OAAO,UAAU,KAAK;IAC5B,IAAI,KAAK,MAAM,MAAM,OAAO,KAAK;GACnC,UAAU;IACR,MAAM,OAAO,MAAM;GACrB;GACA,IAAI;IACF,MAAM,OAAO,WAAW,MAAM;GAChC,SAAS,OAAO;IAEd,IAAI,CAAC,WAAW,IAAI,OAAO,KAAK,CAAC,KAAK,CAAE,MAAM,cAAc,MAAM,QAAQ,IAAI,GAC5E,MAAM;IACR,OAAO;KAAE;KAAM,SAAS;IAAK;GAC/B;GACA,IAAI,KAAK,MAAM,MAAM,WAAW,MAAM;GACtC,OAAO;IAAE;IAAM,SAAS;GAAM;EAChC,UAAU;GACR,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;EAC5D;CACF;CAEA,MAAM,IAAI,MAA+C;EACvD,IAAI,CAAC,KAAK,KAAK,IAAI,GAAG,OAAO,KAAA;EAC7B,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,IAAI;EACJ,IAAI;GACF,QAAQ,MAAM,SAAS,MAAM;EAC/B,QAAQ;GACN;EACF;EACA,IAAI,OAAO,KAAK,MAAM,MAAM;GAE1B,MAAM,OAAO,QAAQ,GAAG,OAAO,GAAG,WAAW,EAAE,SAAS,CAAC,CAAC,YAAY,KAAA,CAAS;GAC/E;EACF;EACA,OAAO,IAAI,WAAW,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;CACxE;AACF;;;ACrGA,MAAM,aAAa;AACnB,MAAM,WAAW;AACjB,MAAM,UAAU;;AAGhB,MAAM,WAAW,OAAO,QAAoB,UAAiC;CAC3E,IAAI,KAAK;CACT,OAAO,KAAK,MAAM,QAAQ;EACxB,MAAM,EAAE,iBAAiB,MAAM,OAAO,MAAM,OAAO,IAAI,MAAM,SAAS,EAAE;EACxE,MAAM;CACR;AACF;;;;;AAMA,gBAAgB,QACd,MACuE;CACvE,MAAM,UAAU,IAAI,YAAY,OAAO;CACvC,IAAI;CACJ,WAAW,MAAM,SAAS,iBAAiB,IAAI,GAA4B;EACzE,MAAM,OAAO,SAAS,KAAA,IAAY,QAAQ,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;EACrE,OAAO,KAAA;EACP,IAAI,QAAQ;EACZ,SAAS;GACP,MAAM,MAAM,KAAK,QAAQ,SAAS,KAAK;GACvC,IAAI,QAAQ,IAAI;GAChB,MAAM;IAAE,MAAM,QAAQ,OAAO,KAAK,SAAS,OAAO,GAAG,CAAC;IAAG,UAAU;GAAK;GACxE,QAAQ,MAAM;EAChB;EACA,IAAI,QAAQ,KAAK,QAAQ,OAAO,OAAO,KAAK,KAAK,SAAS,KAAK,CAAC;CAClE;CACA,IAAI,SAAS,KAAA,KAAa,KAAK,SAAS,GAAG,MAAM;EAAE,MAAM,QAAQ,OAAO,IAAI;EAAG,UAAU;CAAM;AACjG;AAEA,MAAM,YAAY,MAAc,SAC9B,GAAG,KAAK,GAAG,WAAW,KAAK,IAAI,CAAC,GAAG,MAAM;;;;;;;;;;;;;;;;AAiB3C,IAAa,aAAb,MAAa,WAAoC;CAOpC;CANX,0BAA2B,IAAI,IAAwB;CACvD,UAAsC,CAAC;CACvC;CACA;CAEA,YACE,KACA,UAA6B,CAAC,GAC9B;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,OAAe,QAA+B;EAC5C,OAAO,KACL,KAAK,KACL,OAAO,QACP,OAAO,YAAY,MAAM,GAAG,CAAC,GAC7B,GAAG,OAAO,YAAY,MAAM,GAAG,CAAC,EAAE,QACpC;CACF;CAEA,MAAc,UAAU,MAAmC;EACzD,MAAM,QAAQ,KAAK,QAAQ,IAAI,IAAI;EACnC,IAAI,UAAU,KAAA,GAAW,OAAO;EAEhC,MAAM,MAAM,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAC3D,MAAM,SAAS,MAAM,KAAK,MAAM,MAAM,GAAK;EAC3C,KAAK,QAAQ,IAAI,MAAM,MAAM;EAC7B,OAAO;CACT;;CAGA,aAAqB,WAAW,QAAsC;EACpE,MAAM,EAAE,SAAS,MAAM,OAAO,KAAK;EACnC,IAAI,SAAS,GAAG,OAAO;EACvB,MAAM,OAAO,OAAO,MAAM,CAAC;EAC3B,MAAM,OAAO,KAAK,MAAM,GAAG,GAAG,OAAO,CAAC;EACtC,OAAO,KAAK,OAAO;CACrB;CAEA,OAAO,QAAsC;EAC3C,IAAI;EACJ,IAAI;GACF,OAAO,GAAG,KAAK,UAAU,MAAM,EAAE;EACnC,SAAS,OAAO;GACd,OAAO,QAAQ,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC,CAAC;EACjF;EACA,OAAO,IAAI,SAAe,SAAS,WAAW;GAC5C,KAAK,QAAQ,KAAK;IAAE,MAAM,KAAK,OAAO,MAAM;IAAG;IAAM;IAAS;GAAO,CAAC;GACtE,KAAK,KAAK;EACZ,CAAC;CACH;;CAGA,OAAqB;EACnB,IAAI,KAAK,aAAa,KAAA,GAAW;EACjC,KAAK,WAAW,KAAK,MAAM,CAAC,CAAC,cAAc;GACzC,KAAK,WAAW,KAAA;GAChB,IAAI,KAAK,QAAQ,SAAS,GAAG,KAAK,KAAK;EACzC,CAAC;CACH;;CAGA,MAAc,QAAuB;EACnC,OAAO,KAAK,QAAQ,SAAS,GAAG;GAC9B,MAAM,QAAQ,KAAK,QAAQ,OAAO,CAAC;GACnC,MAAM,yBAAS,IAAI,IAAuB;GAC1C,KAAK,MAAM,QAAQ,OAAO;IACxB,MAAM,OAAO,OAAO,IAAI,KAAK,IAAI;IACjC,IAAI,SAAS,KAAA,GAAW,OAAO,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;SAC/C,KAAK,KAAK,IAAI;GACrB;GACA,KAAK,MAAM,CAAC,MAAM,UAAU,QAC1B,IAAI;IACF,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;IACxC,MAAM,OAAQ,MAAM,WAAW,WAAW,MAAM,IAAK,OAAO;IAC5D,MAAM,SACJ,QACA,OAAO,KAAK,GAAG,OAAO,MAAM,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,MAAM,CACzE;IACA,IAAI,KAAK,MAAM,MAAM,OAAO,SAAS;IACrC,KAAK,MAAM,QAAQ,OAAO,KAAK,QAAQ;GACzC,SAAS,OAAO;IAEd,MAAM,SAAS,KAAK,QAAQ,IAAI,IAAI;IACpC,KAAK,QAAQ,OAAO,IAAI;IACxB,MAAM,QAAQ,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;IAC3C,KAAK,MAAM,QAAQ,OAAO,KAAK,OAAO,KAAK;GAC7C;EAEJ;CACF;;CAGA,MAAM,QAAuB;EAC3B,OAAO,KAAK,aAAa,KAAA,GAAW,MAAM,KAAK;CACjD;;CAGA,MAAc,QAAQ,QAAgB,OAA8C;EAClF,MAAM,YAAY,KAAK,KAAK,KAAK,MAAM;EACvC,IAAI,CAAC,WAAW,SAAS,GAAG,OAAO,CAAC;EACpC,MAAM,OAAO,OAAO,MAAM,GAAG,CAAC;EAC9B,MAAM,SAAS,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC,EAAA,CAC5D,QAAQ,UAAU,MAAM,YAAY,KAAK,SAAS,KAAK,MAAM,IAAI,CAAC,CAAC,CACnE,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK;EACR,MAAM,QAAkB,CAAC;EACzB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,SAAS,KAAA,KAAa,OAAO,KAAK,MAAM,GAAG,CAAC,GAAG;GACnD,MAAM,UAAU,MAAM,QAAQ,KAAK,WAAW,IAAI,CAAC,EAAA,CAChD,QAAQ,SAAS,WAAW,KAAK,IAAI,CAAC,CAAC,CACvC,KAAK;GACR,KAAK,MAAM,SAAS,QAAQ;IAC1B,IAAI,SAAS,KAAA,KAAa,SAAS,MAAM,KAAK,IAAI,MAAM;IACxD,MAAM,KAAK,KAAK,WAAW,MAAM,KAAK,CAAC;GACzC;EACF;EACA,OAAO;CACT;;CAGA,OAAe,WACb,QACA,QACA,SAC+B;EAC/B,KAAK,MAAM,QAAQ,MAAM,KAAK,QAAQ,QAAQ,QAAQ,KAAK,GAAG;GAC5D,IAAI,SAAS;GACb,WAAW,MAAM,EAAE,MAAM,cAAc,QAAQ,IAAI,GAAG;IACpD,UAAU;IACV,MAAM,UAAU,KAAK,KAAK;IAC1B,IAAI,QAAQ,WAAW,GAAG;IAC1B,IAAI;IACJ,IAAI;KACF,MAAM,KAAK,MAAM,OAAO;KACxB,IAAI,OAAO,QAAQ,YAAY,QAAQ,QAAQ,OAAO,IAAI,gBAAgB,UACxE,MAAM,IAAI,MAAM,cAAc;IAElC,QAAQ;KAEN,IAAI,UAAU,SAAS,YAAY;MAAE;MAAM,MAAM;KAAO,CAAC;KACzD;IACF;IACA,IAAI,QAAQ,SAAS,KAAA,KAAa,IAAI,SAAS,OAAO,MAAM;IAC5D,IAAI,QAAQ,UAAU,KAAA,KAAa,IAAI,cAAc,OAAO,OAAO;IACnE,MAAM;GACR;EACF;CACF;;;;;;;;CASA,OAAO,KAAK,QAAwB,SAAqD;EACvF,MAAM,KAAK,MAAM;EACjB,MAAM,UACJ,QAAQ,WAAW,KAAA,IACf,CAAC,OAAO,MAAM,IACd,WAAW,KAAK,GAAG,KAChB,MAAM,QAAQ,KAAK,KAAK,EAAE,eAAe,KAAK,CAAC,EAAA,CAC7C,QAAQ,UAAU,MAAM,YAAY,CAAC,CAAC,CACtC,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK,IACR,CAAC;EACT,IAAI,QAAQ,WAAW,GAAG;GACxB,OAAO,KAAK,WAAW,QAAQ,IAAK,QAAQ,OAAO;GACnD;EACF;EAEA,MAAM,UAAU,QAAQ,KAAK,WAC3B,KAAK,WAAW,QAAQ,QAAQ,OAAO,CAAC,CAAC,OAAO,cAAc,CAAC,CACjE;EACA,MAAM,QAA0C,CAAC;EACjD,KAAK,MAAM,UAAU,SACnB,MAAM,MAAM,MAAM,OAAO,KAAK,EAAA,CAAG,KAAkC;EACrE,SAAS;GACP,IAAI,OAAO;GACX,KAAK,MAAM,CAAC,OAAO,SAAS,MAAM,QAAQ,GACxC,IAAI,SAAS,KAAA,MAAc,SAAS,MAAM,KAAK,cAAc,MAAM,KAAK,CAAE,cACxE,OAAO;GAGX,IAAI,SAAS,IAAI;GACjB,MAAM,MAAM;GACZ,MAAM,SAAS,MAAM,QAAQ,KAAK,CAAE,KAAK,EAAA,CAAG;EAC9C;CACF;;CAGA,MAAM,QAAuB;EAC3B,MAAM,KAAK,MAAM;EACjB,MAAM,UAAU,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC;EACzC,KAAK,QAAQ,MAAM;EACnB,KAAK,MAAM,UAAU,SAAS,MAAM,OAAO,MAAM;CACnD;AACF"}

package/build/mcp.d.mts

@@ -1,4 +1,4 @@
-import { a as PoliteCrawler, i as OpenOptions } from "./open-CZqwY7GH.mjs";
+import { a as PoliteCrawler, r as OpenOptions } from "./open-Ca1lcWR3.mjs";
 import { McpServer, McpServer as McpServer$1 } from "@modelcontextprotocol/sdk/server/mcp.js";
 //#region src/mcp.d.ts
 /** These options change the tools of the MCP server. */
@@ -23,9 +23,12 @@ export interface StartMcpOptions extends OpenOptions, CrawlServerOptions {
   readonly log?: ((line: string) => void) | undefined;
 }
 /**
- * This function makes an MCP server with four tools over one crawler: `crawl_fetch`,
- * `crawl_find`, `crawl_read` and `crawl_verify`. The server does not connect to a transport. Each
- * fetch applies the robots.txt rules for `Claude-User` too, unless `onBehalfOf` gives other tokens.
+ * This function makes an MCP server with nine tools over one crawler: `crawl_fetch`, `crawl_find`,
+ * `crawl_read`, `crawl_verify`, `crawl_extract`, `crawl_discover`, `crawl_site`, `crawl_diff` and
+ * `crawl_watch_list`.
+ * The server does not connect to a transport. Each fetch applies the robots.txt rules for
+ * `Claude-User` too, unless `onBehalfOf` gives other tokens. No tool adds a watch or an export,
+ * because they are actions of the operator.
  */
 export declare const createMcpServer: (crawler: PoliteCrawler, options?: CrawlServerOptions) => McpServer$1;
 /**