polite-crawler 0.1.4 → 0.2.0

package/build/index.mjs

@@ -1,21 +1,21 @@
-import { n as FileBlobStore, t as CaptureLog } from "./log-Dx-wXjJl.mjs";
-import { n as RenderRefused, r as playwrightRenderer, t as NO_PLAYWRIGHT } from "./render-C1CUraWF.mjs";
-import { r as checkedRenderer } from "./browser-CKRNbIDT.mjs";
-import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-BPem-XZd.mjs";
-import { B as egressLookup, F as renderedKey, I as makeRecord, M as Crawler, N as DEADLINE_NOTE, O as locate, P as DEFAULT_SECRET_PARAMS, T as readPdf, _ as versionAtTime, a as fetchPage, b as archivedDocId, c as isHeldBack, d as pageExtractor, f as readDocument, g as verifyQuote, h as renderAndIndex, i as fetchAndIndex, j as CAPTURE_SCHEMA, l as isoTime, m as reindex, n as HELD_BACK, o as findDocuments, p as recordOverride, r as catchUpIndex, s as heldBackMeta, t as Extractors, u as pageDocId, v as archiveExtractor, w as isPdf, z as EgressRefused } from "./runtime-D_r2SYge.mjs";
+import { n as FileBlobStore, t as CaptureLog } from "./log-CmBWcyQm.mjs";
+import { n as RenderRefused, r as playwrightRenderer, t as NO_PLAYWRIGHT } from "./render-CKE2LILH.mjs";
+import { t as browserMissing } from "./browser-56dBBLec.mjs";
+import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-Djva7JkN.mjs";
+import { A as DEADLINE_NOTE, F as EgressRefused, I as egressLookup, M as makeRecord, O as CAPTURE_SCHEMA, S as readPdf, T as locate, _ as archivedDocId, a as fetchPage, c as isHeldBack, d as recordOverride, f as reindex, h as archiveExtractor, i as fetchAndIndex, j as DEFAULT_SECRET_PARAMS, k as Crawler, l as isoTime, m as versionAtTime, n as HELD_BACK, o as findDocuments, p as verifyQuote, r as catchUpIndex, s as heldBackMeta, t as Extractors, u as readDocument, x as isPdf } from "./runtime-C_2BQibH.mjs";
 import { DEFAULT_POLICY, MemoryHostStateStore, VALIDATOR_MEMO, nextValidatorMemo, realClock } from "./politeness.mjs";
 import { ALLOW_ALL, decide, parseRobots } from "./robots.mjs";
-import { i as keyDirectory, n as KEY_DIRECTORY_TYPE, o as verifySignature, r as createWebBotAuthSigner, t as KEY_DIRECTORY_PATH } from "./webbotauth-CjJdTAfE.mjs";
-import { A as NO_EMBEDDER, C as DISCOVER_LIMITS, D as readSitemap, E as feedDate, O as DocIndex, S as crawlSite, T as SITEMAP_LIMITS, _ as estimateTokens, b as SqliteHostStateStore, c as WatchStore, d as VectorStore, f as createHybridSearch, g as chunkTokensOf, h as chunkText, i as openCrawler, k as FAST_RANK_SHARE, l as runWatch, m as HYBRID_DEFAULTS, n as defaultUserAgent, o as WATCH_LIMITS, p as CHUNK_LIMITS, r as isMailbox, s as WATCH_SCHEMA, t as createRuntime, u as VECTORS_SCHEMA, v as fuseRanks, w as discoverSite, x as CRAWL_LIMITS, y as unitVector } from "./open-CbpSCEZc.mjs";
+import { i as keyDirectory, n as KEY_DIRECTORY_TYPE, o as verifySignature, r as createWebBotAuthSigner, t as KEY_DIRECTORY_PATH } from "./webbotauth-Dq_dA_u3.mjs";
+import { A as NO_EMBEDDER, C as DISCOVER_LIMITS, D as readSitemap, E as feedDate, O as DocIndex, S as crawlSite, T as SITEMAP_LIMITS, _ as estimateTokens, b as SqliteHostStateStore, c as WatchStore, d as VectorStore, f as createHybridSearch, g as chunkTokensOf, h as chunkText, i as openCrawler, k as FAST_RANK_SHARE, l as runWatch, m as HYBRID_DEFAULTS, n as defaultUserAgent, o as WATCH_LIMITS, p as CHUNK_LIMITS, r as isMailbox, s as WATCH_SCHEMA, t as createRuntime, u as VECTORS_SCHEMA, v as fuseRanks, w as discoverSite, x as CRAWL_LIMITS, y as unitVector } from "./open-CEo22Hpu.mjs";
 import { a as bodyEncoding, o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
-import { n as diffDocument, r as diffTexts, t as DIFF_LIMITS } from "./diff-BUM91Rev.mjs";
-import { n as checkFields, r as extractDocument, t as FIELD_KINDS } from "./extract-CefB5Vfn.mjs";
-import { a as readJson, i as defineRecipe, n as RESERVED_RECIPES, o as registerRecipe, s as runRecipe, t as RECIPE_LIMITS } from "./recipe-GiFQojmo.mjs";
-import { c as startDaemon, i as daemonStateFile, n as DAEMON_STATE_SCHEMA, o as readDaemonState, r as changeCommand, s as scheduleWatches, t as DAEMON_DEFAULTS } from "./daemon-DK6U51_v.mjs";
-import { i as ollamaEmbedder, n as OLLAMA_MODELS, r as isLoopback, t as NO_OLLAMA } from "./ollama-BJxkjny4.mjs";
-import { n as TRANSFORMERS_MODELS, r as transformersEmbedder, t as NO_TRANSFORMERS } from "./transformers-BQMlaGj4.mjs";
+import { a as formatDiff, n as diffDocument, r as diffTexts, t as DIFF_LIMITS } from "./diff-CpKN0Jdc.mjs";
+import { n as checkFields, r as extractDocument, t as FIELD_KINDS } from "./extract-DqeqiNHb.mjs";
+import { a as readJson, i as defineRecipe, n as RESERVED_RECIPES, o as registerRecipe, s as runRecipe, t as RECIPE_LIMITS } from "./recipe-B2SkepDZ.mjs";
+import { c as startDaemon, i as daemonStateFile, n as DAEMON_STATE_SCHEMA, o as readDaemonState, r as changeCommand, s as scheduleWatches, t as DAEMON_DEFAULTS } from "./daemon-BXBbUrni.mjs";
+import { i as ollamaEmbedder, n as OLLAMA_MODELS, r as isLoopback, t as NO_OLLAMA } from "./ollama-D1xRh3hY.mjs";
+import { n as TRANSFORMERS_MODELS, r as transformersEmbedder, t as NO_TRANSFORMERS } from "./transformers-BNiuiLxT.mjs";
 import { DAYS, compareHours, describeWeek, disagreements, fromCompact, fromSchemaOrg, fromText, toHoursWeek } from "./hours.mjs";
-import { a as readAmount, c as locationMatches, d as readVenue, f as venueReadingsOf, i as productOf, l as namesVenue, n as pickProduct, o as readProduct, r as pickVariant, s as VENUE_LIMITS, t as PRODUCT_LIMITS, u as rankHoursLinks } from "./recipe-product-Dellqudu.mjs";
-import { t as exportWacz } from "./wacz-QyzLp4mC.mjs";
-import { n as exportWarc } from "./warc-BHcaRKHw.mjs";
-export { ALLOW_ALL, BOT_PAGE, CAPTURE_SCHEMA, CHUNK_LIMITS, CRAWL_LIMITS, CaptureLog, Crawler, DAEMON_DEFAULTS, DAEMON_STATE_SCHEMA, DAYS, DEADLINE_NOTE, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, DEFAULT_TOKEN, DIFF_LIMITS, DISCOVER_LIMITS, DocIndex, EgressRefused, Extractors, FAST_RANK_SHARE, FIELD_KINDS, FileBlobStore, HELD_BACK, HYBRID_DEFAULTS, KEY_DIRECTORY_PATH, KEY_DIRECTORY_TYPE, MemoryHostStateStore, NO_EMBEDDER, NO_OLLAMA, NO_PLAYWRIGHT, NO_TRANSFORMERS, OLLAMA_MODELS, PRODUCT_LIMITS, RECIPE_LIMITS, RESERVED_RECIPES, RenderRefused, SITEMAP_LIMITS, SqliteHostStateStore, TRANSFORMERS_MODELS, VALIDATOR_MEMO, VECTORS_SCHEMA, VENUE_LIMITS, VERSION, VectorStore, WATCH_LIMITS, WATCH_SCHEMA, WatchStore, archiveExtractor, archivedDocId, bodyEncoding, catchUpIndex, changeCommand, checkFields, checkedRenderer, chunkText, chunkTokensOf, compareHours, crawlSite, createHybridSearch, createRuntime, createWebBotAuthSigner, daemonStateFile, decide, decodeBody, defaultUserAgent, defineRecipe, describeWeek, diffDocument, diffTexts, disagreements, discoverSite, durationMs, egressLookup, estimateTokens, exportWacz, exportWarc, extractDocument, extractPage, feedDate, fetchAndIndex, fetchPage, findDocuments, fromCompact, fromSchemaOrg, fromText, fuseRanks, heldBackMeta, isHeldBack, isLoopback, isMailbox, isPdf, isoTime, keyDirectory, locate, locationMatches, makeRecord, namesVenue, nextValidatorMemo, ollamaEmbedder, openCrawler, pageDocId, pageExtractor, parseRobots, pickProduct, pickVariant, playwrightRenderer, productOf, rankHoursLinks, readAmount, readDaemonState, readDocument, readJson, readPdf, readProduct, readSitemap, readVenue, realClock, recordOverride, registerRecipe, reindex, renderAndIndex, renderedKey, runRecipe, runWatch, scheduleWatches, startDaemon, toHoursWeek, transformersEmbedder, typesOf, unitVector, venueReadingsOf, verifyQuote, verifySignature, versionAtTime };
+import { a as readAmount, c as locationMatches, d as readVenue, f as venueReadingsOf, i as productOf, l as namesVenue, n as pickProduct, o as readProduct, r as pickVariant, s as VENUE_LIMITS, t as PRODUCT_LIMITS, u as rankHoursLinks } from "./recipe-product-CyZE3Ukd.mjs";
+import { t as exportWacz } from "./wacz-DKk2HBmx.mjs";
+import { n as exportWarc } from "./warc-C_ew76bV.mjs";
+export { ALLOW_ALL, BOT_PAGE, CAPTURE_SCHEMA, CHUNK_LIMITS, CRAWL_LIMITS, CaptureLog, Crawler, DAEMON_DEFAULTS, DAEMON_STATE_SCHEMA, DAYS, DEADLINE_NOTE, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, DEFAULT_TOKEN, DIFF_LIMITS, DISCOVER_LIMITS, DocIndex, EgressRefused, Extractors, FAST_RANK_SHARE, FIELD_KINDS, FileBlobStore, HELD_BACK, HYBRID_DEFAULTS, KEY_DIRECTORY_PATH, KEY_DIRECTORY_TYPE, MemoryHostStateStore, NO_EMBEDDER, NO_OLLAMA, NO_PLAYWRIGHT, NO_TRANSFORMERS, OLLAMA_MODELS, PRODUCT_LIMITS, RECIPE_LIMITS, RESERVED_RECIPES, RenderRefused, SITEMAP_LIMITS, SqliteHostStateStore, TRANSFORMERS_MODELS, VALIDATOR_MEMO, VECTORS_SCHEMA, VENUE_LIMITS, VERSION, VectorStore, WATCH_LIMITS, WATCH_SCHEMA, WatchStore, archiveExtractor, archivedDocId, bodyEncoding, browserMissing, catchUpIndex, changeCommand, checkFields, chunkText, chunkTokensOf, compareHours, crawlSite, createHybridSearch, createRuntime, createWebBotAuthSigner, daemonStateFile, decide, decodeBody, defaultUserAgent, defineRecipe, describeWeek, diffDocument, diffTexts, disagreements, discoverSite, durationMs, egressLookup, estimateTokens, exportWacz, exportWarc, extractDocument, extractPage, feedDate, fetchAndIndex, fetchPage, findDocuments, formatDiff, fromCompact, fromSchemaOrg, fromText, fuseRanks, heldBackMeta, isHeldBack, isLoopback, isMailbox, isPdf, isoTime, keyDirectory, locate, locationMatches, makeRecord, namesVenue, nextValidatorMemo, ollamaEmbedder, openCrawler, parseRobots, pickProduct, pickVariant, playwrightRenderer, productOf, rankHoursLinks, readAmount, readDaemonState, readDocument, readJson, readPdf, readProduct, readSitemap, readVenue, realClock, recordOverride, registerRecipe, reindex, runRecipe, runWatch, scheduleWatches, startDaemon, toHoursWeek, transformersEmbedder, typesOf, unitVector, venueReadingsOf, verifyQuote, verifySignature, versionAtTime };

package/build/{log-Dx-wXjJl.mjs → log-CmBWcyQm.mjs}

@@ -40,6 +40,8 @@ const syncFolder = async (path) => {
 *   other was first, the put succeeds if the stored file has the same hash.
 * - The method `get` examines the hash of the bytes. A damaged file gets a new name
 *   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.
+*
+* @beta
 */
 var FileBlobStore = class FileBlobStore {
 	dir;
@@ -175,6 +177,8 @@ const monthKey = (year, file) => `${year}-${MONTH_FILE.exec(file)?.[1] ?? "00"}`
 * - The log does not remove or change a line.
 *
 * Two processes can append to one log, because each batch is one write to a file in append mode.
+*
+* @beta
 */
 var CaptureLog = class CaptureLog {
 	dir;
@@ -339,4 +343,4 @@ var CaptureLog = class CaptureLog {
 //#endregion
 export { FileBlobStore as n, CaptureLog as t };
 
-//# sourceMappingURL=log-Dx-wXjJl.mjs.map
+//# sourceMappingURL=log-CmBWcyQm.mjs.map

package/build/log-CmBWcyQm.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"log-CmBWcyQm.mjs","names":[],"sources":["../src/blobs.ts","../src/log.ts"],"sourcesContent":["import { createHash, randomUUID } from \"node:crypto\";\nimport type { Stats } from \"node:fs\";\nimport { mkdir, open, readFile, rename, rm, stat } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\n\n/**\n * A blob store keeps the bytes of each response, and the hash of the bytes is the key. The crawler\n * needs only these two operations, so an application can give its own store.\n */\nexport interface BlobStore {\n  put(bytes: Uint8Array): Promise<{ readonly hash: string }>;\n  get(hash: string): Promise<Uint8Array | undefined>;\n}\n\n/**\n * These options change a file blob store.\n *\n * @beta\n */\nexport interface FileBlobStoreOptions {\n  /** When this value is true (the default), a blob is on the disk (`fsync`) before `put` finishes. */\n  readonly sync?: boolean | undefined;\n}\n\nconst HASH = /^[0-9a-f]{64}$/;\n\n/** A rename can fail with these codes when a different writer puts the same blob at the same time. */\nconst RACE_CODES: ReadonlySet<string> = new Set([\n  \"EPERM\",\n  \"EEXIST\",\n  \"EBUSY\",\n  \"EACCES\",\n  \"ENOTEMPTY\",\n]);\n\nconst blobPath = (root: string, hash: string): string =>\n  join(root, hash.slice(0, 2), hash.slice(2, 4), hash);\n\nconst hashOf = (bytes: Uint8Array): string => createHash(\"sha256\").update(bytes).digest(\"hex\");\n\nconst codeOf = (error: unknown): string =>\n  typeof error === \"object\" && error !== null && \"code\" in error ? String(error.code) : \"\";\n\nconst statOf = (path: string): Promise<Stats | undefined> => stat(path).catch(() => undefined);\n\n/** This function writes a folder to the disk, so a rename in it stays after a power cut. Windows cannot open a folder, so it skips this step. */\nconst syncFolder = async (path: string): Promise<void> => {\n  if (process.platform === \"win32\") return;\n  try {\n    const handle = await open(path, \"r\");\n    try {\n      await handle.sync();\n    } finally {\n      await handle.close();\n    }\n  } catch {\n    // a file system that cannot sync a folder keeps the blob in any case\n  }\n};\n\n/**\n * This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash\n * of the bytes. These are its rules:\n *\n * - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is\n *   written again.\n * - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.\n *   A crash thus cannot leave half a blob under its name.\n * - Two writers can put the same bytes at the same time. When the rename of one fails because the\n *   other was first, the put succeeds if the stored file has the same hash.\n * - The method `get` examines the hash of the bytes. A damaged file gets a new name\n *   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.\n *\n * @beta\n */\nexport class FileBlobStore implements BlobStore {\n  private readonly writing = new Map<\n    string,\n    Promise<{ readonly hash: string; readonly existed: boolean }>\n  >();\n  private readonly sync: boolean;\n\n  constructor(\n    readonly dir: string,\n    options: FileBlobStoreOptions = {},\n  ) {\n    this.sync = options.sync ?? true;\n  }\n\n  put(bytes: Uint8Array): Promise<{ readonly hash: string; readonly existed: boolean }> {\n    const hash = hashOf(bytes);\n    const running = this.writing.get(hash);\n    if (running !== undefined) return running;\n    const work = this.write(hash, bytes).finally(() => this.writing.delete(hash));\n    this.writing.set(hash, work);\n    return work;\n  }\n\n  /** This method tells if a file holds exactly the bytes of a hash. */\n  private static async holds(path: string, hash: string): Promise<boolean> {\n    try {\n      return hashOf(await readFile(path)) === hash;\n    } catch {\n      return false;\n    }\n  }\n\n  private async write(\n    hash: string,\n    bytes: Uint8Array,\n  ): Promise<{ readonly hash: string; readonly existed: boolean }> {\n    const target = blobPath(this.dir, hash);\n    const existing = await statOf(target);\n    if (existing?.isFile() === true && existing.size === bytes.length)\n      return { hash, existed: true };\n    const folder = dirname(target);\n    await mkdir(folder, { recursive: true, mode: 0o700 });\n    const temporary = `${target}.${randomUUID()}.tmp`;\n    try {\n      const handle = await open(temporary, \"wx\", 0o600);\n      try {\n        await handle.writeFile(bytes);\n        if (this.sync) await handle.sync();\n      } finally {\n        await handle.close();\n      }\n      try {\n        await rename(temporary, target);\n      } catch (error) {\n        // a different writer put the same bytes first, and Windows does not replace a file in use\n        if (!RACE_CODES.has(codeOf(error)) || !(await FileBlobStore.holds(target, hash)))\n          throw error;\n        return { hash, existed: true };\n      }\n      if (this.sync) await syncFolder(folder);\n      return { hash, existed: false };\n    } finally {\n      await rm(temporary, { force: true }).catch(() => undefined);\n    }\n  }\n\n  async get(hash: string): Promise<Uint8Array | undefined> {\n    if (!HASH.test(hash)) return undefined;\n    const target = blobPath(this.dir, hash);\n    let bytes: Buffer;\n    try {\n      bytes = await readFile(target);\n    } catch {\n      return undefined;\n    }\n    if (hashOf(bytes) !== hash) {\n      // a torn or damaged file is never the body: it moves away, and the next put writes it again\n      await rename(target, `${target}.${randomUUID()}.damaged`).catch(() => undefined);\n      return undefined;\n    }\n    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n  }\n}\n","import { createReadStream, existsSync } from \"node:fs\";\nimport type { FileHandle } from \"node:fs/promises\";\nimport { mkdir, open, readdir } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\nimport type { CaptureRecord } from \"./record.ts\";\n\n/** A scan filter selects the records of one source, one kind, or the records after one time. */\nexport interface CaptureFilter {\n  readonly source?: string | undefined;\n  readonly kind?: string | undefined;\n  /** This value is an ISO 8601 time. The scan gives the records received at this time or later. */\n  readonly since?: string | undefined;\n}\n\n/** A damaged line is a line of the log that is not a record: a torn append, or bytes that a crash left. */\nexport interface DamagedLine {\n  /** This value is the path of the file. */\n  readonly path: string;\n  /** This value is the number of the line in the file, from 1. */\n  readonly line: number;\n}\n\n/** These options change a scan. */\nexport interface ScanOptions {\n  /** The scan skips a damaged line, and it gives the line to this function. */\n  readonly onDamaged?: ((damaged: DamagedLine) => void) | undefined;\n}\n\n/** The read side of the capture log is what `reindex` replays. */\nexport interface CaptureReader {\n  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;\n}\n\n/**\n * These options change a capture log.\n *\n * @beta\n */\nexport interface CaptureLogOptions {\n  /**\n   * When this value is true (the default), the log writes each batch of appends to the disk\n   * (`fdatasync`) before the appends finish. Thus a power cut does not lose a record that the\n   * crawler reported.\n   */\n  readonly sync?: boolean | undefined;\n}\n\ninterface Pending {\n  readonly path: string;\n  readonly line: string;\n  readonly resolve: () => void;\n  readonly reject: (error: unknown) => void;\n}\n\nconst MONTH_FILE = /^(\\d{2})(?:\\.[^.]+)?\\.ndjson$/;\nconst YEAR_DIR = /^\\d{4}$/;\nconst NEWLINE = 0x0a;\n\n/** This function writes the whole buffer. A write to a file can write less than the buffer, and the loop writes the rest. */\nconst writeAll = async (handle: FileHandle, bytes: Buffer): Promise<void> => {\n  let at = 0;\n  while (at < bytes.length) {\n    const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);\n    at += bytesWritten;\n  }\n};\n\n/**\n * This function reads a file line by line, as bytes, so a long file does not become one string. It\n * gives each line and tells if a line break ends it.\n */\nasync function* linesOf(\n  path: string,\n): AsyncGenerator<{ readonly text: string; readonly complete: boolean }> {\n  const decoder = new TextDecoder(\"utf-8\");\n  let rest: Buffer | undefined;\n  for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {\n    const data = rest === undefined ? chunk : Buffer.concat([rest, chunk]);\n    rest = undefined;\n    let start = 0;\n    for (;;) {\n      const end = data.indexOf(NEWLINE, start);\n      if (end === -1) break;\n      yield { text: decoder.decode(data.subarray(start, end)), complete: true };\n      start = end + 1;\n    }\n    if (start < data.length) rest = Buffer.from(data.subarray(start));\n  }\n  if (rest !== undefined && rest.length > 0) yield { text: decoder.decode(rest), complete: false };\n}\n\nconst monthKey = (year: string, file: string): string =>\n  `${year}-${MONTH_FILE.exec(file)?.[1] ?? \"00\"}`;\n\n/**\n * The capture log keeps each fetch decision as one line of JSON in\n * `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.\n *\n * These are the rules of the log:\n *\n * - The log appends a batch of records with one write, and it writes the batch to the disk before\n *   the appends finish.\n * - An append always starts on a new line. After a crash leaves a partial line, the next append\n *   starts after a line break. The partial line stays as a damaged line, and the record is good.\n * - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.\n * - The log does not remove or change a line.\n *\n * Two processes can append to one log, because each batch is one write to a file in append mode.\n *\n * @beta\n */\nexport class CaptureLog implements CaptureReader {\n  private readonly handles = new Map<string, FileHandle>();\n  private readonly pending: Pending[] = [];\n  private draining: Promise<void> | undefined;\n  private readonly sync: boolean;\n\n  constructor(\n    readonly dir: string,\n    options: CaptureLogOptions = {},\n  ) {\n    this.sync = options.sync ?? true;\n  }\n\n  private pathOf(record: CaptureRecord): string {\n    return join(\n      this.dir,\n      record.source,\n      record.received_at.slice(0, 4),\n      `${record.received_at.slice(5, 7)}.ndjson`,\n    );\n  }\n\n  private async handleFor(path: string): Promise<FileHandle> {\n    const known = this.handles.get(path);\n    if (known !== undefined) return known;\n    // the folders and the files are for the owner only, where the system has modes\n    await mkdir(dirname(path), { recursive: true, mode: 0o700 });\n    const handle = await open(path, \"a+\", 0o600);\n    this.handles.set(path, handle);\n    return handle;\n  }\n\n  /** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */\n  private static async needsBreak(handle: FileHandle): Promise<boolean> {\n    const { size } = await handle.stat();\n    if (size === 0) return false;\n    const last = Buffer.alloc(1);\n    await handle.read(last, 0, 1, size - 1);\n    return last[0] !== NEWLINE;\n  }\n\n  append(record: CaptureRecord): Promise<void> {\n    let line: string;\n    try {\n      line = `${JSON.stringify(record)}\\n`;\n    } catch (error) {\n      return Promise.reject(error instanceof Error ? error : new Error(String(error)));\n    }\n    return new Promise<void>((resolve, reject) => {\n      this.pending.push({ path: this.pathOf(record), line, resolve, reject });\n      this.kick();\n    });\n  }\n\n  /** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */\n  private kick(): void {\n    if (this.draining !== undefined) return;\n    this.draining = this.drain().finally(() => {\n      this.draining = undefined;\n      if (this.pending.length > 0) this.kick();\n    });\n  }\n\n  /** This method writes the waiting records in batches: one write and one sync for each file. */\n  private async drain(): Promise<void> {\n    while (this.pending.length > 0) {\n      const batch = this.pending.splice(0);\n      const byPath = new Map<string, Pending[]>();\n      for (const item of batch) {\n        const list = byPath.get(item.path);\n        if (list === undefined) byPath.set(item.path, [item]);\n        else list.push(item);\n      }\n      for (const [path, items] of byPath) {\n        try {\n          const handle = await this.handleFor(path);\n          const lead = (await CaptureLog.needsBreak(handle)) ? \"\\n\" : \"\";\n          await writeAll(\n            handle,\n            Buffer.from(`${lead}${items.map((item) => item.line).join(\"\")}`, \"utf8\"),\n          );\n          if (this.sync) await handle.datasync();\n          for (const item of items) item.resolve();\n        } catch (error) {\n          // a handle that failed is opened again for the next append\n          const handle = this.handles.get(path);\n          this.handles.delete(path);\n          await handle?.close().catch(() => undefined);\n          for (const item of items) item.reject(error);\n        }\n      }\n    }\n  }\n\n  /** This method waits until each append in progress is on the disk. */\n  async flush(): Promise<void> {\n    while (this.draining !== undefined) await this.draining;\n  }\n\n  /** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */\n  private async filesOf(source: string, since: string | undefined): Promise<string[]> {\n    const sourceDir = join(this.dir, source);\n    if (!existsSync(sourceDir)) return [];\n    const from = since?.slice(0, 7);\n    const years = (await readdir(sourceDir, { withFileTypes: true }))\n      .filter((entry) => entry.isDirectory() && YEAR_DIR.test(entry.name))\n      .map((entry) => entry.name)\n      .sort();\n    const files: string[] = [];\n    for (const year of years) {\n      if (from !== undefined && year < from.slice(0, 4)) continue;\n      const months = (await readdir(join(sourceDir, year)))\n        .filter((name) => MONTH_FILE.test(name))\n        .sort();\n      for (const month of months) {\n        if (from !== undefined && monthKey(year, month) < from) continue;\n        files.push(join(sourceDir, year, month));\n      }\n    }\n    return files;\n  }\n\n  /** This method reads the records of one source in the sequence of the files. */\n  private async *readSource(\n    source: string,\n    filter: CaptureFilter | undefined,\n    options: ScanOptions | undefined,\n  ): AsyncGenerator<CaptureRecord> {\n    for (const path of await this.filesOf(source, filter?.since)) {\n      let number = 0;\n      for await (const { text, complete } of linesOf(path)) {\n        number += 1;\n        const trimmed = text.trim();\n        if (trimmed.length === 0) continue;\n        let row: CaptureRecord;\n        try {\n          row = JSON.parse(trimmed) as CaptureRecord;\n          if (typeof row !== \"object\" || row === null || typeof row.received_at !== \"string\") {\n            throw new Error(\"not a record\");\n          }\n        } catch {\n          // the append of a different process can be in progress at the end of the file\n          if (complete) options?.onDamaged?.({ path, line: number });\n          continue;\n        }\n        if (filter?.kind !== undefined && row.kind !== filter.kind) continue;\n        if (filter?.since !== undefined && row.received_at < filter.since) continue;\n        yield row;\n      }\n    }\n  }\n\n  /**\n   * This method reads the records that agree with the filter. It first waits for the appends that\n   * are in progress. It reads each file as a stream, so its memory does not grow with the log.\n   *\n   * The records of one source come in the sequence of the files. The records of two or more sources\n   * merge in the sequence of `received_at`.\n   */\n  async *scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord> {\n    await this.flush();\n    const sources =\n      filter?.source !== undefined\n        ? [filter.source]\n        : existsSync(this.dir)\n          ? (await readdir(this.dir, { withFileTypes: true }))\n              .filter((entry) => entry.isDirectory())\n              .map((entry) => entry.name)\n              .sort()\n          : [];\n    if (sources.length === 1) {\n      yield* this.readSource(sources[0]!, filter, options);\n      return;\n    }\n    // a merge of the sources: the next record is the earliest head\n    const streams = sources.map((source) =>\n      this.readSource(source, filter, options)[Symbol.asyncIterator](),\n    );\n    const heads: Array<CaptureRecord | undefined> = [];\n    for (const stream of streams)\n      heads.push((await stream.next()).value as CaptureRecord | undefined);\n    for (;;) {\n      let pick = -1;\n      for (const [index, head] of heads.entries()) {\n        if (head !== undefined && (pick === -1 || head.received_at < heads[pick]!.received_at)) {\n          pick = index;\n        }\n      }\n      if (pick === -1) return;\n      yield heads[pick]!;\n      heads[pick] = (await streams[pick]!.next()).value as CaptureRecord | undefined;\n    }\n  }\n\n  /** This method writes the appends in progress, then closes the files. A later append opens them again. */\n  async close(): Promise<void> {\n    await this.flush();\n    const handles = [...this.handles.values()];\n    this.handles.clear();\n    for (const handle of handles) await handle.close();\n  }\n}\n"],"mappings":";;;;;AAwBA,MAAM,OAAO;;AAGb,MAAM,6BAAkC,IAAI,IAAI;CAC9C;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,YAAY,MAAc,SAC9B,KAAK,MAAM,KAAK,MAAM,GAAG,CAAC,GAAG,KAAK,MAAM,GAAG,CAAC,GAAG,IAAI;AAErD,MAAM,UAAU,UAA8B,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK;AAE7F,MAAM,UAAU,UACd,OAAO,UAAU,YAAY,UAAU,QAAQ,UAAU,QAAQ,OAAO,MAAM,IAAI,IAAI;AAExF,MAAM,UAAU,SAA6C,KAAK,IAAI,CAAC,CAAC,YAAY,KAAA,CAAS;;AAG7F,MAAM,aAAa,OAAO,SAAgC;CACxD,IAAI,QAAQ,aAAa,SAAS;CAClC,IAAI;EACF,MAAM,SAAS,MAAM,KAAK,MAAM,GAAG;EACnC,IAAI;GACF,MAAM,OAAO,KAAK;EACpB,UAAU;GACR,MAAM,OAAO,MAAM;EACrB;CACF,QAAQ,CAER;AACF;;;;;;;;;;;;;;;;AAiBA,IAAa,gBAAb,MAAa,cAAmC;CAQnC;CAPX,0BAA2B,IAAI,IAG7B;CACF;CAEA,YACE,KACA,UAAgC,CAAC,GACjC;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,IAAI,OAAkF;EACpF,MAAM,OAAO,OAAO,KAAK;EACzB,MAAM,UAAU,KAAK,QAAQ,IAAI,IAAI;EACrC,IAAI,YAAY,KAAA,GAAW,OAAO;EAClC,MAAM,OAAO,KAAK,MAAM,MAAM,KAAK,CAAC,CAAC,cAAc,KAAK,QAAQ,OAAO,IAAI,CAAC;EAC5E,KAAK,QAAQ,IAAI,MAAM,IAAI;EAC3B,OAAO;CACT;;CAGA,aAAqB,MAAM,MAAc,MAAgC;EACvE,IAAI;GACF,OAAO,OAAO,MAAM,SAAS,IAAI,CAAC,MAAM;EAC1C,QAAQ;GACN,OAAO;EACT;CACF;CAEA,MAAc,MACZ,MACA,OAC+D;EAC/D,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,MAAM,WAAW,MAAM,OAAO,MAAM;EACpC,IAAI,UAAU,OAAO,MAAM,QAAQ,SAAS,SAAS,MAAM,QACzD,OAAO;GAAE;GAAM,SAAS;EAAK;EAC/B,MAAM,SAAS,QAAQ,MAAM;EAC7B,MAAM,MAAM,QAAQ;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EACpD,MAAM,YAAY,GAAG,OAAO,GAAG,WAAW,EAAE;EAC5C,IAAI;GACF,MAAM,SAAS,MAAM,KAAK,WAAW,MAAM,GAAK;GAChD,IAAI;IACF,MAAM,OAAO,UAAU,KAAK;IAC5B,IAAI,KAAK,MAAM,MAAM,OAAO,KAAK;GACnC,UAAU;IACR,MAAM,OAAO,MAAM;GACrB;GACA,IAAI;IACF,MAAM,OAAO,WAAW,MAAM;GAChC,SAAS,OAAO;IAEd,IAAI,CAAC,WAAW,IAAI,OAAO,KAAK,CAAC,KAAK,CAAE,MAAM,cAAc,MAAM,QAAQ,IAAI,GAC5E,MAAM;IACR,OAAO;KAAE;KAAM,SAAS;IAAK;GAC/B;GACA,IAAI,KAAK,MAAM,MAAM,WAAW,MAAM;GACtC,OAAO;IAAE;IAAM,SAAS;GAAM;EAChC,UAAU;GACR,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;EAC5D;CACF;CAEA,MAAM,IAAI,MAA+C;EACvD,IAAI,CAAC,KAAK,KAAK,IAAI,GAAG,OAAO,KAAA;EAC7B,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,IAAI;EACJ,IAAI;GACF,QAAQ,MAAM,SAAS,MAAM;EAC/B,QAAQ;GACN;EACF;EACA,IAAI,OAAO,KAAK,MAAM,MAAM;GAE1B,MAAM,OAAO,QAAQ,GAAG,OAAO,GAAG,WAAW,EAAE,SAAS,CAAC,CAAC,YAAY,KAAA,CAAS;GAC/E;EACF;EACA,OAAO,IAAI,WAAW,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;CACxE;AACF;;;ACvGA,MAAM,aAAa;AACnB,MAAM,WAAW;AACjB,MAAM,UAAU;;AAGhB,MAAM,WAAW,OAAO,QAAoB,UAAiC;CAC3E,IAAI,KAAK;CACT,OAAO,KAAK,MAAM,QAAQ;EACxB,MAAM,EAAE,iBAAiB,MAAM,OAAO,MAAM,OAAO,IAAI,MAAM,SAAS,EAAE;EACxE,MAAM;CACR;AACF;;;;;AAMA,gBAAgB,QACd,MACuE;CACvE,MAAM,UAAU,IAAI,YAAY,OAAO;CACvC,IAAI;CACJ,WAAW,MAAM,SAAS,iBAAiB,IAAI,GAA4B;EACzE,MAAM,OAAO,SAAS,KAAA,IAAY,QAAQ,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;EACrE,OAAO,KAAA;EACP,IAAI,QAAQ;EACZ,SAAS;GACP,MAAM,MAAM,KAAK,QAAQ,SAAS,KAAK;GACvC,IAAI,QAAQ,IAAI;GAChB,MAAM;IAAE,MAAM,QAAQ,OAAO,KAAK,SAAS,OAAO,GAAG,CAAC;IAAG,UAAU;GAAK;GACxE,QAAQ,MAAM;EAChB;EACA,IAAI,QAAQ,KAAK,QAAQ,OAAO,OAAO,KAAK,KAAK,SAAS,KAAK,CAAC;CAClE;CACA,IAAI,SAAS,KAAA,KAAa,KAAK,SAAS,GAAG,MAAM;EAAE,MAAM,QAAQ,OAAO,IAAI;EAAG,UAAU;CAAM;AACjG;AAEA,MAAM,YAAY,MAAc,SAC9B,GAAG,KAAK,GAAG,WAAW,KAAK,IAAI,CAAC,GAAG,MAAM;;;;;;;;;;;;;;;;;;AAmB3C,IAAa,aAAb,MAAa,WAAoC;CAOpC;CANX,0BAA2B,IAAI,IAAwB;CACvD,UAAsC,CAAC;CACvC;CACA;CAEA,YACE,KACA,UAA6B,CAAC,GAC9B;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,OAAe,QAA+B;EAC5C,OAAO,KACL,KAAK,KACL,OAAO,QACP,OAAO,YAAY,MAAM,GAAG,CAAC,GAC7B,GAAG,OAAO,YAAY,MAAM,GAAG,CAAC,EAAE,QACpC;CACF;CAEA,MAAc,UAAU,MAAmC;EACzD,MAAM,QAAQ,KAAK,QAAQ,IAAI,IAAI;EACnC,IAAI,UAAU,KAAA,GAAW,OAAO;EAEhC,MAAM,MAAM,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAC3D,MAAM,SAAS,MAAM,KAAK,MAAM,MAAM,GAAK;EAC3C,KAAK,QAAQ,IAAI,MAAM,MAAM;EAC7B,OAAO;CACT;;CAGA,aAAqB,WAAW,QAAsC;EACpE,MAAM,EAAE,SAAS,MAAM,OAAO,KAAK;EACnC,IAAI,SAAS,GAAG,OAAO;EACvB,MAAM,OAAO,OAAO,MAAM,CAAC;EAC3B,MAAM,OAAO,KAAK,MAAM,GAAG,GAAG,OAAO,CAAC;EACtC,OAAO,KAAK,OAAO;CACrB;CAEA,OAAO,QAAsC;EAC3C,IAAI;EACJ,IAAI;GACF,OAAO,GAAG,KAAK,UAAU,MAAM,EAAE;EACnC,SAAS,OAAO;GACd,OAAO,QAAQ,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC,CAAC;EACjF;EACA,OAAO,IAAI,SAAe,SAAS,WAAW;GAC5C,KAAK,QAAQ,KAAK;IAAE,MAAM,KAAK,OAAO,MAAM;IAAG;IAAM;IAAS;GAAO,CAAC;GACtE,KAAK,KAAK;EACZ,CAAC;CACH;;CAGA,OAAqB;EACnB,IAAI,KAAK,aAAa,KAAA,GAAW;EACjC,KAAK,WAAW,KAAK,MAAM,CAAC,CAAC,cAAc;GACzC,KAAK,WAAW,KAAA;GAChB,IAAI,KAAK,QAAQ,SAAS,GAAG,KAAK,KAAK;EACzC,CAAC;CACH;;CAGA,MAAc,QAAuB;EACnC,OAAO,KAAK,QAAQ,SAAS,GAAG;GAC9B,MAAM,QAAQ,KAAK,QAAQ,OAAO,CAAC;GACnC,MAAM,yBAAS,IAAI,IAAuB;GAC1C,KAAK,MAAM,QAAQ,OAAO;IACxB,MAAM,OAAO,OAAO,IAAI,KAAK,IAAI;IACjC,IAAI,SAAS,KAAA,GAAW,OAAO,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;SAC/C,KAAK,KAAK,IAAI;GACrB;GACA,KAAK,MAAM,CAAC,MAAM,UAAU,QAC1B,IAAI;IACF,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;IACxC,MAAM,OAAQ,MAAM,WAAW,WAAW,MAAM,IAAK,OAAO;IAC5D,MAAM,SACJ,QACA,OAAO,KAAK,GAAG,OAAO,MAAM,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,MAAM,CACzE;IACA,IAAI,KAAK,MAAM,MAAM,OAAO,SAAS;IACrC,KAAK,MAAM,QAAQ,OAAO,KAAK,QAAQ;GACzC,SAAS,OAAO;IAEd,MAAM,SAAS,KAAK,QAAQ,IAAI,IAAI;IACpC,KAAK,QAAQ,OAAO,IAAI;IACxB,MAAM,QAAQ,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;IAC3C,KAAK,MAAM,QAAQ,OAAO,KAAK,OAAO,KAAK;GAC7C;EAEJ;CACF;;CAGA,MAAM,QAAuB;EAC3B,OAAO,KAAK,aAAa,KAAA,GAAW,MAAM,KAAK;CACjD;;CAGA,MAAc,QAAQ,QAAgB,OAA8C;EAClF,MAAM,YAAY,KAAK,KAAK,KAAK,MAAM;EACvC,IAAI,CAAC,WAAW,SAAS,GAAG,OAAO,CAAC;EACpC,MAAM,OAAO,OAAO,MAAM,GAAG,CAAC;EAC9B,MAAM,SAAS,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC,EAAA,CAC5D,QAAQ,UAAU,MAAM,YAAY,KAAK,SAAS,KAAK,MAAM,IAAI,CAAC,CAAC,CACnE,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK;EACR,MAAM,QAAkB,CAAC;EACzB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,SAAS,KAAA,KAAa,OAAO,KAAK,MAAM,GAAG,CAAC,GAAG;GACnD,MAAM,UAAU,MAAM,QAAQ,KAAK,WAAW,IAAI,CAAC,EAAA,CAChD,QAAQ,SAAS,WAAW,KAAK,IAAI,CAAC,CAAC,CACvC,KAAK;GACR,KAAK,MAAM,SAAS,QAAQ;IAC1B,IAAI,SAAS,KAAA,KAAa,SAAS,MAAM,KAAK,IAAI,MAAM;IACxD,MAAM,KAAK,KAAK,WAAW,MAAM,KAAK,CAAC;GACzC;EACF;EACA,OAAO;CACT;;CAGA,OAAe,WACb,QACA,QACA,SAC+B;EAC/B,KAAK,MAAM,QAAQ,MAAM,KAAK,QAAQ,QAAQ,QAAQ,KAAK,GAAG;GAC5D,IAAI,SAAS;GACb,WAAW,MAAM,EAAE,MAAM,cAAc,QAAQ,IAAI,GAAG;IACpD,UAAU;IACV,MAAM,UAAU,KAAK,KAAK;IAC1B,IAAI,QAAQ,WAAW,GAAG;IAC1B,IAAI;IACJ,IAAI;KACF,MAAM,KAAK,MAAM,OAAO;KACxB,IAAI,OAAO,QAAQ,YAAY,QAAQ,QAAQ,OAAO,IAAI,gBAAgB,UACxE,MAAM,IAAI,MAAM,cAAc;IAElC,QAAQ;KAEN,IAAI,UAAU,SAAS,YAAY;MAAE;MAAM,MAAM;KAAO,CAAC;KACzD;IACF;IACA,IAAI,QAAQ,SAAS,KAAA,KAAa,IAAI,SAAS,OAAO,MAAM;IAC5D,IAAI,QAAQ,UAAU,KAAA,KAAa,IAAI,cAAc,OAAO,OAAO;IACnE,MAAM;GACR;EACF;CACF;;;;;;;;CASA,OAAO,KAAK,QAAwB,SAAqD;EACvF,MAAM,KAAK,MAAM;EACjB,MAAM,UACJ,QAAQ,WAAW,KAAA,IACf,CAAC,OAAO,MAAM,IACd,WAAW,KAAK,GAAG,KAChB,MAAM,QAAQ,KAAK,KAAK,EAAE,eAAe,KAAK,CAAC,EAAA,CAC7C,QAAQ,UAAU,MAAM,YAAY,CAAC,CAAC,CACtC,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK,IACR,CAAC;EACT,IAAI,QAAQ,WAAW,GAAG;GACxB,OAAO,KAAK,WAAW,QAAQ,IAAK,QAAQ,OAAO;GACnD;EACF;EAEA,MAAM,UAAU,QAAQ,KAAK,WAC3B,KAAK,WAAW,QAAQ,QAAQ,OAAO,CAAC,CAAC,OAAO,cAAc,CAAC,CACjE;EACA,MAAM,QAA0C,CAAC;EACjD,KAAK,MAAM,UAAU,SACnB,MAAM,MAAM,MAAM,OAAO,KAAK,EAAA,CAAG,KAAkC;EACrE,SAAS;GACP,IAAI,OAAO;GACX,KAAK,MAAM,CAAC,OAAO,SAAS,MAAM,QAAQ,GACxC,IAAI,SAAS,KAAA,MAAc,SAAS,MAAM,KAAK,cAAc,MAAM,KAAK,CAAE,cACxE,OAAO;GAGX,IAAI,SAAS,IAAI;GACjB,MAAM,MAAM;GACZ,MAAM,SAAS,MAAM,QAAQ,KAAK,CAAE,KAAK,EAAA,CAAG;EAC9C;CACF;;CAGA,MAAM,QAAuB;EAC3B,MAAM,KAAK,MAAM;EACjB,MAAM,UAAU,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC;EACzC,KAAK,QAAQ,MAAM;EACnB,KAAK,MAAM,UAAU,SAAS,MAAM,OAAO,MAAM;CACnD;AACF"}

package/build/mcp-package-Y0bREnOS.mjs

@@ -0,0 +1,40 @@
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+//#region src/mcp-package.ts
+/** The name of the package of the MCP server. */
+const MCP_PACKAGE = "polite-crawler-mcp";
+/** This text tells where the MCP server is. */
+const MCP_POINTER = "The MCP server of polite-crawler is the package polite-crawler-mcp since 0.2.0. Start it with npx polite-crawler-mcp, or install polite-crawler-mcp next to polite-crawler.";
+/** This error tells that polite-crawler-mcp is not there. Its message is the pointer. */
+var McpPackageMissing = class extends Error {
+	constructor() {
+		super(MCP_POINTER);
+		this.name = "McpPackageMissing";
+	}
+};
+/**
+* This function gives the places where polite-crawler looks for polite-crawler-mcp, as file URLs:
+* only next to polite-crawler.
+*/
+const mcpPackagePlaces = () => [import.meta.url];
+/**
+* This function loads polite-crawler-mcp from the first place that has it. It throws
+* `McpPackageMissing` when no place has it. A test gives another `name`: Node.js also looks in the
+* folders of `NODE_PATH`, and pnpm sets it to the packages of the workspace.
+*/
+const loadMcpPackage = async (places = mcpPackagePlaces(), name = MCP_PACKAGE) => {
+	for (const from of places) {
+		let path;
+		try {
+			path = createRequire(from).resolve(name);
+		} catch {
+			continue;
+		}
+		return await import(pathToFileURL(path).href);
+	}
+	throw new McpPackageMissing();
+};
+//#endregion
+export { McpPackageMissing, loadMcpPackage };
+
+//# sourceMappingURL=mcp-package-Y0bREnOS.mjs.map

package/build/mcp-package-Y0bREnOS.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp-package-Y0bREnOS.mjs","names":[],"sources":["../src/mcp-package.ts"],"sourcesContent":["import { createRequire } from \"node:module\";\nimport { pathToFileURL } from \"node:url\";\nimport type { DaemonMcpHost } from \"./daemon.ts\";\nimport type { OpenOptions } from \"./open.ts\";\n\n/*\n * The MCP server of polite-crawler is the package polite-crawler-mcp since polite-crawler 0.2.0.\n * polite-crawler does not depend on it, so the library loads no MCP SDK and no zod. The command\n * `polite-crawler mcp` and the MCP server of the daemon load the package when it is there: next to\n * polite-crawler. They do not look in the working folder: an MCP client can start the command in any\n * project, and the code of that project must not start.\n */\n\n/** The name of the package of the MCP server. */\nexport const MCP_PACKAGE = \"polite-crawler-mcp\";\n\n/** This text tells where the MCP server is. */\nexport const MCP_POINTER =\n  \"The MCP server of polite-crawler is the package polite-crawler-mcp since 0.2.0. Start it with npx polite-crawler-mcp, or install polite-crawler-mcp next to polite-crawler.\";\n\n/** The parts of polite-crawler-mcp that polite-crawler uses. */\nexport interface McpPackage {\n  startMcpServer(\n    options: OpenOptions & {\n      readonly allowUserProvided?: boolean | undefined;\n      readonly log?: ((line: string) => void) | undefined;\n    },\n  ): Promise<void>;\n  readonly mcpHttpHost: DaemonMcpHost;\n}\n\n/** This error tells that polite-crawler-mcp is not there. Its message is the pointer. */\nexport class McpPackageMissing extends Error {\n  constructor() {\n    super(MCP_POINTER);\n    this.name = \"McpPackageMissing\";\n  }\n}\n\n/**\n * This function gives the places where polite-crawler looks for polite-crawler-mcp, as file URLs:\n * only next to polite-crawler.\n */\nexport const mcpPackagePlaces = (): string[] => [import.meta.url];\n\n/**\n * This function loads polite-crawler-mcp from the first place that has it. It throws\n * `McpPackageMissing` when no place has it. A test gives another `name`: Node.js also looks in the\n * folders of `NODE_PATH`, and pnpm sets it to the packages of the workspace.\n */\nexport const loadMcpPackage = async (\n  places: readonly string[] = mcpPackagePlaces(),\n  name: string = MCP_PACKAGE,\n): Promise<McpPackage> => {\n  for (const from of places) {\n    let path: string;\n    try {\n      path = createRequire(from).resolve(name);\n    } catch {\n      continue;\n    }\n    return (await import(pathToFileURL(path).href)) as McpPackage;\n  }\n  throw new McpPackageMissing();\n};\n"],"mappings":";;;;AAcA,MAAa,cAAc;;AAG3B,MAAa,cACX;;AAcF,IAAa,oBAAb,cAAuC,MAAM;CAC3C,cAAc;EACZ,MAAM,WAAW;EACjB,KAAK,OAAO;CACd;AACF;;;;;AAMA,MAAa,yBAAmC,CAAC,YAAY,GAAG;;;;;;AAOhE,MAAa,iBAAiB,OAC5B,SAA4B,iBAAiB,GAC7C,OAAe,gBACS;CACxB,KAAK,MAAM,QAAQ,QAAQ;EACzB,IAAI;EACJ,IAAI;GACF,OAAO,cAAc,IAAI,CAAC,CAAC,QAAQ,IAAI;EACzC,QAAQ;GACN;EACF;EACA,OAAQ,MAAM,OAAO,cAAc,IAAI,CAAC,CAAC;CAC3C;CACA,MAAM,IAAI,kBAAkB;AAC9B"}

package/build/{ollama-BJxkjny4.mjs → ollama-D1xRh3hY.mjs}

@@ -1,4 +1,4 @@
-import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
 import { createHash } from "node:crypto";
 //#region src/ollama.ts
 var ollama_exports = /* @__PURE__ */ __exportAll({
@@ -27,7 +27,11 @@ const OLLAMA_MODELS = {
 };
 /** The text of the error when no Ollama server answers. */
 const NO_OLLAMA = "Ollama does not answer: install it from https://ollama.com, then start it with ollama serve";
-/** This function tells if a host name is a loopback address: `localhost`, `127.0.0.0/8` or `::1`. */
+/**
+* This function tells if a host name is a loopback address: `localhost`, `127.0.0.0/8` or `::1`.
+*
+* @beta
+*/
 const isLoopback = (host) => {
 	const name = host.replace(/^\[|\]$/g, "").toLowerCase();
 	return name === "localhost" || name.endsWith(".localhost") || name === "::1" || /^127(?:\.\d{1,3}){3}$/.test(name) || /^::ffff:127(?:\.\d{1,3}){3}$/.test(name);
@@ -157,4 +161,4 @@ const ollamaEmbedder = (options = {}) => {
 //#endregion
 export { ollama_exports as a, ollamaEmbedder as i, OLLAMA_MODELS as n, isLoopback as r, NO_OLLAMA as t };
 
-//# sourceMappingURL=ollama-BJxkjny4.mjs.map
+//# sourceMappingURL=ollama-D1xRh3hY.mjs.map

package/build/ollama-D1xRh3hY.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ollama-D1xRh3hY.mjs","names":[],"sources":["../src/ollama.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { EmbedCallOptions, Embedder } from \"./embed.ts\";\n\n/**\n * The Ollama embedder sends texts to the local HTTP API of Ollama (`POST /api/embed`). The package\n * has no dependency for it: you install Ollama, start it (`ollama serve`) and pull a model\n * (`ollama pull nomic-embed-text`).\n *\n * The page text goes to the Ollama server. Thus the embedder accepts only a server on this\n * computer (a loopback address), unless `allowRemote` is true.\n */\n\n/** These are the facts of the embedding models that the Ollama library has. */\nexport interface EmbeddingModel {\n  readonly dimensions: number;\n  readonly maxInputTokens: number;\n  /** This prefix goes before the text of a question. */\n  readonly queryPrefix?: string | undefined;\n  /** This prefix goes before each chunk of a stored text. */\n  readonly documentPrefix?: string | undefined;\n}\n\n/** These are the facts of three embedding models of Ollama. The name has no tag (`nomic-embed-text`, not `nomic-embed-text:latest`). */\nexport const OLLAMA_MODELS: Readonly<Record<string, EmbeddingModel>> = {\n  \"nomic-embed-text\": {\n    dimensions: 768,\n    maxInputTokens: 8192,\n    queryPrefix: \"search_query: \",\n    documentPrefix: \"search_document: \",\n  },\n  \"all-minilm\": { dimensions: 384, maxInputTokens: 256 },\n  \"mxbai-embed-large\": {\n    dimensions: 1024,\n    maxInputTokens: 512,\n    queryPrefix: \"Represent this sentence for searching relevant passages: \",\n  },\n};\n\n/** These options make an Ollama embedder. */\nexport interface OllamaEmbedderOptions {\n  /** This value is the model of Ollama, with a tag when you want one. The default is `nomic-embed-text`. */\n  readonly model?: string | undefined;\n  /** This URL is the Ollama server. The default is `http://127.0.0.1:11434`. */\n  readonly baseUrl?: string | undefined;\n  /** This value is the number of texts in one request. The default is 32. */\n  readonly batch?: number | undefined;\n  /**\n   * The value true lets `baseUrl` be a server on a different computer. The default is false,\n   * because the embedder sends the text of the stored pages to the server.\n   */\n  readonly allowRemote?: boolean | undefined;\n  /** This value is the length of each vector. The default comes from `OLLAMA_MODELS`, or from the first answer. */\n  readonly dimensions?: number | undefined;\n  /** This value is the input limit of the model in tokens. The default comes from `OLLAMA_MODELS`, or it is 256. */\n  readonly maxInputTokens?: number | undefined;\n  /** This prefix goes before the text of a question. The default comes from `OLLAMA_MODELS`. */\n  readonly queryPrefix?: string | undefined;\n  /** This prefix goes before each chunk of a stored text. The default comes from `OLLAMA_MODELS`. */\n  readonly documentPrefix?: string | undefined;\n  /** This value is the longest time of one request in milliseconds. The default is 2 minutes, because the first request loads the model. */\n  readonly timeoutMs?: number | undefined;\n}\n\n/** The text of the error when no Ollama server answers. */\nexport const NO_OLLAMA =\n  \"Ollama does not answer: install it from https://ollama.com, then start it with ollama serve\";\n\n/**\n * This function tells if a host name is a loopback address: `localhost`, `127.0.0.0/8` or `::1`.\n *\n * @beta\n */\nexport const isLoopback = (host: string): boolean => {\n  const name = host.replace(/^\\[|\\]$/g, \"\").toLowerCase();\n  return (\n    name === \"localhost\" ||\n    name.endsWith(\".localhost\") ||\n    name === \"::1\" ||\n    /^127(?:\\.\\d{1,3}){3}$/.test(name) ||\n    /^::ffff:127(?:\\.\\d{1,3}){3}$/.test(name)\n  );\n};\n\n/** This function gives the name of a model with no tag, for `OLLAMA_MODELS`. */\nconst baseName = (model: string): string => {\n  const slash = model.lastIndexOf(\"/\");\n  const colon = model.indexOf(\":\", slash + 1);\n  return colon === -1 ? model : model.slice(0, colon);\n};\n\nconst messageOf = (error: unknown): string =>\n  error instanceof Error ? error.message : String(error);\n\n/** This function gives the code of a network error, also when it is the cause of the error of `fetch`. */\nconst codeOf = (error: unknown): string | undefined => {\n  for (\n    let at: unknown = error, depth = 0;\n    at !== undefined && at !== null && depth < 4;\n    depth += 1\n  ) {\n    const code = (at as { code?: unknown }).code;\n    if (typeof code === \"string\") return code;\n    at = (at as { cause?: unknown }).cause;\n  }\n  return undefined;\n};\n\nconst NETWORK = new Set([\n  \"ECONNREFUSED\",\n  \"ECONNRESET\",\n  \"ENOTFOUND\",\n  \"EHOSTUNREACH\",\n  \"ENETUNREACH\",\n  \"EAI_AGAIN\",\n  \"UND_ERR_SOCKET\",\n  \"UND_ERR_CONNECT_TIMEOUT\",\n]);\n\n/**\n * This function makes an embedder that sends texts to Ollama (`POST {baseUrl}/api/embed`). It\n * sends `batch` texts in each request, and it stops when the signal aborts. The errors tell what\n * to do: `ollama serve` when no server answers, and `ollama pull <model>` when the model is not\n * there. A `baseUrl` on a different computer throws a `RangeError`, unless `allowRemote` is true.\n */\nexport const ollamaEmbedder = (options: OllamaEmbedderOptions = {}): Embedder => {\n  const model = options.model ?? \"nomic-embed-text\";\n  if (model.trim() === \"\") throw new RangeError(\"the Ollama model must have a name\");\n  const base = options.baseUrl ?? \"http://127.0.0.1:11434\";\n  let url: URL;\n  try {\n    url = new URL(base);\n  } catch (cause) {\n    throw new RangeError(`the Ollama URL ${base} is not a URL`, { cause });\n  }\n  if (url.protocol !== \"http:\" && url.protocol !== \"https:\")\n    throw new RangeError(`the Ollama URL ${base} must be an http or https URL`);\n  if (url.username !== \"\" || url.password !== \"\")\n    throw new RangeError(\"the Ollama URL must not hold a user name or a password\");\n  if (!isLoopback(url.hostname) && options.allowRemote !== true) {\n    throw new RangeError(\n      `the Ollama URL ${url.origin} is not on this computer, and the embedder sends the text of the stored pages to it: give allowRemote: true (--allow-remote-ollama) to permit it`,\n    );\n  }\n  const endpoint = new URL(\"api/embed\", url.href.endsWith(\"/\") ? url.href : `${url.href}/`).href;\n  const known = OLLAMA_MODELS[baseName(model)];\n  const batch = Math.max(1, Math.floor(options.batch ?? 32));\n  const timeoutMs = Math.max(1, Math.floor(options.timeoutMs ?? 120_000));\n  const queryPrefix = options.queryPrefix ?? known?.queryPrefix ?? \"\";\n  const documentPrefix = options.documentPrefix ?? known?.documentPrefix ?? \"\";\n  // a document prefix of your own changes each stored vector, so it changes the id\n  const custom =\n    options.documentPrefix !== undefined && options.documentPrefix !== (known?.documentPrefix ?? \"\")\n      ? `#${createHash(\"sha256\").update(options.documentPrefix).digest(\"hex\").slice(0, 8)}`\n      : \"\";\n  let dimensions = options.dimensions ?? known?.dimensions ?? 0;\n  const maxInputTokens = options.maxInputTokens ?? known?.maxInputTokens ?? 256;\n\n  const request = async (\n    input: readonly string[],\n    signal: AbortSignal | undefined,\n  ): Promise<number[][]> => {\n    const timeout = AbortSignal.timeout(timeoutMs);\n    const both = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);\n    let response: Response;\n    try {\n      response = await fetch(endpoint, {\n        method: \"POST\",\n        headers: { \"content-type\": \"application/json\" },\n        body: JSON.stringify({ model, input, truncate: true }),\n        signal: both,\n      });\n    } catch (error) {\n      if (signal?.aborted === true) throw signal.reason;\n      if (timeout.aborted)\n        throw new Error(`Ollama at ${url.origin} did not answer in ${String(timeoutMs)} ms`, {\n          cause: error,\n        });\n      const code = codeOf(error);\n      if (code !== undefined && NETWORK.has(code))\n        throw new Error(`${NO_OLLAMA} (no answer at ${url.origin}: ${code})`, { cause: error });\n      // fetch says only \"fetch failed\": the cause tells why\n      const cause = (error as { cause?: unknown }).cause;\n      throw new Error(\n        `the request to Ollama at ${url.origin} failed: ${messageOf(error)}${cause === undefined ? \"\" : ` (${messageOf(cause)})`}`,\n        { cause: error },\n      );\n    }\n    let text: string;\n    try {\n      text = await response.text();\n    } catch (error) {\n      if (signal?.aborted === true) throw signal.reason;\n      throw new Error(`the answer of Ollama at ${url.origin} stopped: ${messageOf(error)}`, {\n        cause: error,\n      });\n    }\n    let body: { embeddings?: unknown; error?: unknown } | undefined;\n    try {\n      body = JSON.parse(text) as { embeddings?: unknown; error?: unknown };\n    } catch {\n      body = undefined;\n    }\n    const said = typeof body?.error === \"string\" ? body.error : undefined;\n    if (!response.ok) {\n      if (said !== undefined && /not found|pull/i.test(said))\n        throw new Error(\n          `the model ${model} is not in Ollama: ollama pull ${model} (Ollama said: ${said})`,\n        );\n      if (response.status === 404)\n        throw new Error(\n          `the server at ${url.origin} has no /api/embed: update Ollama to version 0.3.0 or later`,\n        );\n      if (said !== undefined && /embed/i.test(said))\n        throw new Error(\n          `the model ${model} does not make embeddings: use an embedding model, such as nomic-embed-text (Ollama said: ${said})`,\n        );\n      throw new Error(\n        `Ollama answered ${String(response.status)}${said === undefined ? \"\" : `: ${said}`}`,\n      );\n    }\n    const embeddings = body?.embeddings;\n    if (\n      !Array.isArray(embeddings) ||\n      embeddings.length !== input.length ||\n      !embeddings.every((vector) => Array.isArray(vector))\n    ) {\n      throw new Error(\n        `Ollama gave no embedding for each text (${String(input.length)} texts): its answer is not the answer of /api/embed`,\n      );\n    }\n    return embeddings as number[][];\n  };\n\n  return {\n    id: `ollama:${model}${custom}`,\n    get dimensions() {\n      return dimensions;\n    },\n    maxInputTokens,\n    async embed(texts: readonly string[], call: EmbedCallOptions = {}): Promise<Float32Array[]> {\n      const prefix = call.purpose === \"query\" ? queryPrefix : documentPrefix;\n      const out: Float32Array[] = [];\n      for (let at = 0; at < texts.length; at += batch) {\n        call.signal?.throwIfAborted();\n        const part = texts.slice(at, at + batch).map((text) => `${prefix}${text}`);\n        for (const vector of await request(part, call.signal)) {\n          if (dimensions === 0) dimensions = vector.length;\n          if (vector.length !== dimensions) {\n            throw new Error(\n              `the model ${model} gave a vector of ${String(vector.length)} dimensions, not ${String(dimensions)}`,\n            );\n          }\n          out.push(Float32Array.from(vector));\n        }\n      }\n      return out;\n    },\n  };\n};\n"],"mappings":";;;;;;;;;;AAuBA,MAAa,gBAA0D;CACrE,oBAAoB;EAClB,YAAY;EACZ,gBAAgB;EAChB,aAAa;EACb,gBAAgB;CAClB;CACA,cAAc;EAAE,YAAY;EAAK,gBAAgB;CAAI;CACrD,qBAAqB;EACnB,YAAY;EACZ,gBAAgB;EAChB,aAAa;CACf;AACF;;AA4BA,MAAa,YACX;;;;;;AAOF,MAAa,cAAc,SAA0B;CACnD,MAAM,OAAO,KAAK,QAAQ,YAAY,EAAE,CAAC,CAAC,YAAY;CACtD,OACE,SAAS,eACT,KAAK,SAAS,YAAY,KAC1B,SAAS,SACT,wBAAwB,KAAK,IAAI,KACjC,+BAA+B,KAAK,IAAI;AAE5C;;AAGA,MAAM,YAAY,UAA0B;CAC1C,MAAM,QAAQ,MAAM,YAAY,GAAG;CACnC,MAAM,QAAQ,MAAM,QAAQ,KAAK,QAAQ,CAAC;CAC1C,OAAO,UAAU,KAAK,QAAQ,MAAM,MAAM,GAAG,KAAK;AACpD;AAEA,MAAM,aAAa,UACjB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;;AAGvD,MAAM,UAAU,UAAuC;CACrD,KACE,IAAI,KAAc,OAAO,QAAQ,GACjC,OAAO,KAAA,KAAa,OAAO,QAAQ,QAAQ,GAC3C,SAAS,GACT;EACA,MAAM,OAAQ,GAA0B;EACxC,IAAI,OAAO,SAAS,UAAU,OAAO;EACrC,KAAM,GAA2B;CACnC;AAEF;AAEA,MAAM,0BAAU,IAAI,IAAI;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;AAQD,MAAa,kBAAkB,UAAiC,CAAC,MAAgB;CAC/E,MAAM,QAAQ,QAAQ,SAAS;CAC/B,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,IAAI,WAAW,mCAAmC;CACjF,MAAM,OAAO,QAAQ,WAAW;CAChC,IAAI;CACJ,IAAI;EACF,MAAM,IAAI,IAAI,IAAI;CACpB,SAAS,OAAO;EACd,MAAM,IAAI,WAAW,kBAAkB,KAAK,gBAAgB,EAAE,MAAM,CAAC;CACvE;CACA,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAC/C,MAAM,IAAI,WAAW,kBAAkB,KAAK,8BAA8B;CAC5E,IAAI,IAAI,aAAa,MAAM,IAAI,aAAa,IAC1C,MAAM,IAAI,WAAW,wDAAwD;CAC/E,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,QAAQ,gBAAgB,MACvD,MAAM,IAAI,WACR,kBAAkB,IAAI,OAAO,iJAC/B;CAEF,MAAM,WAAW,IAAI,IAAI,aAAa,IAAI,KAAK,SAAS,GAAG,IAAI,IAAI,OAAO,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;CAC1F,MAAM,QAAQ,cAAc,SAAS,KAAK;CAC1C,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,EAAE,CAAC;CACzD,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,aAAa,IAAO,CAAC;CACtE,MAAM,cAAc,QAAQ,eAAe,OAAO,eAAe;CACjE,MAAM,iBAAiB,QAAQ,kBAAkB,OAAO,kBAAkB;CAE1E,MAAM,SACJ,QAAQ,mBAAmB,KAAA,KAAa,QAAQ,oBAAoB,OAAO,kBAAkB,MACzF,IAAI,WAAW,QAAQ,CAAC,CAAC,OAAO,QAAQ,cAAc,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,MAChF;CACN,IAAI,aAAa,QAAQ,cAAc,OAAO,cAAc;CAC5D,MAAM,iBAAiB,QAAQ,kBAAkB,OAAO,kBAAkB;CAE1E,MAAM,UAAU,OACd,OACA,WACwB;EACxB,MAAM,UAAU,YAAY,QAAQ,SAAS;EAC7C,MAAM,OAAO,WAAW,KAAA,IAAY,UAAU,YAAY,IAAI,CAAC,QAAQ,OAAO,CAAC;EAC/E,IAAI;EACJ,IAAI;GACF,WAAW,MAAM,MAAM,UAAU;IAC/B,QAAQ;IACR,SAAS,EAAE,gBAAgB,mBAAmB;IAC9C,MAAM,KAAK,UAAU;KAAE;KAAO;KAAO,UAAU;IAAK,CAAC;IACrD,QAAQ;GACV,CAAC;EACH,SAAS,OAAO;GACd,IAAI,QAAQ,YAAY,MAAM,MAAM,OAAO;GAC3C,IAAI,QAAQ,SACV,MAAM,IAAI,MAAM,aAAa,IAAI,OAAO,qBAAqB,OAAO,SAAS,EAAE,MAAM,EACnF,OAAO,MACT,CAAC;GACH,MAAM,OAAO,OAAO,KAAK;GACzB,IAAI,SAAS,KAAA,KAAa,QAAQ,IAAI,IAAI,GACxC,MAAM,IAAI,MAAM,GAAG,UAAU,iBAAiB,IAAI,OAAO,IAAI,KAAK,IAAI,EAAE,OAAO,MAAM,CAAC;GAExF,MAAM,QAAS,MAA8B;GAC7C,MAAM,IAAI,MACR,4BAA4B,IAAI,OAAO,WAAW,UAAU,KAAK,IAAI,UAAU,KAAA,IAAY,KAAK,KAAK,UAAU,KAAK,EAAE,MACtH,EAAE,OAAO,MAAM,CACjB;EACF;EACA,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,SAAS,KAAK;EAC7B,SAAS,OAAO;GACd,IAAI,QAAQ,YAAY,MAAM,MAAM,OAAO;GAC3C,MAAM,IAAI,MAAM,2BAA2B,IAAI,OAAO,YAAY,UAAU,KAAK,KAAK,EACpF,OAAO,MACT,CAAC;EACH;EACA,IAAI;EACJ,IAAI;GACF,OAAO,KAAK,MAAM,IAAI;EACxB,QAAQ;GACN,OAAO,KAAA;EACT;EACA,MAAM,OAAO,OAAO,MAAM,UAAU,WAAW,KAAK,QAAQ,KAAA;EAC5D,IAAI,CAAC,SAAS,IAAI;GAChB,IAAI,SAAS,KAAA,KAAa,kBAAkB,KAAK,IAAI,GACnD,MAAM,IAAI,MACR,aAAa,MAAM,iCAAiC,MAAM,iBAAiB,KAAK,EAClF;GACF,IAAI,SAAS,WAAW,KACtB,MAAM,IAAI,MACR,iBAAiB,IAAI,OAAO,4DAC9B;GACF,IAAI,SAAS,KAAA,KAAa,SAAS,KAAK,IAAI,GAC1C,MAAM,IAAI,MACR,aAAa,MAAM,4FAA4F,KAAK,EACtH;GACF,MAAM,IAAI,MACR,mBAAmB,OAAO,SAAS,MAAM,IAAI,SAAS,KAAA,IAAY,KAAK,KAAK,QAC9E;EACF;EACA,MAAM,aAAa,MAAM;EACzB,IACE,CAAC,MAAM,QAAQ,UAAU,KACzB,WAAW,WAAW,MAAM,UAC5B,CAAC,WAAW,OAAO,WAAW,MAAM,QAAQ,MAAM,CAAC,GAEnD,MAAM,IAAI,MACR,2CAA2C,OAAO,MAAM,MAAM,EAAE,oDAClE;EAEF,OAAO;CACT;CAEA,OAAO;EACL,IAAI,UAAU,QAAQ;EACtB,IAAI,aAAa;GACf,OAAO;EACT;EACA;EACA,MAAM,MAAM,OAA0B,OAAyB,CAAC,GAA4B;GAC1F,MAAM,SAAS,KAAK,YAAY,UAAU,cAAc;GACxD,MAAM,MAAsB,CAAC;GAC7B,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,QAAQ,MAAM,OAAO;IAC/C,KAAK,QAAQ,eAAe;IAC5B,MAAM,OAAO,MAAM,MAAM,IAAI,KAAK,KAAK,CAAC,CAAC,KAAK,SAAS,GAAG,SAAS,MAAM;IACzE,KAAK,MAAM,UAAU,MAAM,QAAQ,MAAM,KAAK,MAAM,GAAG;KACrD,IAAI,eAAe,GAAG,aAAa,OAAO;KAC1C,IAAI,OAAO,WAAW,YACpB,MAAM,IAAI,MACR,aAAa,MAAM,oBAAoB,OAAO,OAAO,MAAM,EAAE,mBAAmB,OAAO,UAAU,GACnG;KAEF,IAAI,KAAK,aAAa,KAAK,MAAM,CAAC;IACpC;GACF;GACA,OAAO;EACT;CACF;AACF"}

package/build/{open-CbpSCEZc.mjs → open-CEo22Hpu.mjs}

@@ -1,16 +1,16 @@
-import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
-import { n as FileBlobStore, t as CaptureLog } from "./log-Dx-wXjJl.mjs";
-import { r as playwrightRenderer } from "./render-C1CUraWF.mjs";
-import { r as checkedRenderer } from "./browser-CKRNbIDT.mjs";
-import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-BPem-XZd.mjs";
-import { A as snippetOf, E as fold, L as sha256$1, M as Crawler, N as DEADLINE_NOTE, R as stableStringify, S as archivedOf, a as fetchPage, c as isHeldBack, f as readDocument, g as verifyQuote, i as fetchAndIndex, k as parseQuery, m as reindex, o as findDocuments, r as catchUpIndex, t as Extractors } from "./runtime-D_r2SYge.mjs";
+import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
+import { n as FileBlobStore, t as CaptureLog } from "./log-CmBWcyQm.mjs";
+import { r as playwrightRenderer } from "./render-CKE2LILH.mjs";
+import { r as checkedRenderer } from "./browser-56dBBLec.mjs";
+import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-Djva7JkN.mjs";
+import { A as DEADLINE_NOTE, C as fold, D as snippetOf, E as parseQuery, N as sha256$1, P as stableStringify, a as fetchPage, c as isHeldBack, f as reindex, i as fetchAndIndex, k as Crawler, o as findDocuments, p as verifyQuote, r as catchUpIndex, t as Extractors, u as readDocument, y as archivedOf } from "./runtime-C_2BQibH.mjs";
 import { MemoryHostStateStore, nextValidatorMemo, realClock } from "./politeness.mjs";
 import { matchesPattern } from "./robots.mjs";
 import { n as canonicalUrl, r as scopeTest, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
 import { o as decodeBody, t as extractPage } from "./html-DkQUAE_d.mjs";
-import { n as diffDocument, r as diffTexts } from "./diff-BUM91Rev.mjs";
-import { r as extractDocument } from "./extract-CefB5Vfn.mjs";
-import { o as registerRecipe } from "./recipe-GiFQojmo.mjs";
+import { n as diffDocument, r as diffTexts } from "./diff-CpKN0Jdc.mjs";
+import { r as extractDocument } from "./extract-DqeqiNHb.mjs";
+import { o as registerRecipe } from "./recipe-B2SkepDZ.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { dirname, join } from "node:path";
 import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
@@ -18,7 +18,11 @@ import { createGunzip } from "node:zlib";
 import { DatabaseSync } from "node:sqlite";
 /** The search index reads this many characters of each text, at most. */
 const MAX_INDEX_CHARS = 2e5;
-/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
+/**
+* In the mode `fast`, a part is common when more than this share of the documents hold it.
+*
+* @beta
+*/
 const FAST_RANK_SHARE = .2;
 /** The text of the error of hybrid search with no embedder. */
 const NO_EMBEDDER = "hybrid search needs an embedder: open the crawler with the option embedder (transformersEmbedder or ollamaEmbedder), or give --embedder transformers or ollama to the command line";
@@ -164,6 +168,7 @@ const toVersion = (row) => ({
 	cut: row.full_sha !== null
 });
 const messageOf = (error) => error instanceof Error ? error.message : String(error);
+/** @beta */
 var DocIndex = class {
 	path;
 	db;
@@ -814,7 +819,11 @@ var DocIndex = class {
 };
 //#endregion
 //#region src/sitemaps.ts
-/** These are the default limits of the reader, and the maximum length of one value. */
+/**
+* These are the default limits of the reader, and the maximum length of one value.
+*
+* @beta
+*/
 const SITEMAP_LIMITS = {
 	maxEntries: 5e4,
 	maxDepth: 32,
@@ -856,6 +865,8 @@ const short = (name) => name === void 0 ? "nothing" : name.length > 40 ? `${name
 * This function changes a date of a sitemap or a feed into ISO 8601. It reads the W3C form of the
 * sitemap protocol and Atom, and the mail form of RSS. A time with no zone is UTC. It gives null
 * for a text that it cannot read.
+*
+* @beta
 */
 const feedDate = (value) => {
 	const text = value.trim();
@@ -1305,6 +1316,8 @@ const scanSitemap = (text, base, limits = {}, meter = { steps: 0 }) => {
 * This function reads a sitemap, a sitemap index, an RSS or Atom feed, or a text sitemap. The
 * root element of the file tells its kind. The entries of a feed can have URLs relative to `base`
 * (the URL of the file). The entries of a sitemap must have absolute URLs.
+*
+* @beta
 */
 const readSitemap = (text, base, limits) => scanSitemap(text, base, limits);
 //#endregion
@@ -1323,7 +1336,11 @@ const readSitemap = (text, base, limits) => scanSitemap(text, base, limits);
 * Discovery keeps a URL only when it is in scope (the same site by default) and robots.txt permits
 * it. It stops at a maximum of files, a maximum of URLs and a maximum time.
 */
-/** These are the default limits of `discover`. */
+/**
+* These are the default limits of `discover`.
+*
+* @beta
+*/
 const DISCOVER_LIMITS = {
 	/** The most sitemap and feed files that one discovery reads. */
 	maxFiles: 20,
@@ -1413,6 +1430,8 @@ const pastDeadline = (note) => note?.startsWith(DEADLINE_NOTE) === true;
 * This function finds the pages of a site from its sitemaps and its feeds. `PoliteCrawler.discover`
 * uses it. Each file is a fetch of the crawler with the recipe `sitemap` or `feed`, so the
 * capture log records it, and robots.txt applies to it.
+*
+* @beta
 */
 const discoverSite = async (runtime, site, options = {}) => {
 	const start = parsedUrl(site);
@@ -1722,7 +1741,11 @@ const discoverSite = async (runtime, site, options = {}) => {
 *
 * The state of the frontier is in memory, for one call. A later version can make it resumable.
 */
-/** These are the default limits of `crawl`, and the limits of the memory of the frontier. */
+/**
+* These are the default limits of `crawl`, and the limits of the memory of the frontier.
+*
+* @beta
+*/
 const CRAWL_LIMITS = {
 	/** The most pages that one crawl gives. */
 	maxPages: 100,
@@ -1820,6 +1843,8 @@ const REFUSALS$1 = /* @__PURE__ */ new Set([
 * This function crawls a site from its seeds. `PoliteCrawler.crawl` uses it. It gives each page
 * when its fetch ends, and a summary at the end. The seeds come first, then the pages of the
 * sitemaps (the newest first), then the links, the lower depth first.
+*
+* @beta
 */
 const crawlSite = (runtime, seeds, options = {}) => {
 	let resolveSummary = () => void 0;
@@ -2170,6 +2195,8 @@ const memoOf = (row) => ({
 *
 * The state is not a projection of the capture log. A deletion of the file loses only the pace,
 * the counts, the stops and the memos, and the crawler starts them again.
+*
+* @beta
 */
 var SqliteHostStateStore = class {
 	path;
@@ -2351,7 +2378,11 @@ var SqliteHostStateStore = class {
 };
 //#endregion
 //#region src/embed.ts
-/** These are the limits of the chunks of a text. */
+/**
+* These are the limits of the chunks of a text.
+*
+* @beta
+*/
 const CHUNK_LIMITS = {
 	maxTokens: 512,
 	defaultTokens: 256,
@@ -2378,6 +2409,8 @@ const kindOf = (point) => {
 * This function estimates the number of tokens of a part of a text. A word is one token for each
 * six letters. A Chinese, Japanese or Thai character is one token. Each other character that is
 * not a space is one token. The estimate is a little high for English text, so a chunk fits.
+*
+* @beta
 */
 const estimateTokens = (text, start = 0, end = text.length) => {
 	let tokens = 0;
@@ -2504,6 +2537,8 @@ const positive = (name, value) => {
 * sentence that is longer than the window is cut at its spaces. Two chunks next to each other share
 * a few sentences (`overlapTokens`), so a passage at a cut is in one chunk. Each chunk keeps its
 * position in the text, so a hit points at a passage.
+*
+* @beta
 */
 const chunkText = (text, options) => {
 	const size = positive("maxTokens", options.maxTokens);
@@ -2553,13 +2588,19 @@ const chunkText = (text, options) => {
 /**
 * This function gives the number of tokens of a chunk for an embedder. It is the input limit with a
 * margin for the estimate, and `CHUNK_LIMITS.maxTokens` at most.
+*
+* @beta
 */
 const chunkTokensOf = (embedder) => {
 	const limit = embedder.maxInputTokens;
 	const tokens = limit === void 0 || !Number.isFinite(limit) || limit < 1 ? CHUNK_LIMITS.defaultTokens : Math.min(limit, CHUNK_LIMITS.maxTokens);
 	return Math.max(8, Math.floor(tokens * CHUNK_LIMITS.fill));
 };
-/** This function gives a copy of a vector with a length of 1. A vector of zeros stays zeros. A value that is not finite throws a `RangeError`. */
+/**
+* This function gives a copy of a vector with a length of 1. A vector of zeros stays zeros. A value that is not finite throws a `RangeError`.
+*
+* @beta
+*/
 const unitVector = (vector) => {
 	const out = Float32Array.from(vector);
 	let sum = 0;
@@ -2583,6 +2624,8 @@ const HYBRID_DEFAULTS = {
 * 1 / (k + rank) over the lists that hold it, with the rank from 1. The best item is first. A tie goes to
 * the item with the best rank in a list, then to the earlier list, then to the smaller id. An item
 * that is two times in one list counts at its first place.
+*
+* @beta
 */
 const fuseRanks = (lists, k = HYBRID_DEFAULTS.fusionK) => {
 	if (!Number.isFinite(k) || k < 0) throw new RangeError(`the fusion constant k must be a number of 0 or more, not ${String(k)}`);
@@ -2659,7 +2702,11 @@ const round = (value, digits) => {
 	const scale = 10 ** digits;
 	return Math.round(value * scale) / scale;
 };
-/** This function makes a hybrid search over a runtime. The vectors go into the store, with the id of the embedder. */
+/**
+* This function makes a hybrid search over a runtime. The vectors go into the store, with the id of the embedder.
+*
+* @beta
+*/
 const createHybridSearch = (runtime, options) => {
 	const { embedder, store } = options;
 	const tokens = chunkTokensOf(embedder);
@@ -3040,7 +3087,11 @@ async function* pagesOf(runtime) {
 * one transaction (`BEGIN IMMEDIATE`). The file has a schema version (`VECTORS_SCHEMA`). A file of
 * a different version opens empty.
 */
-/** This value is the version of the schema of the vector file. */
+/**
+* This value is the version of the schema of the vector file.
+*
+* @beta
+*/
 const VECTORS_SCHEMA = 1;
 const DDL = `
 CREATE TABLE IF NOT EXISTS vectors_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
@@ -3091,6 +3142,7 @@ const dot = (data, row, query, dims) => {
 	for (let at = 0; at < dims; at += 1) sum += data[base + at] * query[at];
 	return sum;
 };
+/** @beta */
 var VectorStore = class {
 	path;
 	db;
@@ -3496,6 +3548,8 @@ const targetOf = (url) => {
 *
 * The list is the decision of the operator, not a projection of the capture log. Thus it has its
 * own file: a deletion of `hosts.sqlite` or of the index does not lose it.
+*
+* @beta
 */
 var WatchStore = class {
 	path;
@@ -3695,6 +3749,8 @@ const sameState = (a, b) => a !== void 0 && a.state === b.state && (a.state ===
 * it. Each check is a fetch of the page with `fetchPage`, with no stored copy (`maxAgeMs: 0`), so
 * each polite rule applies. A check that the budget stops before its request stays due. The store
 * can be a function that opens it: the run uses it before the first check.
+*
+* @beta
 */
 const runWatch = (runtime, source, options = {}) => {
 	let resolveSummary = () => void 0;
@@ -3987,6 +4043,8 @@ const MAILBOX = /^(?:[^\s<>@:/]+@[^\s<>@/]+|[^<>]*<[^\s<>@:/]+@[^\s<>@/]+>)$/;
 * This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,
 * section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in
 * the `User-Agent` only.
+*
+* @beta
 */
 const isMailbox = (contact) => MAILBOX.test(contact.trim());
 /** The parts of a store that can be a symbolic link to a different place only by mistake or by attack. */
@@ -4024,6 +4082,8 @@ const guardStore = (dir) => {
 * This function makes a runtime. It is the one place that puts the log, the blob store, the index,
 * the host state, the crawler and the extractors together. The function `openCrawler` uses it.
 * An application that needs the parts uses it too. The runtime and the crawler share one clock.
+*
+* @beta
 */
 const createRuntime = (options) => {
 	const dir = options.dir;
@@ -4269,4 +4329,4 @@ const openCrawler = (options) => {
 //#endregion
 export { NO_EMBEDDER as A, DISCOVER_LIMITS as C, readSitemap as D, feedDate as E, DocIndex as O, crawlSite as S, SITEMAP_LIMITS as T, estimateTokens as _, open_exports as a, SqliteHostStateStore as b, WatchStore as c, VectorStore as d, createHybridSearch as f, chunkTokensOf as g, chunkText as h, openCrawler as i, FAST_RANK_SHARE as k, runWatch as l, HYBRID_DEFAULTS as m, defaultUserAgent as n, WATCH_LIMITS as o, CHUNK_LIMITS as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, VECTORS_SCHEMA as u, fuseRanks as v, discoverSite as w, CRAWL_LIMITS as x, unitVector as y };
 
-//# sourceMappingURL=open-CbpSCEZc.mjs.map
+//# sourceMappingURL=open-CEo22Hpu.mjs.map