polite-crawler 0.1.3 → 0.2.0

package/build/{log-Dx-wXjJl.mjs → log-CmBWcyQm.mjs}

@@ -40,6 +40,8 @@ const syncFolder = async (path) => {
 *   other was first, the put succeeds if the stored file has the same hash.
 * - The method `get` examines the hash of the bytes. A damaged file gets a new name
 *   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.
+*
+* @beta
 */
 var FileBlobStore = class FileBlobStore {
 	dir;
@@ -175,6 +177,8 @@ const monthKey = (year, file) => `${year}-${MONTH_FILE.exec(file)?.[1] ?? "00"}`
 * - The log does not remove or change a line.
 *
 * Two processes can append to one log, because each batch is one write to a file in append mode.
+*
+* @beta
 */
 var CaptureLog = class CaptureLog {
 	dir;
@@ -339,4 +343,4 @@ var CaptureLog = class CaptureLog {
 //#endregion
 export { FileBlobStore as n, CaptureLog as t };
 
-//# sourceMappingURL=log-Dx-wXjJl.mjs.map
+//# sourceMappingURL=log-CmBWcyQm.mjs.map

package/build/log-CmBWcyQm.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"log-CmBWcyQm.mjs","names":[],"sources":["../src/blobs.ts","../src/log.ts"],"sourcesContent":["import { createHash, randomUUID } from \"node:crypto\";\nimport type { Stats } from \"node:fs\";\nimport { mkdir, open, readFile, rename, rm, stat } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\n\n/**\n * A blob store keeps the bytes of each response, and the hash of the bytes is the key. The crawler\n * needs only these two operations, so an application can give its own store.\n */\nexport interface BlobStore {\n  put(bytes: Uint8Array): Promise<{ readonly hash: string }>;\n  get(hash: string): Promise<Uint8Array | undefined>;\n}\n\n/**\n * These options change a file blob store.\n *\n * @beta\n */\nexport interface FileBlobStoreOptions {\n  /** When this value is true (the default), a blob is on the disk (`fsync`) before `put` finishes. */\n  readonly sync?: boolean | undefined;\n}\n\nconst HASH = /^[0-9a-f]{64}$/;\n\n/** A rename can fail with these codes when a different writer puts the same blob at the same time. */\nconst RACE_CODES: ReadonlySet<string> = new Set([\n  \"EPERM\",\n  \"EEXIST\",\n  \"EBUSY\",\n  \"EACCES\",\n  \"ENOTEMPTY\",\n]);\n\nconst blobPath = (root: string, hash: string): string =>\n  join(root, hash.slice(0, 2), hash.slice(2, 4), hash);\n\nconst hashOf = (bytes: Uint8Array): string => createHash(\"sha256\").update(bytes).digest(\"hex\");\n\nconst codeOf = (error: unknown): string =>\n  typeof error === \"object\" && error !== null && \"code\" in error ? String(error.code) : \"\";\n\nconst statOf = (path: string): Promise<Stats | undefined> => stat(path).catch(() => undefined);\n\n/** This function writes a folder to the disk, so a rename in it stays after a power cut. Windows cannot open a folder, so it skips this step. */\nconst syncFolder = async (path: string): Promise<void> => {\n  if (process.platform === \"win32\") return;\n  try {\n    const handle = await open(path, \"r\");\n    try {\n      await handle.sync();\n    } finally {\n      await handle.close();\n    }\n  } catch {\n    // a file system that cannot sync a folder keeps the blob in any case\n  }\n};\n\n/**\n * This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash\n * of the bytes. These are its rules:\n *\n * - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is\n *   written again.\n * - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.\n *   A crash thus cannot leave half a blob under its name.\n * - Two writers can put the same bytes at the same time. When the rename of one fails because the\n *   other was first, the put succeeds if the stored file has the same hash.\n * - The method `get` examines the hash of the bytes. A damaged file gets a new name\n *   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.\n *\n * @beta\n */\nexport class FileBlobStore implements BlobStore {\n  private readonly writing = new Map<\n    string,\n    Promise<{ readonly hash: string; readonly existed: boolean }>\n  >();\n  private readonly sync: boolean;\n\n  constructor(\n    readonly dir: string,\n    options: FileBlobStoreOptions = {},\n  ) {\n    this.sync = options.sync ?? true;\n  }\n\n  put(bytes: Uint8Array): Promise<{ readonly hash: string; readonly existed: boolean }> {\n    const hash = hashOf(bytes);\n    const running = this.writing.get(hash);\n    if (running !== undefined) return running;\n    const work = this.write(hash, bytes).finally(() => this.writing.delete(hash));\n    this.writing.set(hash, work);\n    return work;\n  }\n\n  /** This method tells if a file holds exactly the bytes of a hash. */\n  private static async holds(path: string, hash: string): Promise<boolean> {\n    try {\n      return hashOf(await readFile(path)) === hash;\n    } catch {\n      return false;\n    }\n  }\n\n  private async write(\n    hash: string,\n    bytes: Uint8Array,\n  ): Promise<{ readonly hash: string; readonly existed: boolean }> {\n    const target = blobPath(this.dir, hash);\n    const existing = await statOf(target);\n    if (existing?.isFile() === true && existing.size === bytes.length)\n      return { hash, existed: true };\n    const folder = dirname(target);\n    await mkdir(folder, { recursive: true, mode: 0o700 });\n    const temporary = `${target}.${randomUUID()}.tmp`;\n    try {\n      const handle = await open(temporary, \"wx\", 0o600);\n      try {\n        await handle.writeFile(bytes);\n        if (this.sync) await handle.sync();\n      } finally {\n        await handle.close();\n      }\n      try {\n        await rename(temporary, target);\n      } catch (error) {\n        // a different writer put the same bytes first, and Windows does not replace a file in use\n        if (!RACE_CODES.has(codeOf(error)) || !(await FileBlobStore.holds(target, hash)))\n          throw error;\n        return { hash, existed: true };\n      }\n      if (this.sync) await syncFolder(folder);\n      return { hash, existed: false };\n    } finally {\n      await rm(temporary, { force: true }).catch(() => undefined);\n    }\n  }\n\n  async get(hash: string): Promise<Uint8Array | undefined> {\n    if (!HASH.test(hash)) return undefined;\n    const target = blobPath(this.dir, hash);\n    let bytes: Buffer;\n    try {\n      bytes = await readFile(target);\n    } catch {\n      return undefined;\n    }\n    if (hashOf(bytes) !== hash) {\n      // a torn or damaged file is never the body: it moves away, and the next put writes it again\n      await rename(target, `${target}.${randomUUID()}.damaged`).catch(() => undefined);\n      return undefined;\n    }\n    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n  }\n}\n","import { createReadStream, existsSync } from \"node:fs\";\nimport type { FileHandle } from \"node:fs/promises\";\nimport { mkdir, open, readdir } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\nimport type { CaptureRecord } from \"./record.ts\";\n\n/** A scan filter selects the records of one source, one kind, or the records after one time. */\nexport interface CaptureFilter {\n  readonly source?: string | undefined;\n  readonly kind?: string | undefined;\n  /** This value is an ISO 8601 time. The scan gives the records received at this time or later. */\n  readonly since?: string | undefined;\n}\n\n/** A damaged line is a line of the log that is not a record: a torn append, or bytes that a crash left. */\nexport interface DamagedLine {\n  /** This value is the path of the file. */\n  readonly path: string;\n  /** This value is the number of the line in the file, from 1. */\n  readonly line: number;\n}\n\n/** These options change a scan. */\nexport interface ScanOptions {\n  /** The scan skips a damaged line, and it gives the line to this function. */\n  readonly onDamaged?: ((damaged: DamagedLine) => void) | undefined;\n}\n\n/** The read side of the capture log is what `reindex` replays. */\nexport interface CaptureReader {\n  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;\n}\n\n/**\n * These options change a capture log.\n *\n * @beta\n */\nexport interface CaptureLogOptions {\n  /**\n   * When this value is true (the default), the log writes each batch of appends to the disk\n   * (`fdatasync`) before the appends finish. Thus a power cut does not lose a record that the\n   * crawler reported.\n   */\n  readonly sync?: boolean | undefined;\n}\n\ninterface Pending {\n  readonly path: string;\n  readonly line: string;\n  readonly resolve: () => void;\n  readonly reject: (error: unknown) => void;\n}\n\nconst MONTH_FILE = /^(\\d{2})(?:\\.[^.]+)?\\.ndjson$/;\nconst YEAR_DIR = /^\\d{4}$/;\nconst NEWLINE = 0x0a;\n\n/** This function writes the whole buffer. A write to a file can write less than the buffer, and the loop writes the rest. */\nconst writeAll = async (handle: FileHandle, bytes: Buffer): Promise<void> => {\n  let at = 0;\n  while (at < bytes.length) {\n    const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);\n    at += bytesWritten;\n  }\n};\n\n/**\n * This function reads a file line by line, as bytes, so a long file does not become one string. It\n * gives each line and tells if a line break ends it.\n */\nasync function* linesOf(\n  path: string,\n): AsyncGenerator<{ readonly text: string; readonly complete: boolean }> {\n  const decoder = new TextDecoder(\"utf-8\");\n  let rest: Buffer | undefined;\n  for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {\n    const data = rest === undefined ? chunk : Buffer.concat([rest, chunk]);\n    rest = undefined;\n    let start = 0;\n    for (;;) {\n      const end = data.indexOf(NEWLINE, start);\n      if (end === -1) break;\n      yield { text: decoder.decode(data.subarray(start, end)), complete: true };\n      start = end + 1;\n    }\n    if (start < data.length) rest = Buffer.from(data.subarray(start));\n  }\n  if (rest !== undefined && rest.length > 0) yield { text: decoder.decode(rest), complete: false };\n}\n\nconst monthKey = (year: string, file: string): string =>\n  `${year}-${MONTH_FILE.exec(file)?.[1] ?? \"00\"}`;\n\n/**\n * The capture log keeps each fetch decision as one line of JSON in\n * `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.\n *\n * These are the rules of the log:\n *\n * - The log appends a batch of records with one write, and it writes the batch to the disk before\n *   the appends finish.\n * - An append always starts on a new line. After a crash leaves a partial line, the next append\n *   starts after a line break. The partial line stays as a damaged line, and the record is good.\n * - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.\n * - The log does not remove or change a line.\n *\n * Two processes can append to one log, because each batch is one write to a file in append mode.\n *\n * @beta\n */\nexport class CaptureLog implements CaptureReader {\n  private readonly handles = new Map<string, FileHandle>();\n  private readonly pending: Pending[] = [];\n  private draining: Promise<void> | undefined;\n  private readonly sync: boolean;\n\n  constructor(\n    readonly dir: string,\n    options: CaptureLogOptions = {},\n  ) {\n    this.sync = options.sync ?? true;\n  }\n\n  private pathOf(record: CaptureRecord): string {\n    return join(\n      this.dir,\n      record.source,\n      record.received_at.slice(0, 4),\n      `${record.received_at.slice(5, 7)}.ndjson`,\n    );\n  }\n\n  private async handleFor(path: string): Promise<FileHandle> {\n    const known = this.handles.get(path);\n    if (known !== undefined) return known;\n    // the folders and the files are for the owner only, where the system has modes\n    await mkdir(dirname(path), { recursive: true, mode: 0o700 });\n    const handle = await open(path, \"a+\", 0o600);\n    this.handles.set(path, handle);\n    return handle;\n  }\n\n  /** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */\n  private static async needsBreak(handle: FileHandle): Promise<boolean> {\n    const { size } = await handle.stat();\n    if (size === 0) return false;\n    const last = Buffer.alloc(1);\n    await handle.read(last, 0, 1, size - 1);\n    return last[0] !== NEWLINE;\n  }\n\n  append(record: CaptureRecord): Promise<void> {\n    let line: string;\n    try {\n      line = `${JSON.stringify(record)}\\n`;\n    } catch (error) {\n      return Promise.reject(error instanceof Error ? error : new Error(String(error)));\n    }\n    return new Promise<void>((resolve, reject) => {\n      this.pending.push({ path: this.pathOf(record), line, resolve, reject });\n      this.kick();\n    });\n  }\n\n  /** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */\n  private kick(): void {\n    if (this.draining !== undefined) return;\n    this.draining = this.drain().finally(() => {\n      this.draining = undefined;\n      if (this.pending.length > 0) this.kick();\n    });\n  }\n\n  /** This method writes the waiting records in batches: one write and one sync for each file. */\n  private async drain(): Promise<void> {\n    while (this.pending.length > 0) {\n      const batch = this.pending.splice(0);\n      const byPath = new Map<string, Pending[]>();\n      for (const item of batch) {\n        const list = byPath.get(item.path);\n        if (list === undefined) byPath.set(item.path, [item]);\n        else list.push(item);\n      }\n      for (const [path, items] of byPath) {\n        try {\n          const handle = await this.handleFor(path);\n          const lead = (await CaptureLog.needsBreak(handle)) ? \"\\n\" : \"\";\n          await writeAll(\n            handle,\n            Buffer.from(`${lead}${items.map((item) => item.line).join(\"\")}`, \"utf8\"),\n          );\n          if (this.sync) await handle.datasync();\n          for (const item of items) item.resolve();\n        } catch (error) {\n          // a handle that failed is opened again for the next append\n          const handle = this.handles.get(path);\n          this.handles.delete(path);\n          await handle?.close().catch(() => undefined);\n          for (const item of items) item.reject(error);\n        }\n      }\n    }\n  }\n\n  /** This method waits until each append in progress is on the disk. */\n  async flush(): Promise<void> {\n    while (this.draining !== undefined) await this.draining;\n  }\n\n  /** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */\n  private async filesOf(source: string, since: string | undefined): Promise<string[]> {\n    const sourceDir = join(this.dir, source);\n    if (!existsSync(sourceDir)) return [];\n    const from = since?.slice(0, 7);\n    const years = (await readdir(sourceDir, { withFileTypes: true }))\n      .filter((entry) => entry.isDirectory() && YEAR_DIR.test(entry.name))\n      .map((entry) => entry.name)\n      .sort();\n    const files: string[] = [];\n    for (const year of years) {\n      if (from !== undefined && year < from.slice(0, 4)) continue;\n      const months = (await readdir(join(sourceDir, year)))\n        .filter((name) => MONTH_FILE.test(name))\n        .sort();\n      for (const month of months) {\n        if (from !== undefined && monthKey(year, month) < from) continue;\n        files.push(join(sourceDir, year, month));\n      }\n    }\n    return files;\n  }\n\n  /** This method reads the records of one source in the sequence of the files. */\n  private async *readSource(\n    source: string,\n    filter: CaptureFilter | undefined,\n    options: ScanOptions | undefined,\n  ): AsyncGenerator<CaptureRecord> {\n    for (const path of await this.filesOf(source, filter?.since)) {\n      let number = 0;\n      for await (const { text, complete } of linesOf(path)) {\n        number += 1;\n        const trimmed = text.trim();\n        if (trimmed.length === 0) continue;\n        let row: CaptureRecord;\n        try {\n          row = JSON.parse(trimmed) as CaptureRecord;\n          if (typeof row !== \"object\" || row === null || typeof row.received_at !== \"string\") {\n            throw new Error(\"not a record\");\n          }\n        } catch {\n          // the append of a different process can be in progress at the end of the file\n          if (complete) options?.onDamaged?.({ path, line: number });\n          continue;\n        }\n        if (filter?.kind !== undefined && row.kind !== filter.kind) continue;\n        if (filter?.since !== undefined && row.received_at < filter.since) continue;\n        yield row;\n      }\n    }\n  }\n\n  /**\n   * This method reads the records that agree with the filter. It first waits for the appends that\n   * are in progress. It reads each file as a stream, so its memory does not grow with the log.\n   *\n   * The records of one source come in the sequence of the files. The records of two or more sources\n   * merge in the sequence of `received_at`.\n   */\n  async *scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord> {\n    await this.flush();\n    const sources =\n      filter?.source !== undefined\n        ? [filter.source]\n        : existsSync(this.dir)\n          ? (await readdir(this.dir, { withFileTypes: true }))\n              .filter((entry) => entry.isDirectory())\n              .map((entry) => entry.name)\n              .sort()\n          : [];\n    if (sources.length === 1) {\n      yield* this.readSource(sources[0]!, filter, options);\n      return;\n    }\n    // a merge of the sources: the next record is the earliest head\n    const streams = sources.map((source) =>\n      this.readSource(source, filter, options)[Symbol.asyncIterator](),\n    );\n    const heads: Array<CaptureRecord | undefined> = [];\n    for (const stream of streams)\n      heads.push((await stream.next()).value as CaptureRecord | undefined);\n    for (;;) {\n      let pick = -1;\n      for (const [index, head] of heads.entries()) {\n        if (head !== undefined && (pick === -1 || head.received_at < heads[pick]!.received_at)) {\n          pick = index;\n        }\n      }\n      if (pick === -1) return;\n      yield heads[pick]!;\n      heads[pick] = (await streams[pick]!.next()).value as CaptureRecord | undefined;\n    }\n  }\n\n  /** This method writes the appends in progress, then closes the files. A later append opens them again. */\n  async close(): Promise<void> {\n    await this.flush();\n    const handles = [...this.handles.values()];\n    this.handles.clear();\n    for (const handle of handles) await handle.close();\n  }\n}\n"],"mappings":";;;;;AAwBA,MAAM,OAAO;;AAGb,MAAM,6BAAkC,IAAI,IAAI;CAC9C;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,YAAY,MAAc,SAC9B,KAAK,MAAM,KAAK,MAAM,GAAG,CAAC,GAAG,KAAK,MAAM,GAAG,CAAC,GAAG,IAAI;AAErD,MAAM,UAAU,UAA8B,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK;AAE7F,MAAM,UAAU,UACd,OAAO,UAAU,YAAY,UAAU,QAAQ,UAAU,QAAQ,OAAO,MAAM,IAAI,IAAI;AAExF,MAAM,UAAU,SAA6C,KAAK,IAAI,CAAC,CAAC,YAAY,KAAA,CAAS;;AAG7F,MAAM,aAAa,OAAO,SAAgC;CACxD,IAAI,QAAQ,aAAa,SAAS;CAClC,IAAI;EACF,MAAM,SAAS,MAAM,KAAK,MAAM,GAAG;EACnC,IAAI;GACF,MAAM,OAAO,KAAK;EACpB,UAAU;GACR,MAAM,OAAO,MAAM;EACrB;CACF,QAAQ,CAER;AACF;;;;;;;;;;;;;;;;AAiBA,IAAa,gBAAb,MAAa,cAAmC;CAQnC;CAPX,0BAA2B,IAAI,IAG7B;CACF;CAEA,YACE,KACA,UAAgC,CAAC,GACjC;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,IAAI,OAAkF;EACpF,MAAM,OAAO,OAAO,KAAK;EACzB,MAAM,UAAU,KAAK,QAAQ,IAAI,IAAI;EACrC,IAAI,YAAY,KAAA,GAAW,OAAO;EAClC,MAAM,OAAO,KAAK,MAAM,MAAM,KAAK,CAAC,CAAC,cAAc,KAAK,QAAQ,OAAO,IAAI,CAAC;EAC5E,KAAK,QAAQ,IAAI,MAAM,IAAI;EAC3B,OAAO;CACT;;CAGA,aAAqB,MAAM,MAAc,MAAgC;EACvE,IAAI;GACF,OAAO,OAAO,MAAM,SAAS,IAAI,CAAC,MAAM;EAC1C,QAAQ;GACN,OAAO;EACT;CACF;CAEA,MAAc,MACZ,MACA,OAC+D;EAC/D,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,MAAM,WAAW,MAAM,OAAO,MAAM;EACpC,IAAI,UAAU,OAAO,MAAM,QAAQ,SAAS,SAAS,MAAM,QACzD,OAAO;GAAE;GAAM,SAAS;EAAK;EAC/B,MAAM,SAAS,QAAQ,MAAM;EAC7B,MAAM,MAAM,QAAQ;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EACpD,MAAM,YAAY,GAAG,OAAO,GAAG,WAAW,EAAE;EAC5C,IAAI;GACF,MAAM,SAAS,MAAM,KAAK,WAAW,MAAM,GAAK;GAChD,IAAI;IACF,MAAM,OAAO,UAAU,KAAK;IAC5B,IAAI,KAAK,MAAM,MAAM,OAAO,KAAK;GACnC,UAAU;IACR,MAAM,OAAO,MAAM;GACrB;GACA,IAAI;IACF,MAAM,OAAO,WAAW,MAAM;GAChC,SAAS,OAAO;IAEd,IAAI,CAAC,WAAW,IAAI,OAAO,KAAK,CAAC,KAAK,CAAE,MAAM,cAAc,MAAM,QAAQ,IAAI,GAC5E,MAAM;IACR,OAAO;KAAE;KAAM,SAAS;IAAK;GAC/B;GACA,IAAI,KAAK,MAAM,MAAM,WAAW,MAAM;GACtC,OAAO;IAAE;IAAM,SAAS;GAAM;EAChC,UAAU;GACR,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;EAC5D;CACF;CAEA,MAAM,IAAI,MAA+C;EACvD,IAAI,CAAC,KAAK,KAAK,IAAI,GAAG,OAAO,KAAA;EAC7B,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,IAAI;EACJ,IAAI;GACF,QAAQ,MAAM,SAAS,MAAM;EAC/B,QAAQ;GACN;EACF;EACA,IAAI,OAAO,KAAK,MAAM,MAAM;GAE1B,MAAM,OAAO,QAAQ,GAAG,OAAO,GAAG,WAAW,EAAE,SAAS,CAAC,CAAC,YAAY,KAAA,CAAS;GAC/E;EACF;EACA,OAAO,IAAI,WAAW,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;CACxE;AACF;;;ACvGA,MAAM,aAAa;AACnB,MAAM,WAAW;AACjB,MAAM,UAAU;;AAGhB,MAAM,WAAW,OAAO,QAAoB,UAAiC;CAC3E,IAAI,KAAK;CACT,OAAO,KAAK,MAAM,QAAQ;EACxB,MAAM,EAAE,iBAAiB,MAAM,OAAO,MAAM,OAAO,IAAI,MAAM,SAAS,EAAE;EACxE,MAAM;CACR;AACF;;;;;AAMA,gBAAgB,QACd,MACuE;CACvE,MAAM,UAAU,IAAI,YAAY,OAAO;CACvC,IAAI;CACJ,WAAW,MAAM,SAAS,iBAAiB,IAAI,GAA4B;EACzE,MAAM,OAAO,SAAS,KAAA,IAAY,QAAQ,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;EACrE,OAAO,KAAA;EACP,IAAI,QAAQ;EACZ,SAAS;GACP,MAAM,MAAM,KAAK,QAAQ,SAAS,KAAK;GACvC,IAAI,QAAQ,IAAI;GAChB,MAAM;IAAE,MAAM,QAAQ,OAAO,KAAK,SAAS,OAAO,GAAG,CAAC;IAAG,UAAU;GAAK;GACxE,QAAQ,MAAM;EAChB;EACA,IAAI,QAAQ,KAAK,QAAQ,OAAO,OAAO,KAAK,KAAK,SAAS,KAAK,CAAC;CAClE;CACA,IAAI,SAAS,KAAA,KAAa,KAAK,SAAS,GAAG,MAAM;EAAE,MAAM,QAAQ,OAAO,IAAI;EAAG,UAAU;CAAM;AACjG;AAEA,MAAM,YAAY,MAAc,SAC9B,GAAG,KAAK,GAAG,WAAW,KAAK,IAAI,CAAC,GAAG,MAAM;;;;;;;;;;;;;;;;;;AAmB3C,IAAa,aAAb,MAAa,WAAoC;CAOpC;CANX,0BAA2B,IAAI,IAAwB;CACvD,UAAsC,CAAC;CACvC;CACA;CAEA,YACE,KACA,UAA6B,CAAC,GAC9B;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,OAAe,QAA+B;EAC5C,OAAO,KACL,KAAK,KACL,OAAO,QACP,OAAO,YAAY,MAAM,GAAG,CAAC,GAC7B,GAAG,OAAO,YAAY,MAAM,GAAG,CAAC,EAAE,QACpC;CACF;CAEA,MAAc,UAAU,MAAmC;EACzD,MAAM,QAAQ,KAAK,QAAQ,IAAI,IAAI;EACnC,IAAI,UAAU,KAAA,GAAW,OAAO;EAEhC,MAAM,MAAM,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAC3D,MAAM,SAAS,MAAM,KAAK,MAAM,MAAM,GAAK;EAC3C,KAAK,QAAQ,IAAI,MAAM,MAAM;EAC7B,OAAO;CACT;;CAGA,aAAqB,WAAW,QAAsC;EACpE,MAAM,EAAE,SAAS,MAAM,OAAO,KAAK;EACnC,IAAI,SAAS,GAAG,OAAO;EACvB,MAAM,OAAO,OAAO,MAAM,CAAC;EAC3B,MAAM,OAAO,KAAK,MAAM,GAAG,GAAG,OAAO,CAAC;EACtC,OAAO,KAAK,OAAO;CACrB;CAEA,OAAO,QAAsC;EAC3C,IAAI;EACJ,IAAI;GACF,OAAO,GAAG,KAAK,UAAU,MAAM,EAAE;EACnC,SAAS,OAAO;GACd,OAAO,QAAQ,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC,CAAC;EACjF;EACA,OAAO,IAAI,SAAe,SAAS,WAAW;GAC5C,KAAK,QAAQ,KAAK;IAAE,MAAM,KAAK,OAAO,MAAM;IAAG;IAAM;IAAS;GAAO,CAAC;GACtE,KAAK,KAAK;EACZ,CAAC;CACH;;CAGA,OAAqB;EACnB,IAAI,KAAK,aAAa,KAAA,GAAW;EACjC,KAAK,WAAW,KAAK,MAAM,CAAC,CAAC,cAAc;GACzC,KAAK,WAAW,KAAA;GAChB,IAAI,KAAK,QAAQ,SAAS,GAAG,KAAK,KAAK;EACzC,CAAC;CACH;;CAGA,MAAc,QAAuB;EACnC,OAAO,KAAK,QAAQ,SAAS,GAAG;GAC9B,MAAM,QAAQ,KAAK,QAAQ,OAAO,CAAC;GACnC,MAAM,yBAAS,IAAI,IAAuB;GAC1C,KAAK,MAAM,QAAQ,OAAO;IACxB,MAAM,OAAO,OAAO,IAAI,KAAK,IAAI;IACjC,IAAI,SAAS,KAAA,GAAW,OAAO,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;SAC/C,KAAK,KAAK,IAAI;GACrB;GACA,KAAK,MAAM,CAAC,MAAM,UAAU,QAC1B,IAAI;IACF,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;IACxC,MAAM,OAAQ,MAAM,WAAW,WAAW,MAAM,IAAK,OAAO;IAC5D,MAAM,SACJ,QACA,OAAO,KAAK,GAAG,OAAO,MAAM,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,MAAM,CACzE;IACA,IAAI,KAAK,MAAM,MAAM,OAAO,SAAS;IACrC,KAAK,MAAM,QAAQ,OAAO,KAAK,QAAQ;GACzC,SAAS,OAAO;IAEd,MAAM,SAAS,KAAK,QAAQ,IAAI,IAAI;IACpC,KAAK,QAAQ,OAAO,IAAI;IACxB,MAAM,QAAQ,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;IAC3C,KAAK,MAAM,QAAQ,OAAO,KAAK,OAAO,KAAK;GAC7C;EAEJ;CACF;;CAGA,MAAM,QAAuB;EAC3B,OAAO,KAAK,aAAa,KAAA,GAAW,MAAM,KAAK;CACjD;;CAGA,MAAc,QAAQ,QAAgB,OAA8C;EAClF,MAAM,YAAY,KAAK,KAAK,KAAK,MAAM;EACvC,IAAI,CAAC,WAAW,SAAS,GAAG,OAAO,CAAC;EACpC,MAAM,OAAO,OAAO,MAAM,GAAG,CAAC;EAC9B,MAAM,SAAS,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC,EAAA,CAC5D,QAAQ,UAAU,MAAM,YAAY,KAAK,SAAS,KAAK,MAAM,IAAI,CAAC,CAAC,CACnE,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK;EACR,MAAM,QAAkB,CAAC;EACzB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,SAAS,KAAA,KAAa,OAAO,KAAK,MAAM,GAAG,CAAC,GAAG;GACnD,MAAM,UAAU,MAAM,QAAQ,KAAK,WAAW,IAAI,CAAC,EAAA,CAChD,QAAQ,SAAS,WAAW,KAAK,IAAI,CAAC,CAAC,CACvC,KAAK;GACR,KAAK,MAAM,SAAS,QAAQ;IAC1B,IAAI,SAAS,KAAA,KAAa,SAAS,MAAM,KAAK,IAAI,MAAM;IACxD,MAAM,KAAK,KAAK,WAAW,MAAM,KAAK,CAAC;GACzC;EACF;EACA,OAAO;CACT;;CAGA,OAAe,WACb,QACA,QACA,SAC+B;EAC/B,KAAK,MAAM,QAAQ,MAAM,KAAK,QAAQ,QAAQ,QAAQ,KAAK,GAAG;GAC5D,IAAI,SAAS;GACb,WAAW,MAAM,EAAE,MAAM,cAAc,QAAQ,IAAI,GAAG;IACpD,UAAU;IACV,MAAM,UAAU,KAAK,KAAK;IAC1B,IAAI,QAAQ,WAAW,GAAG;IAC1B,IAAI;IACJ,IAAI;KACF,MAAM,KAAK,MAAM,OAAO;KACxB,IAAI,OAAO,QAAQ,YAAY,QAAQ,QAAQ,OAAO,IAAI,gBAAgB,UACxE,MAAM,IAAI,MAAM,cAAc;IAElC,QAAQ;KAEN,IAAI,UAAU,SAAS,YAAY;MAAE;MAAM,MAAM;KAAO,CAAC;KACzD;IACF;IACA,IAAI,QAAQ,SAAS,KAAA,KAAa,IAAI,SAAS,OAAO,MAAM;IAC5D,IAAI,QAAQ,UAAU,KAAA,KAAa,IAAI,cAAc,OAAO,OAAO;IACnE,MAAM;GACR;EACF;CACF;;;;;;;;CASA,OAAO,KAAK,QAAwB,SAAqD;EACvF,MAAM,KAAK,MAAM;EACjB,MAAM,UACJ,QAAQ,WAAW,KAAA,IACf,CAAC,OAAO,MAAM,IACd,WAAW,KAAK,GAAG,KAChB,MAAM,QAAQ,KAAK,KAAK,EAAE,eAAe,KAAK,CAAC,EAAA,CAC7C,QAAQ,UAAU,MAAM,YAAY,CAAC,CAAC,CACtC,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK,IACR,CAAC;EACT,IAAI,QAAQ,WAAW,GAAG;GACxB,OAAO,KAAK,WAAW,QAAQ,IAAK,QAAQ,OAAO;GACnD;EACF;EAEA,MAAM,UAAU,QAAQ,KAAK,WAC3B,KAAK,WAAW,QAAQ,QAAQ,OAAO,CAAC,CAAC,OAAO,cAAc,CAAC,CACjE;EACA,MAAM,QAA0C,CAAC;EACjD,KAAK,MAAM,UAAU,SACnB,MAAM,MAAM,MAAM,OAAO,KAAK,EAAA,CAAG,KAAkC;EACrE,SAAS;GACP,IAAI,OAAO;GACX,KAAK,MAAM,CAAC,OAAO,SAAS,MAAM,QAAQ,GACxC,IAAI,SAAS,KAAA,MAAc,SAAS,MAAM,KAAK,cAAc,MAAM,KAAK,CAAE,cACxE,OAAO;GAGX,IAAI,SAAS,IAAI;GACjB,MAAM,MAAM;GACZ,MAAM,SAAS,MAAM,QAAQ,KAAK,CAAE,KAAK,EAAA,CAAG;EAC9C;CACF;;CAGA,MAAM,QAAuB;EAC3B,MAAM,KAAK,MAAM;EACjB,MAAM,UAAU,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC;EACzC,KAAK,QAAQ,MAAM;EACnB,KAAK,MAAM,UAAU,SAAS,MAAM,OAAO,MAAM;CACnD;AACF"}

package/build/mcp-package-Y0bREnOS.mjs

@@ -0,0 +1,40 @@
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+//#region src/mcp-package.ts
+/** The name of the package of the MCP server. */
+const MCP_PACKAGE = "polite-crawler-mcp";
+/** This text tells where the MCP server is. */
+const MCP_POINTER = "The MCP server of polite-crawler is the package polite-crawler-mcp since 0.2.0. Start it with npx polite-crawler-mcp, or install polite-crawler-mcp next to polite-crawler.";
+/** This error tells that polite-crawler-mcp is not there. Its message is the pointer. */
+var McpPackageMissing = class extends Error {
+	constructor() {
+		super(MCP_POINTER);
+		this.name = "McpPackageMissing";
+	}
+};
+/**
+* This function gives the places where polite-crawler looks for polite-crawler-mcp, as file URLs:
+* only next to polite-crawler.
+*/
+const mcpPackagePlaces = () => [import.meta.url];
+/**
+* This function loads polite-crawler-mcp from the first place that has it. It throws
+* `McpPackageMissing` when no place has it. A test gives another `name`: Node.js also looks in the
+* folders of `NODE_PATH`, and pnpm sets it to the packages of the workspace.
+*/
+const loadMcpPackage = async (places = mcpPackagePlaces(), name = MCP_PACKAGE) => {
+	for (const from of places) {
+		let path;
+		try {
+			path = createRequire(from).resolve(name);
+		} catch {
+			continue;
+		}
+		return await import(pathToFileURL(path).href);
+	}
+	throw new McpPackageMissing();
+};
+//#endregion
+export { McpPackageMissing, loadMcpPackage };
+
+//# sourceMappingURL=mcp-package-Y0bREnOS.mjs.map

package/build/mcp-package-Y0bREnOS.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp-package-Y0bREnOS.mjs","names":[],"sources":["../src/mcp-package.ts"],"sourcesContent":["import { createRequire } from \"node:module\";\nimport { pathToFileURL } from \"node:url\";\nimport type { DaemonMcpHost } from \"./daemon.ts\";\nimport type { OpenOptions } from \"./open.ts\";\n\n/*\n * The MCP server of polite-crawler is the package polite-crawler-mcp since polite-crawler 0.2.0.\n * polite-crawler does not depend on it, so the library loads no MCP SDK and no zod. The command\n * `polite-crawler mcp` and the MCP server of the daemon load the package when it is there: next to\n * polite-crawler. They do not look in the working folder: an MCP client can start the command in any\n * project, and the code of that project must not start.\n */\n\n/** The name of the package of the MCP server. */\nexport const MCP_PACKAGE = \"polite-crawler-mcp\";\n\n/** This text tells where the MCP server is. */\nexport const MCP_POINTER =\n  \"The MCP server of polite-crawler is the package polite-crawler-mcp since 0.2.0. Start it with npx polite-crawler-mcp, or install polite-crawler-mcp next to polite-crawler.\";\n\n/** The parts of polite-crawler-mcp that polite-crawler uses. */\nexport interface McpPackage {\n  startMcpServer(\n    options: OpenOptions & {\n      readonly allowUserProvided?: boolean | undefined;\n      readonly log?: ((line: string) => void) | undefined;\n    },\n  ): Promise<void>;\n  readonly mcpHttpHost: DaemonMcpHost;\n}\n\n/** This error tells that polite-crawler-mcp is not there. Its message is the pointer. */\nexport class McpPackageMissing extends Error {\n  constructor() {\n    super(MCP_POINTER);\n    this.name = \"McpPackageMissing\";\n  }\n}\n\n/**\n * This function gives the places where polite-crawler looks for polite-crawler-mcp, as file URLs:\n * only next to polite-crawler.\n */\nexport const mcpPackagePlaces = (): string[] => [import.meta.url];\n\n/**\n * This function loads polite-crawler-mcp from the first place that has it. It throws\n * `McpPackageMissing` when no place has it. A test gives another `name`: Node.js also looks in the\n * folders of `NODE_PATH`, and pnpm sets it to the packages of the workspace.\n */\nexport const loadMcpPackage = async (\n  places: readonly string[] = mcpPackagePlaces(),\n  name: string = MCP_PACKAGE,\n): Promise<McpPackage> => {\n  for (const from of places) {\n    let path: string;\n    try {\n      path = createRequire(from).resolve(name);\n    } catch {\n      continue;\n    }\n    return (await import(pathToFileURL(path).href)) as McpPackage;\n  }\n  throw new McpPackageMissing();\n};\n"],"mappings":";;;;AAcA,MAAa,cAAc;;AAG3B,MAAa,cACX;;AAcF,IAAa,oBAAb,cAAuC,MAAM;CAC3C,cAAc;EACZ,MAAM,WAAW;EACjB,KAAK,OAAO;CACd;AACF;;;;;AAMA,MAAa,yBAAmC,CAAC,YAAY,GAAG;;;;;;AAOhE,MAAa,iBAAiB,OAC5B,SAA4B,iBAAiB,GAC7C,OAAe,gBACS;CACxB,KAAK,MAAM,QAAQ,QAAQ;EACzB,IAAI;EACJ,IAAI;GACF,OAAO,cAAc,IAAI,CAAC,CAAC,QAAQ,IAAI;EACzC,QAAQ;GACN;EACF;EACA,OAAQ,MAAM,OAAO,cAAc,IAAI,CAAC,CAAC;CAC3C;CACA,MAAM,IAAI,kBAAkB;AAC9B"}

package/build/{ollama-BJxkjny4.mjs → ollama-D1xRh3hY.mjs}

@@ -1,4 +1,4 @@
-import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
 import { createHash } from "node:crypto";
 //#region src/ollama.ts
 var ollama_exports = /* @__PURE__ */ __exportAll({
@@ -27,7 +27,11 @@ const OLLAMA_MODELS = {
 };
 /** The text of the error when no Ollama server answers. */
 const NO_OLLAMA = "Ollama does not answer: install it from https://ollama.com, then start it with ollama serve";
-/** This function tells if a host name is a loopback address: `localhost`, `127.0.0.0/8` or `::1`. */
+/**
+* This function tells if a host name is a loopback address: `localhost`, `127.0.0.0/8` or `::1`.
+*
+* @beta
+*/
 const isLoopback = (host) => {
 	const name = host.replace(/^\[|\]$/g, "").toLowerCase();
 	return name === "localhost" || name.endsWith(".localhost") || name === "::1" || /^127(?:\.\d{1,3}){3}$/.test(name) || /^::ffff:127(?:\.\d{1,3}){3}$/.test(name);
@@ -157,4 +161,4 @@ const ollamaEmbedder = (options = {}) => {
 //#endregion
 export { ollama_exports as a, ollamaEmbedder as i, OLLAMA_MODELS as n, isLoopback as r, NO_OLLAMA as t };
 
-//# sourceMappingURL=ollama-BJxkjny4.mjs.map
+//# sourceMappingURL=ollama-D1xRh3hY.mjs.map

package/build/ollama-D1xRh3hY.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ollama-D1xRh3hY.mjs","names":[],"sources":["../src/ollama.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { EmbedCallOptions, Embedder } from \"./embed.ts\";\n\n/**\n * The Ollama embedder sends texts to the local HTTP API of Ollama (`POST /api/embed`). The package\n * has no dependency for it: you install Ollama, start it (`ollama serve`) and pull a model\n * (`ollama pull nomic-embed-text`).\n *\n * The page text goes to the Ollama server. Thus the embedder accepts only a server on this\n * computer (a loopback address), unless `allowRemote` is true.\n */\n\n/** These are the facts of the embedding models that the Ollama library has. */\nexport interface EmbeddingModel {\n  readonly dimensions: number;\n  readonly maxInputTokens: number;\n  /** This prefix goes before the text of a question. */\n  readonly queryPrefix?: string | undefined;\n  /** This prefix goes before each chunk of a stored text. */\n  readonly documentPrefix?: string | undefined;\n}\n\n/** These are the facts of three embedding models of Ollama. The name has no tag (`nomic-embed-text`, not `nomic-embed-text:latest`). */\nexport const OLLAMA_MODELS: Readonly<Record<string, EmbeddingModel>> = {\n  \"nomic-embed-text\": {\n    dimensions: 768,\n    maxInputTokens: 8192,\n    queryPrefix: \"search_query: \",\n    documentPrefix: \"search_document: \",\n  },\n  \"all-minilm\": { dimensions: 384, maxInputTokens: 256 },\n  \"mxbai-embed-large\": {\n    dimensions: 1024,\n    maxInputTokens: 512,\n    queryPrefix: \"Represent this sentence for searching relevant passages: \",\n  },\n};\n\n/** These options make an Ollama embedder. */\nexport interface OllamaEmbedderOptions {\n  /** This value is the model of Ollama, with a tag when you want one. The default is `nomic-embed-text`. */\n  readonly model?: string | undefined;\n  /** This URL is the Ollama server. The default is `http://127.0.0.1:11434`. */\n  readonly baseUrl?: string | undefined;\n  /** This value is the number of texts in one request. The default is 32. */\n  readonly batch?: number | undefined;\n  /**\n   * The value true lets `baseUrl` be a server on a different computer. The default is false,\n   * because the embedder sends the text of the stored pages to the server.\n   */\n  readonly allowRemote?: boolean | undefined;\n  /** This value is the length of each vector. The default comes from `OLLAMA_MODELS`, or from the first answer. */\n  readonly dimensions?: number | undefined;\n  /** This value is the input limit of the model in tokens. The default comes from `OLLAMA_MODELS`, or it is 256. */\n  readonly maxInputTokens?: number | undefined;\n  /** This prefix goes before the text of a question. The default comes from `OLLAMA_MODELS`. */\n  readonly queryPrefix?: string | undefined;\n  /** This prefix goes before each chunk of a stored text. The default comes from `OLLAMA_MODELS`. */\n  readonly documentPrefix?: string | undefined;\n  /** This value is the longest time of one request in milliseconds. The default is 2 minutes, because the first request loads the model. */\n  readonly timeoutMs?: number | undefined;\n}\n\n/** The text of the error when no Ollama server answers. */\nexport const NO_OLLAMA =\n  \"Ollama does not answer: install it from https://ollama.com, then start it with ollama serve\";\n\n/**\n * This function tells if a host name is a loopback address: `localhost`, `127.0.0.0/8` or `::1`.\n *\n * @beta\n */\nexport const isLoopback = (host: string): boolean => {\n  const name = host.replace(/^\\[|\\]$/g, \"\").toLowerCase();\n  return (\n    name === \"localhost\" ||\n    name.endsWith(\".localhost\") ||\n    name === \"::1\" ||\n    /^127(?:\\.\\d{1,3}){3}$/.test(name) ||\n    /^::ffff:127(?:\\.\\d{1,3}){3}$/.test(name)\n  );\n};\n\n/** This function gives the name of a model with no tag, for `OLLAMA_MODELS`. */\nconst baseName = (model: string): string => {\n  const slash = model.lastIndexOf(\"/\");\n  const colon = model.indexOf(\":\", slash + 1);\n  return colon === -1 ? model : model.slice(0, colon);\n};\n\nconst messageOf = (error: unknown): string =>\n  error instanceof Error ? error.message : String(error);\n\n/** This function gives the code of a network error, also when it is the cause of the error of `fetch`. */\nconst codeOf = (error: unknown): string | undefined => {\n  for (\n    let at: unknown = error, depth = 0;\n    at !== undefined && at !== null && depth < 4;\n    depth += 1\n  ) {\n    const code = (at as { code?: unknown }).code;\n    if (typeof code === \"string\") return code;\n    at = (at as { cause?: unknown }).cause;\n  }\n  return undefined;\n};\n\nconst NETWORK = new Set([\n  \"ECONNREFUSED\",\n  \"ECONNRESET\",\n  \"ENOTFOUND\",\n  \"EHOSTUNREACH\",\n  \"ENETUNREACH\",\n  \"EAI_AGAIN\",\n  \"UND_ERR_SOCKET\",\n  \"UND_ERR_CONNECT_TIMEOUT\",\n]);\n\n/**\n * This function makes an embedder that sends texts to Ollama (`POST {baseUrl}/api/embed`). It\n * sends `batch` texts in each request, and it stops when the signal aborts. The errors tell what\n * to do: `ollama serve` when no server answers, and `ollama pull <model>` when the model is not\n * there. A `baseUrl` on a different computer throws a `RangeError`, unless `allowRemote` is true.\n */\nexport const ollamaEmbedder = (options: OllamaEmbedderOptions = {}): Embedder => {\n  const model = options.model ?? \"nomic-embed-text\";\n  if (model.trim() === \"\") throw new RangeError(\"the Ollama model must have a name\");\n  const base = options.baseUrl ?? \"http://127.0.0.1:11434\";\n  let url: URL;\n  try {\n    url = new URL(base);\n  } catch (cause) {\n    throw new RangeError(`the Ollama URL ${base} is not a URL`, { cause });\n  }\n  if (url.protocol !== \"http:\" && url.protocol !== \"https:\")\n    throw new RangeError(`the Ollama URL ${base} must be an http or https URL`);\n  if (url.username !== \"\" || url.password !== \"\")\n    throw new RangeError(\"the Ollama URL must not hold a user name or a password\");\n  if (!isLoopback(url.hostname) && options.allowRemote !== true) {\n    throw new RangeError(\n      `the Ollama URL ${url.origin} is not on this computer, and the embedder sends the text of the stored pages to it: give allowRemote: true (--allow-remote-ollama) to permit it`,\n    );\n  }\n  const endpoint = new URL(\"api/embed\", url.href.endsWith(\"/\") ? url.href : `${url.href}/`).href;\n  const known = OLLAMA_MODELS[baseName(model)];\n  const batch = Math.max(1, Math.floor(options.batch ?? 32));\n  const timeoutMs = Math.max(1, Math.floor(options.timeoutMs ?? 120_000));\n  const queryPrefix = options.queryPrefix ?? known?.queryPrefix ?? \"\";\n  const documentPrefix = options.documentPrefix ?? known?.documentPrefix ?? \"\";\n  // a document prefix of your own changes each stored vector, so it changes the id\n  const custom =\n    options.documentPrefix !== undefined && options.documentPrefix !== (known?.documentPrefix ?? \"\")\n      ? `#${createHash(\"sha256\").update(options.documentPrefix).digest(\"hex\").slice(0, 8)}`\n      : \"\";\n  let dimensions = options.dimensions ?? known?.dimensions ?? 0;\n  const maxInputTokens = options.maxInputTokens ?? known?.maxInputTokens ?? 256;\n\n  const request = async (\n    input: readonly string[],\n    signal: AbortSignal | undefined,\n  ): Promise<number[][]> => {\n    const timeout = AbortSignal.timeout(timeoutMs);\n    const both = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);\n    let response: Response;\n    try {\n      response = await fetch(endpoint, {\n        method: \"POST\",\n        headers: { \"content-type\": \"application/json\" },\n        body: JSON.stringify({ model, input, truncate: true }),\n        signal: both,\n      });\n    } catch (error) {\n      if (signal?.aborted === true) throw signal.reason;\n      if (timeout.aborted)\n        throw new Error(`Ollama at ${url.origin} did not answer in ${String(timeoutMs)} ms`, {\n          cause: error,\n        });\n      const code = codeOf(error);\n      if (code !== undefined && NETWORK.has(code))\n        throw new Error(`${NO_OLLAMA} (no answer at ${url.origin}: ${code})`, { cause: error });\n      // fetch says only \"fetch failed\": the cause tells why\n      const cause = (error as { cause?: unknown }).cause;\n      throw new Error(\n        `the request to Ollama at ${url.origin} failed: ${messageOf(error)}${cause === undefined ? \"\" : ` (${messageOf(cause)})`}`,\n        { cause: error },\n      );\n    }\n    let text: string;\n    try {\n      text = await response.text();\n    } catch (error) {\n      if (signal?.aborted === true) throw signal.reason;\n      throw new Error(`the answer of Ollama at ${url.origin} stopped: ${messageOf(error)}`, {\n        cause: error,\n      });\n    }\n    let body: { embeddings?: unknown; error?: unknown } | undefined;\n    try {\n      body = JSON.parse(text) as { embeddings?: unknown; error?: unknown };\n    } catch {\n      body = undefined;\n    }\n    const said = typeof body?.error === \"string\" ? body.error : undefined;\n    if (!response.ok) {\n      if (said !== undefined && /not found|pull/i.test(said))\n        throw new Error(\n          `the model ${model} is not in Ollama: ollama pull ${model} (Ollama said: ${said})`,\n        );\n      if (response.status === 404)\n        throw new Error(\n          `the server at ${url.origin} has no /api/embed: update Ollama to version 0.3.0 or later`,\n        );\n      if (said !== undefined && /embed/i.test(said))\n        throw new Error(\n          `the model ${model} does not make embeddings: use an embedding model, such as nomic-embed-text (Ollama said: ${said})`,\n        );\n      throw new Error(\n        `Ollama answered ${String(response.status)}${said === undefined ? \"\" : `: ${said}`}`,\n      );\n    }\n    const embeddings = body?.embeddings;\n    if (\n      !Array.isArray(embeddings) ||\n      embeddings.length !== input.length ||\n      !embeddings.every((vector) => Array.isArray(vector))\n    ) {\n      throw new Error(\n        `Ollama gave no embedding for each text (${String(input.length)} texts): its answer is not the answer of /api/embed`,\n      );\n    }\n    return embeddings as number[][];\n  };\n\n  return {\n    id: `ollama:${model}${custom}`,\n    get dimensions() {\n      return dimensions;\n    },\n    maxInputTokens,\n    async embed(texts: readonly string[], call: EmbedCallOptions = {}): Promise<Float32Array[]> {\n      const prefix = call.purpose === \"query\" ? queryPrefix : documentPrefix;\n      const out: Float32Array[] = [];\n      for (let at = 0; at < texts.length; at += batch) {\n        call.signal?.throwIfAborted();\n        const part = texts.slice(at, at + batch).map((text) => `${prefix}${text}`);\n        for (const vector of await request(part, call.signal)) {\n          if (dimensions === 0) dimensions = vector.length;\n          if (vector.length !== dimensions) {\n            throw new Error(\n              `the model ${model} gave a vector of ${String(vector.length)} dimensions, not ${String(dimensions)}`,\n            );\n          }\n          out.push(Float32Array.from(vector));\n        }\n      }\n      return out;\n    },\n  };\n};\n"],"mappings":";;;;;;;;;;AAuBA,MAAa,gBAA0D;CACrE,oBAAoB;EAClB,YAAY;EACZ,gBAAgB;EAChB,aAAa;EACb,gBAAgB;CAClB;CACA,cAAc;EAAE,YAAY;EAAK,gBAAgB;CAAI;CACrD,qBAAqB;EACnB,YAAY;EACZ,gBAAgB;EAChB,aAAa;CACf;AACF;;AA4BA,MAAa,YACX;;;;;;AAOF,MAAa,cAAc,SAA0B;CACnD,MAAM,OAAO,KAAK,QAAQ,YAAY,EAAE,CAAC,CAAC,YAAY;CACtD,OACE,SAAS,eACT,KAAK,SAAS,YAAY,KAC1B,SAAS,SACT,wBAAwB,KAAK,IAAI,KACjC,+BAA+B,KAAK,IAAI;AAE5C;;AAGA,MAAM,YAAY,UAA0B;CAC1C,MAAM,QAAQ,MAAM,YAAY,GAAG;CACnC,MAAM,QAAQ,MAAM,QAAQ,KAAK,QAAQ,CAAC;CAC1C,OAAO,UAAU,KAAK,QAAQ,MAAM,MAAM,GAAG,KAAK;AACpD;AAEA,MAAM,aAAa,UACjB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;;AAGvD,MAAM,UAAU,UAAuC;CACrD,KACE,IAAI,KAAc,OAAO,QAAQ,GACjC,OAAO,KAAA,KAAa,OAAO,QAAQ,QAAQ,GAC3C,SAAS,GACT;EACA,MAAM,OAAQ,GAA0B;EACxC,IAAI,OAAO,SAAS,UAAU,OAAO;EACrC,KAAM,GAA2B;CACnC;AAEF;AAEA,MAAM,0BAAU,IAAI,IAAI;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;AAQD,MAAa,kBAAkB,UAAiC,CAAC,MAAgB;CAC/E,MAAM,QAAQ,QAAQ,SAAS;CAC/B,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,IAAI,WAAW,mCAAmC;CACjF,MAAM,OAAO,QAAQ,WAAW;CAChC,IAAI;CACJ,IAAI;EACF,MAAM,IAAI,IAAI,IAAI;CACpB,SAAS,OAAO;EACd,MAAM,IAAI,WAAW,kBAAkB,KAAK,gBAAgB,EAAE,MAAM,CAAC;CACvE;CACA,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAC/C,MAAM,IAAI,WAAW,kBAAkB,KAAK,8BAA8B;CAC5E,IAAI,IAAI,aAAa,MAAM,IAAI,aAAa,IAC1C,MAAM,IAAI,WAAW,wDAAwD;CAC/E,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,QAAQ,gBAAgB,MACvD,MAAM,IAAI,WACR,kBAAkB,IAAI,OAAO,iJAC/B;CAEF,MAAM,WAAW,IAAI,IAAI,aAAa,IAAI,KAAK,SAAS,GAAG,IAAI,IAAI,OAAO,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;CAC1F,MAAM,QAAQ,cAAc,SAAS,KAAK;CAC1C,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,EAAE,CAAC;CACzD,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,aAAa,IAAO,CAAC;CACtE,MAAM,cAAc,QAAQ,eAAe,OAAO,eAAe;CACjE,MAAM,iBAAiB,QAAQ,kBAAkB,OAAO,kBAAkB;CAE1E,MAAM,SACJ,QAAQ,mBAAmB,KAAA,KAAa,QAAQ,oBAAoB,OAAO,kBAAkB,MACzF,IAAI,WAAW,QAAQ,CAAC,CAAC,OAAO,QAAQ,cAAc,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,MAChF;CACN,IAAI,aAAa,QAAQ,cAAc,OAAO,cAAc;CAC5D,MAAM,iBAAiB,QAAQ,kBAAkB,OAAO,kBAAkB;CAE1E,MAAM,UAAU,OACd,OACA,WACwB;EACxB,MAAM,UAAU,YAAY,QAAQ,SAAS;EAC7C,MAAM,OAAO,WAAW,KAAA,IAAY,UAAU,YAAY,IAAI,CAAC,QAAQ,OAAO,CAAC;EAC/E,IAAI;EACJ,IAAI;GACF,WAAW,MAAM,MAAM,UAAU;IAC/B,QAAQ;IACR,SAAS,EAAE,gBAAgB,mBAAmB;IAC9C,MAAM,KAAK,UAAU;KAAE;KAAO;KAAO,UAAU;IAAK,CAAC;IACrD,QAAQ;GACV,CAAC;EACH,SAAS,OAAO;GACd,IAAI,QAAQ,YAAY,MAAM,MAAM,OAAO;GAC3C,IAAI,QAAQ,SACV,MAAM,IAAI,MAAM,aAAa,IAAI,OAAO,qBAAqB,OAAO,SAAS,EAAE,MAAM,EACnF,OAAO,MACT,CAAC;GACH,MAAM,OAAO,OAAO,KAAK;GACzB,IAAI,SAAS,KAAA,KAAa,QAAQ,IAAI,IAAI,GACxC,MAAM,IAAI,MAAM,GAAG,UAAU,iBAAiB,IAAI,OAAO,IAAI,KAAK,IAAI,EAAE,OAAO,MAAM,CAAC;GAExF,MAAM,QAAS,MAA8B;GAC7C,MAAM,IAAI,MACR,4BAA4B,IAAI,OAAO,WAAW,UAAU,KAAK,IAAI,UAAU,KAAA,IAAY,KAAK,KAAK,UAAU,KAAK,EAAE,MACtH,EAAE,OAAO,MAAM,CACjB;EACF;EACA,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,SAAS,KAAK;EAC7B,SAAS,OAAO;GACd,IAAI,QAAQ,YAAY,MAAM,MAAM,OAAO;GAC3C,MAAM,IAAI,MAAM,2BAA2B,IAAI,OAAO,YAAY,UAAU,KAAK,KAAK,EACpF,OAAO,MACT,CAAC;EACH;EACA,IAAI;EACJ,IAAI;GACF,OAAO,KAAK,MAAM,IAAI;EACxB,QAAQ;GACN,OAAO,KAAA;EACT;EACA,MAAM,OAAO,OAAO,MAAM,UAAU,WAAW,KAAK,QAAQ,KAAA;EAC5D,IAAI,CAAC,SAAS,IAAI;GAChB,IAAI,SAAS,KAAA,KAAa,kBAAkB,KAAK,IAAI,GACnD,MAAM,IAAI,MACR,aAAa,MAAM,iCAAiC,MAAM,iBAAiB,KAAK,EAClF;GACF,IAAI,SAAS,WAAW,KACtB,MAAM,IAAI,MACR,iBAAiB,IAAI,OAAO,4DAC9B;GACF,IAAI,SAAS,KAAA,KAAa,SAAS,KAAK,IAAI,GAC1C,MAAM,IAAI,MACR,aAAa,MAAM,4FAA4F,KAAK,EACtH;GACF,MAAM,IAAI,MACR,mBAAmB,OAAO,SAAS,MAAM,IAAI,SAAS,KAAA,IAAY,KAAK,KAAK,QAC9E;EACF;EACA,MAAM,aAAa,MAAM;EACzB,IACE,CAAC,MAAM,QAAQ,UAAU,KACzB,WAAW,WAAW,MAAM,UAC5B,CAAC,WAAW,OAAO,WAAW,MAAM,QAAQ,MAAM,CAAC,GAEnD,MAAM,IAAI,MACR,2CAA2C,OAAO,MAAM,MAAM,EAAE,oDAClE;EAEF,OAAO;CACT;CAEA,OAAO;EACL,IAAI,UAAU,QAAQ;EACtB,IAAI,aAAa;GACf,OAAO;EACT;EACA;EACA,MAAM,MAAM,OAA0B,OAAyB,CAAC,GAA4B;GAC1F,MAAM,SAAS,KAAK,YAAY,UAAU,cAAc;GACxD,MAAM,MAAsB,CAAC;GAC7B,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,QAAQ,MAAM,OAAO;IAC/C,KAAK,QAAQ,eAAe;IAC5B,MAAM,OAAO,MAAM,MAAM,IAAI,KAAK,KAAK,CAAC,CAAC,KAAK,SAAS,GAAG,SAAS,MAAM;IACzE,KAAK,MAAM,UAAU,MAAM,QAAQ,MAAM,KAAK,MAAM,GAAG;KACrD,IAAI,eAAe,GAAG,aAAa,OAAO;KAC1C,IAAI,OAAO,WAAW,YACpB,MAAM,IAAI,MACR,aAAa,MAAM,oBAAoB,OAAO,OAAO,MAAM,EAAE,mBAAmB,OAAO,UAAU,GACnG;KAEF,IAAI,KAAK,aAAa,KAAK,MAAM,CAAC;IACpC;GACF;GACA,OAAO;EACT;CACF;AACF"}