polite-crawler 0.1.4 → 0.2.0

package/build/signals-BHpy2xb2.d.mts

@@ -1,46 +0,0 @@
-//#region src/signals.d.ts
-/**
- * Usage signals are the preferences that a page states about the use of its content. The crawler
- * records them with each capture, so a later use can obey them.
- *
- * The signal that changes behavior is AI input: the `ai-input` content signal of Cloudflare and the
- * `ai-use` signal of the IETF AIPREF draft. The crawler keeps a page that says no, but it holds back
- * the text from an AI reader by default. The crawler records a training opt-out (`ai-train`,
- * `train-ai`, and the `noai` and `noimageai` tags of `X-Robots-Tag`), but the opt-out does not
- * apply, because the crawler trains no model. Search engine tags (`noarchive`, `nosnippet`) are for
- * public caches and snippets. The crawler records them, but they do not block.
- *
- * The values can be structured fields (RFC 9651): `ai-input=?0`, `ai-input="no"` and `ai-input=no`
- * have the same meaning.
- *
- * The crawler also records two other kinds of statement, and they change nothing. The first is the
- * `License` lines of RSL in robots.txt. The second is the TDMRep reservation of text and data
- * mining: the headers `tdm-reservation` and `tdm-policy`, and the `<meta>` tags with these names.
- * A TDM reservation is about mining, not about AI input, so it is not a no for AI input.
- */
-type AiInput = "yes" | "no" | "unset";
-interface Signals {
-  readonly ai_input: AiInput;
-  /** the raw lines and headers the verdict came from, for the audit trail */
-  readonly raw: readonly string[];
-  /** This value is the training preference. The crawler records it, and it does not change what the crawler does. */
-  readonly ai_train?: AiInput | undefined;
-  /** This line of `raw` decided `ai_input`. When no line states AI input, there is no line. */
-  readonly deciding?: string | undefined;
-}
-declare const NO_SIGNALS: Signals;
-/**
- * This function reads the TDMRep tags of a page (`<meta name="tdm-reservation" content="1">` and
- * `tdm-policy`) as lines for `readSignals`, for example `meta tdm-reservation: 1`. It reads the first
- * 64 KB of the page, and its time grows in proportion to that length.
- */
-declare const metaSignals: (html: string) => string[];
-/**
- * Read the signals from a site's robots.txt lines (`content-signal: …`,
- * `content-usage: …`) and the response's own headers. A "no" anywhere wins:
- * the most specific statement a site made is the one to respect.
- */
-declare const readSignals: (robotsLines: readonly string[], header: (name: string) => string | undefined, pageLines?: readonly string[]) => Signals;
-//#endregion
-export { readSignals as a, metaSignals as i, NO_SIGNALS as n, Signals as r, AiInput as t };
-//# sourceMappingURL=signals-BHpy2xb2.d.mts.map

package/build/transformers-BQMlaGj4.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"transformers-BQMlaGj4.mjs","names":[],"sources":["../src/transformers.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { EmbedCallOptions, Embedder } from \"./embed.ts\";\nimport type { EmbeddingModel } from \"./ollama.ts\";\n\n/**\n * The transformers embedder uses an embedding model in this process, with transformers.js\n * (`@huggingface/transformers`). The package is an optional peer: the embedder loads it at its\n * first text, as the renderer loads `playwright-core` at its first page.\n *\n * The first use of a model downloads its files from the Hugging Face Hub (huggingface.co). That\n * download does not go through the polite crawler. transformers.js keeps the files in `cacheDir`\n * (its default is `node_modules/@huggingface/transformers/.cache/`), so the next use reads the disk.\n */\n\n/** The text of the error when the optional package `@huggingface/transformers` is not installed. */\nexport const NO_TRANSFORMERS =\n  \"the transformers embedder needs the package @huggingface/transformers: npm install @huggingface/transformers. For the MCP server with npx, start it with npx -y -p polite-crawler -p @huggingface/transformers polite-crawler mcp --embedder transformers\";\n\n/** The pooling of the vectors of the tokens: their mean, or the vector of the first token (`cls`). */\nexport type Pooling = \"mean\" | \"cls\";\n\n/** These are the facts of the models that the embedder knows. Each one is on the Hugging Face Hub with ONNX files. */\nexport const TRANSFORMERS_MODELS: Readonly<\n  Record<string, EmbeddingModel & { readonly pooling: Pooling }>\n> = {\n  \"Xenova/bge-small-en-v1.5\": {\n    dimensions: 384,\n    maxInputTokens: 512,\n    pooling: \"cls\",\n    queryPrefix: \"Represent this sentence for searching relevant passages: \",\n  },\n  \"Xenova/bge-base-en-v1.5\": {\n    dimensions: 768,\n    maxInputTokens: 512,\n    pooling: \"cls\",\n    queryPrefix: \"Represent this sentence for searching relevant passages: \",\n  },\n  \"Xenova/all-MiniLM-L6-v2\": { dimensions: 384, maxInputTokens: 256, pooling: \"mean\" },\n  \"nomic-ai/nomic-embed-text-v1.5\": {\n    dimensions: 768,\n    maxInputTokens: 8192,\n    pooling: \"mean\",\n    queryPrefix: \"search_query: \",\n    documentPrefix: \"search_document: \",\n  },\n};\n\n/** These options make a transformers embedder. */\nexport interface TransformersEmbedderOptions {\n  /** This value is the model on the Hugging Face Hub. The default is `Xenova/bge-small-en-v1.5` (384 dimensions, approximately 34 MB quantized). */\n  readonly model?: string | undefined;\n  /** The value true (the default) loads the 8-bit model (`dtype: \"q8\"`). The value false loads the 32-bit model, which is larger. */\n  readonly quantized?: boolean | undefined;\n  /** This folder keeps the files of the model. The default is the cache of transformers.js. */\n  readonly cacheDir?: string | undefined;\n  /** The value true uses only the files in the cache, with no download. The default is false. */\n  readonly localOnly?: boolean | undefined;\n  /** This value is the pooling of the model. The default comes from `TRANSFORMERS_MODELS`, or it is `mean`. */\n  readonly pooling?: Pooling | undefined;\n  /** This value is the length of each vector. The default comes from `TRANSFORMERS_MODELS`, or from the first answer. */\n  readonly dimensions?: number | undefined;\n  /** This value is the input limit of the model in tokens. The default comes from `TRANSFORMERS_MODELS`, or it is 256. */\n  readonly maxInputTokens?: number | undefined;\n  /** This prefix goes before the text of a question. The default comes from `TRANSFORMERS_MODELS`. */\n  readonly queryPrefix?: string | undefined;\n  /** This prefix goes before each chunk of a stored text. The default comes from `TRANSFORMERS_MODELS`. */\n  readonly documentPrefix?: string | undefined;\n  /** This value is the number of texts that the model reads at one time. The default is 8. */\n  readonly batch?: number | undefined;\n}\n\n/** This is the part of a feature-extraction pipeline of transformers.js that the embedder uses. */\nexport interface FeatureExtractor {\n  (\n    texts: string[],\n    options: { readonly pooling: Pooling; readonly normalize: boolean },\n  ): Promise<{ readonly data: ArrayLike<number>; readonly dims: readonly number[] }>;\n  dispose?(): Promise<unknown>;\n}\n\n/** This is the part of the module `@huggingface/transformers` that the embedder uses. */\nexport interface TransformersModule {\n  pipeline(\n    task: \"feature-extraction\",\n    model: string,\n    options: {\n      readonly dtype: \"q8\" | \"fp32\";\n      readonly cache_dir?: string | undefined;\n      readonly local_files_only?: boolean | undefined;\n    },\n  ): Promise<FeatureExtractor>;\n}\n\n/** This function loads the optional peer. A test gives a module of its own. */\nconst loadTransformers = async (): Promise<TransformersModule> =>\n  (await import(\"@huggingface/transformers\")) as unknown as TransformersModule;\n\n/**\n * This function makes an embedder that uses a model in this process with transformers.js. It loads\n * the package and the model at its first text, one time. It pools the vectors of the tokens as the\n * model expects (`cls` for BGE, `mean` for the others) and gives vectors with a length of 1. When\n * the package is not installed, `embed` rejects with `NO_TRANSFORMERS`. The signal stops the work\n * between two batches.\n */\nexport const transformersEmbedder = (\n  options: TransformersEmbedderOptions = {},\n  load: () => Promise<TransformersModule> = loadTransformers,\n): Embedder => {\n  const model = options.model ?? \"Xenova/bge-small-en-v1.5\";\n  if (model.trim() === \"\") throw new RangeError(\"the transformers model must have a name\");\n  const known = TRANSFORMERS_MODELS[model];\n  const quantized = options.quantized ?? true;\n  const pooling = options.pooling ?? known?.pooling ?? \"mean\";\n  if (pooling !== \"mean\" && pooling !== \"cls\")\n    throw new RangeError(`pooling must be \"mean\" or \"cls\", not ${JSON.stringify(pooling)}`);\n  const batch = Math.max(1, Math.floor(options.batch ?? 8));\n  const queryPrefix = options.queryPrefix ?? known?.queryPrefix ?? \"\";\n  const documentPrefix = options.documentPrefix ?? known?.documentPrefix ?? \"\";\n  const custom =\n    options.documentPrefix !== undefined && options.documentPrefix !== (known?.documentPrefix ?? \"\")\n      ? `#${createHash(\"sha256\").update(options.documentPrefix).digest(\"hex\").slice(0, 8)}`\n      : \"\";\n  let dimensions = options.dimensions ?? known?.dimensions ?? 0;\n  let extractor: Promise<FeatureExtractor> | undefined;\n\n  const open = (): Promise<FeatureExtractor> => {\n    extractor ??= (async () => {\n      let module: TransformersModule;\n      try {\n        module = await load();\n      } catch (cause) {\n        throw new Error(NO_TRANSFORMERS, { cause });\n      }\n      return module.pipeline(\"feature-extraction\", model, {\n        dtype: quantized ? \"q8\" : \"fp32\",\n        ...(options.cacheDir === undefined ? {} : { cache_dir: options.cacheDir }),\n        ...(options.localOnly === true ? { local_files_only: true } : {}),\n      });\n    })();\n    // a failed load is not kept: the next text tries again, for example after an install\n    extractor.catch(() => {\n      extractor = undefined;\n    });\n    return extractor;\n  };\n\n  return {\n    id: `transformers:${model}:${quantized ? \"q8\" : \"fp32\"}:${pooling}${custom}`,\n    get dimensions() {\n      return dimensions;\n    },\n    maxInputTokens: options.maxInputTokens ?? known?.maxInputTokens ?? 256,\n    async embed(texts: readonly string[], call: EmbedCallOptions = {}): Promise<Float32Array[]> {\n      call.signal?.throwIfAborted();\n      if (texts.length === 0) return [];\n      const run = await open();\n      const prefix = call.purpose === \"query\" ? queryPrefix : documentPrefix;\n      const out: Float32Array[] = [];\n      for (let at = 0; at < texts.length; at += batch) {\n        call.signal?.throwIfAborted();\n        const part = texts.slice(at, at + batch).map((text) => `${prefix}${text}`);\n        const tensor = await run(part, { pooling, normalize: true });\n        const width = tensor.dims.at(-1) ?? 0;\n        if (tensor.dims.length !== 2 || tensor.dims[0] !== part.length || width === 0) {\n          throw new Error(\n            `the model ${model} gave a tensor of the shape [${tensor.dims.join(\", \")}], not [${String(part.length)}, dimensions]`,\n          );\n        }\n        if (dimensions === 0) dimensions = width;\n        if (width !== dimensions) {\n          throw new Error(\n            `the model ${model} gave vectors of ${String(width)} dimensions, not ${String(dimensions)}`,\n          );\n        }\n        const data = tensor.data;\n        for (let row = 0; row < part.length; row += 1) {\n          const vector = new Float32Array(width);\n          for (let column = 0; column < width; column += 1)\n            vector[column] = data[row * width + column]!;\n          out.push(vector);\n        }\n      }\n      return out;\n    },\n    async close() {\n      const running = extractor;\n      extractor = undefined;\n      if (running === undefined) return;\n      try {\n        await (await running).dispose?.();\n      } catch {\n        // a model that did not load has nothing to release\n      }\n    },\n  };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAeA,MAAa,kBACX;;AAMF,MAAa,sBAET;CACF,4BAA4B;EAC1B,YAAY;EACZ,gBAAgB;EAChB,SAAS;EACT,aAAa;CACf;CACA,2BAA2B;EACzB,YAAY;EACZ,gBAAgB;EAChB,SAAS;EACT,aAAa;CACf;CACA,2BAA2B;EAAE,YAAY;EAAK,gBAAgB;EAAK,SAAS;CAAO;CACnF,kCAAkC;EAChC,YAAY;EACZ,gBAAgB;EAChB,SAAS;EACT,aAAa;EACb,gBAAgB;CAClB;AACF;;AAiDA,MAAM,mBAAmB,YACtB,MAAM,OAAO;;;;;;;;AAShB,MAAa,wBACX,UAAuC,CAAC,GACxC,OAA0C,qBAC7B;CACb,MAAM,QAAQ,QAAQ,SAAS;CAC/B,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,IAAI,WAAW,yCAAyC;CACvF,MAAM,QAAQ,oBAAoB;CAClC,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,UAAU,QAAQ,WAAW,OAAO,WAAW;CACrD,IAAI,YAAY,UAAU,YAAY,OACpC,MAAM,IAAI,WAAW,wCAAwC,KAAK,UAAU,OAAO,GAAG;CACxF,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,CAAC,CAAC;CACxD,MAAM,cAAc,QAAQ,eAAe,OAAO,eAAe;CACjE,MAAM,iBAAiB,QAAQ,kBAAkB,OAAO,kBAAkB;CAC1E,MAAM,SACJ,QAAQ,mBAAmB,KAAA,KAAa,QAAQ,oBAAoB,OAAO,kBAAkB,MACzF,IAAI,WAAW,QAAQ,CAAC,CAAC,OAAO,QAAQ,cAAc,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,MAChF;CACN,IAAI,aAAa,QAAQ,cAAc,OAAO,cAAc;CAC5D,IAAI;CAEJ,MAAM,aAAwC;EAC5C,eAAe,YAAY;GACzB,IAAI;GACJ,IAAI;IACF,SAAS,MAAM,KAAK;GACtB,SAAS,OAAO;IACd,MAAM,IAAI,MAAM,iBAAiB,EAAE,MAAM,CAAC;GAC5C;GACA,OAAO,OAAO,SAAS,sBAAsB,OAAO;IAClD,OAAO,YAAY,OAAO;IAC1B,GAAI,QAAQ,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,QAAQ,SAAS;IACxE,GAAI,QAAQ,cAAc,OAAO,EAAE,kBAAkB,KAAK,IAAI,CAAC;GACjE,CAAC;EACH,EAAA,CAAG;EAEH,UAAU,YAAY;GACpB,YAAY,KAAA;EACd,CAAC;EACD,OAAO;CACT;CAEA,OAAO;EACL,IAAI,gBAAgB,MAAM,GAAG,YAAY,OAAO,OAAO,GAAG,UAAU;EACpE,IAAI,aAAa;GACf,OAAO;EACT;EACA,gBAAgB,QAAQ,kBAAkB,OAAO,kBAAkB;EACnE,MAAM,MAAM,OAA0B,OAAyB,CAAC,GAA4B;GAC1F,KAAK,QAAQ,eAAe;GAC5B,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC;GAChC,MAAM,MAAM,MAAM,KAAK;GACvB,MAAM,SAAS,KAAK,YAAY,UAAU,cAAc;GACxD,MAAM,MAAsB,CAAC;GAC7B,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,QAAQ,MAAM,OAAO;IAC/C,KAAK,QAAQ,eAAe;IAC5B,MAAM,OAAO,MAAM,MAAM,IAAI,KAAK,KAAK,CAAC,CAAC,KAAK,SAAS,GAAG,SAAS,MAAM;IACzE,MAAM,SAAS,MAAM,IAAI,MAAM;KAAE;KAAS,WAAW;IAAK,CAAC;IAC3D,MAAM,QAAQ,OAAO,KAAK,GAAG,EAAE,KAAK;IACpC,IAAI,OAAO,KAAK,WAAW,KAAK,OAAO,KAAK,OAAO,KAAK,UAAU,UAAU,GAC1E,MAAM,IAAI,MACR,aAAa,MAAM,+BAA+B,OAAO,KAAK,KAAK,IAAI,EAAE,UAAU,OAAO,KAAK,MAAM,EAAE,cACzG;IAEF,IAAI,eAAe,GAAG,aAAa;IACnC,IAAI,UAAU,YACZ,MAAM,IAAI,MACR,aAAa,MAAM,mBAAmB,OAAO,KAAK,EAAE,mBAAmB,OAAO,UAAU,GAC1F;IAEF,MAAM,OAAO,OAAO;IACpB,KAAK,IAAI,MAAM,GAAG,MAAM,KAAK,QAAQ,OAAO,GAAG;KAC7C,MAAM,SAAS,IAAI,aAAa,KAAK;KACrC,KAAK,IAAI,SAAS,GAAG,SAAS,OAAO,UAAU,GAC7C,OAAO,UAAU,KAAK,MAAM,QAAQ;KACtC,IAAI,KAAK,MAAM;IACjB;GACF;GACA,OAAO;EACT;EACA,MAAM,QAAQ;GACZ,MAAM,UAAU;GAChB,YAAY,KAAA;GACZ,IAAI,YAAY,KAAA,GAAW;GAC3B,IAAI;IACF,OAAO,MAAM,QAAA,CAAS,UAAU;GAClC,QAAQ,CAER;EACF;CACF;AACF"}

package/build/urls-BTDSgDLC.d.mts

@@ -1,16 +0,0 @@
-//#region src/urls.d.ts
-/**
- * This function gives one URL for each page: a lowercase host, and no fragment, tracking
- * parameters, empty query or default port. It decodes the escapes of unreserved characters.
- */
-declare const canonicalUrl: (value: string) => string;
-/** These options set the hosts that a crawl or a discovery keeps. */
-interface ScopeOptions {
-  /** With the default value true, the URLs on the sites of the start URLs are in scope. */
-  readonly sameSite?: boolean | undefined;
-  /** The URLs on these hosts are in scope too. A name that starts with `*.` names the hosts under it. */
-  readonly allowHosts?: readonly string[] | undefined;
-}
-//#endregion
-export { canonicalUrl as n, ScopeOptions as t };
-//# sourceMappingURL=urls-BTDSgDLC.d.mts.map

package/build/version-BPem-XZd.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"version-BPem-XZd.mjs","names":[],"sources":["../src/duration.ts","../src/version.ts"],"sourcesContent":["/** A duration is a count of milliseconds, or a text, for example `\"30m\"`, `\"24h\"` or `\"7d\"`. */\nexport type Duration = number | string;\n\nconst UNITS: Readonly<Record<string, number>> = {\n  ms: 1,\n  s: 1000,\n  m: 60_000,\n  h: 3_600_000,\n  d: 86_400_000,\n  w: 604_800_000,\n};\n\n/**\n * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,\n * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`\n * is zero. The function throws an error for a text that it cannot read.\n */\nexport const durationMs = (duration: Duration): number => {\n  if (typeof duration === \"number\") {\n    if (!Number.isFinite(duration) || duration < 0)\n      throw new Error(`not a duration: ${String(duration)}`);\n    return duration;\n  }\n  if (/^\\s*0+(\\.0+)?\\s*$/.test(duration)) return 0;\n  const match = /^\\s*(\\d+(?:\\.\\d+)?)\\s*(ms|s|m|h|d|w)\\s*$/i.exec(duration);\n  const unit = match?.[2];\n  const factor = unit === undefined || unit === \"M\" ? undefined : UNITS[unit.toLowerCase()];\n  if (match === null || factor === undefined) {\n    throw new Error(\n      `not a duration: \"${duration}\" (write a number and ms, s, m, h, d or w; m is minutes)`,\n    );\n  }\n  return Number(match[1]) * factor;\n};\n","/** This is the version of the package. A test makes sure that it agrees with `package.json`. */\nexport const VERSION = \"0.1.4\";\n\n/** This is the product token that robots.txt rules match when an application gives no token. */\nexport const DEFAULT_TOKEN = \"PoliteCrawler\";\n\n/** This page tells a site owner what the crawler is and how to control it. */\nexport const BOT_PAGE = \"https://mark1russell7.github.io/crawler/bot/\";\n"],"mappings":";AAGA,MAAM,QAA0C;CAC9C,IAAI;CACJ,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;AACL;;;;;;AAOA,MAAa,cAAc,aAA+B;CACxD,IAAI,OAAO,aAAa,UAAU;EAChC,IAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC3C,MAAM,IAAI,MAAM,mBAAmB,OAAO,QAAQ,GAAG;EACvD,OAAO;CACT;CACA,IAAI,oBAAoB,KAAK,QAAQ,GAAG,OAAO;CAC/C,MAAM,QAAQ,4CAA4C,KAAK,QAAQ;CACvE,MAAM,OAAO,QAAQ;CACrB,MAAM,SAAS,SAAS,KAAA,KAAa,SAAS,MAAM,KAAA,IAAY,MAAM,KAAK,YAAY;CACvF,IAAI,UAAU,QAAQ,WAAW,KAAA,GAC/B,MAAM,IAAI,MACR,oBAAoB,SAAS,yDAC/B;CAEF,OAAO,OAAO,MAAM,EAAE,IAAI;AAC5B;;;;AChCA,MAAa,UAAU;;AAGvB,MAAa,gBAAgB;;AAG7B,MAAa,WAAW"}