polite-crawler 0.1.2 → 0.1.3

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "polite-crawler",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "mcpName": "io.github.mark1russell7/polite-crawler",
   "description": "A polite web crawler for research and AI agents. It fetches a source once, keeps it, and reads it many times: robots.txt obeyed, one request per site, full-text search, an MCP server.",
   "keywords": [
@@ -103,20 +103,28 @@
     "zod": "^4.6.6"
   },
   "peerDependencies": {
+    "@huggingface/transformers": "^3.0.0 || ^4.0.0",
     "playwright-core": ">=1.40"
   },
   "peerDependenciesMeta": {
+    "@huggingface/transformers": {
+      "optional": true
+    },
     "playwright-core": {
       "optional": true
     }
   },
   "devDependencies": {
+    "@huggingface/transformers": "^4.3.1",
     "@mark1russell7/cue": "github:mark1russell7/cue",
     "@types/node": "^22.19.0",
+    "@types/turndown": "^5.0.6",
     "@vitest/coverage-v8": "5.0.3",
     "fast-check": "^4.10.2",
+    "gpt-tokenizer": "^4.0.3",
     "playwright-core": "^1.64.0",
     "tsdown": "^0.23.1",
+    "turndown": "^7.2.4",
     "typescript": "^7.0.2",
     "vitest": "^5.0.3"
   },
@@ -128,7 +136,7 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
-    "typecheck": "tsc -p tsconfig.json --noEmit --emitDeclarationOnly false && tsc -p tsconfig.test.json"
+    "typecheck": "tsc -p tsconfig.json --noEmit --emitDeclarationOnly false && tsc -p tsconfig.test.json && tsc -p eval/tsconfig.json"
   },
   "types": "./build/index.d.mts"
 }

package/build/duration-iIuMMofi.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"duration-iIuMMofi.mjs","names":[],"sources":["../src/version.ts","../src/duration.ts"],"sourcesContent":["/** This is the version of the package. A test makes sure that it agrees with `package.json`. */\nexport const VERSION = \"0.1.2\";\n\n/** This is the product token that robots.txt rules match when an application gives no token. */\nexport const DEFAULT_TOKEN = \"PoliteCrawler\";\n\n/** This page tells a site owner what the crawler is and how to control it. */\nexport const BOT_PAGE = \"https://mark1russell7.github.io/crawler/bot/\";\n","/** A duration is a count of milliseconds, or a text, for example `\"30m\"`, `\"24h\"` or `\"7d\"`. */\nexport type Duration = number | string;\n\nconst UNITS: Readonly<Record<string, number>> = {\n  ms: 1,\n  s: 1000,\n  m: 60_000,\n  h: 3_600_000,\n  d: 86_400_000,\n  w: 604_800_000,\n};\n\n/**\n * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,\n * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`\n * is zero. The function throws an error for a text that it cannot read.\n */\nexport const durationMs = (duration: Duration): number => {\n  if (typeof duration === \"number\") {\n    if (!Number.isFinite(duration) || duration < 0)\n      throw new Error(`not a duration: ${String(duration)}`);\n    return duration;\n  }\n  if (/^\\s*0+(\\.0+)?\\s*$/.test(duration)) return 0;\n  const match = /^\\s*(\\d+(?:\\.\\d+)?)\\s*(ms|s|m|h|d|w)\\s*$/i.exec(duration);\n  const unit = match?.[2];\n  const factor = unit === undefined || unit === \"M\" ? undefined : UNITS[unit.toLowerCase()];\n  if (match === null || factor === undefined) {\n    throw new Error(\n      `not a duration: \"${duration}\" (write a number and ms, s, m, h, d or w; m is minutes)`,\n    );\n  }\n  return Number(match[1]) * factor;\n};\n"],"mappings":";;AACA,MAAa,UAAU;;AAGvB,MAAa,gBAAgB;;AAG7B,MAAa,WAAW;;;ACJxB,MAAM,QAA0C;CAC9C,IAAI;CACJ,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;AACL;;;;;;AAOA,MAAa,cAAc,aAA+B;CACxD,IAAI,OAAO,aAAa,UAAU;EAChC,IAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC3C,MAAM,IAAI,MAAM,mBAAmB,OAAO,QAAQ,GAAG;EACvD,OAAO;CACT;CACA,IAAI,oBAAoB,KAAK,QAAQ,GAAG,OAAO;CAC/C,MAAM,QAAQ,4CAA4C,KAAK,QAAQ;CACvE,MAAM,OAAO,QAAQ;CACrB,MAAM,SAAS,SAAS,KAAA,KAAa,SAAS,MAAM,KAAA,IAAY,MAAM,KAAK,YAAY;CACvF,IAAI,UAAU,QAAQ,WAAW,KAAA,GAC/B,MAAM,IAAI,MACR,oBAAoB,SAAS,yDAC/B;CAEF,OAAO,OAAO,MAAM,EAAE,IAAI;AAC5B"}