polite-crawler 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/build/{cli-daemon-CehzGYMQ.mjs → cli-daemon-Br17snPs.mjs} +2 -2
  2. package/build/{cli-daemon-CehzGYMQ.mjs.map → cli-daemon-Br17snPs.mjs.map} +1 -1
  3. package/build/{cli-recipes-InEU5y1L.mjs → cli-recipes-Cjq6ohWH.mjs} +2 -2
  4. package/build/{cli-recipes-InEU5y1L.mjs.map → cli-recipes-Cjq6ohWH.mjs.map} +1 -1
  5. package/build/cli.mjs +7 -7
  6. package/build/{daemon-1RNbvE5p.mjs → daemon-BPswUOgu.mjs} +3 -3
  7. package/build/{daemon-1RNbvE5p.mjs.map → daemon-BPswUOgu.mjs.map} +1 -1
  8. package/build/{diff-DLyQ2mkg.mjs → diff-DuD9pinS.mjs} +2 -2
  9. package/build/{diff-DLyQ2mkg.mjs.map → diff-DuD9pinS.mjs.map} +1 -1
  10. package/build/{extract-mnKsYeaM.mjs → extract-BISJyKWN.mjs} +2 -2
  11. package/build/{extract-mnKsYeaM.mjs.map → extract-BISJyKWN.mjs.map} +1 -1
  12. package/build/index.d.mts +1 -1
  13. package/build/index.mjs +10 -10
  14. package/build/{open-CDFgV-FC.mjs → open-CsFRWOjf.mjs} +6 -6
  15. package/build/{open-CDFgV-FC.mjs.map → open-CsFRWOjf.mjs.map} +1 -1
  16. package/build/{recipe-BNjpjJ_8.mjs → recipe-30dN3orz.mjs} +3 -3
  17. package/build/{recipe-BNjpjJ_8.mjs.map → recipe-30dN3orz.mjs.map} +1 -1
  18. package/build/{recipe-product-BqDdxZxL.mjs → recipe-product-BuJ_gEwn.mjs} +4 -4
  19. package/build/{recipe-product-BqDdxZxL.mjs.map → recipe-product-BuJ_gEwn.mjs.map} +1 -1
  20. package/build/{runtime-p5WN2UAA.mjs → runtime-C2VRY89U.mjs} +2 -2
  21. package/build/{runtime-p5WN2UAA.mjs.map → runtime-C2VRY89U.mjs.map} +1 -1
  22. package/build/{version-BVrf4OJP.mjs → version-B9CUZfx5.mjs} +2 -2
  23. package/build/{version-BVrf4OJP.mjs.map → version-B9CUZfx5.mjs.map} +1 -1
  24. package/build/{wacz-BXMxWyfg.mjs → wacz-D5bvxsiM.mjs} +3 -3
  25. package/build/{wacz-BXMxWyfg.mjs.map → wacz-D5bvxsiM.mjs.map} +1 -1
  26. package/build/{warc-DmzB-FkC.mjs → warc-DsB3oP78.mjs} +2 -2
  27. package/build/{warc-DmzB-FkC.mjs.map → warc-DsB3oP78.mjs.map} +1 -1
  28. package/package.json +1 -1
@@ -6,7 +6,7 @@ import { n as describeDaemonStatus } from "./cli.mjs";
6
6
  * error at start rejects, so the command exits with 1.
7
7
  */
8
8
  const daemonCommand = async (line, options, io) => {
9
- const { changeCommand, readDaemonState, startDaemon } = await import("./daemon-1RNbvE5p.mjs").then((n) => n.a);
9
+ const { changeCommand, readDaemonState, startDaemon } = await import("./daemon-BPswUOgu.mjs").then((n) => n.a);
10
10
  if (line.action === "status") {
11
11
  const state = readDaemonState(options.dir);
12
12
  io.out(io.json ? JSON.stringify(state ?? null, null, 2) : describeDaemonStatus(state, options.dir));
@@ -35,4 +35,4 @@ const daemonCommand = async (line, options, io) => {
35
35
  //#endregion
36
36
  export { daemonCommand };
37
37
 
38
- //# sourceMappingURL=cli-daemon-CehzGYMQ.mjs.map
38
+ //# sourceMappingURL=cli-daemon-Br17snPs.mjs.map
@@ -1 +1 @@
1
- {"version":3,"file":"cli-daemon-CehzGYMQ.mjs","names":[],"sources":["../src/cli-daemon.ts"],"sourcesContent":["/*\n * The command `daemon` of the command line: `daemon` runs the daemon until `SIGINT` or `SIGTERM`, and\n * `daemon status` prints the state file. `cli.ts` loads this file only for the command `daemon`.\n */\nimport type { DaemonLine } from \"./cli-line.ts\";\nimport { describeDaemonStatus } from \"./cli-line.ts\";\nimport type { OpenOptions } from \"./open.ts\";\n\n/** These are the parts of the command line that the command `daemon` uses. */\nexport interface DaemonIo {\n /** The value true lets `user_provided` work when the MCP client cannot ask the user. */\n readonly allowUserProvided: boolean;\n /** The value true prints the status as JSON. */\n readonly json: boolean;\n /** This function prints a line on the standard output. */\n readonly out: (text: string) => void;\n}\n\n/**\n * This function does the command `daemon`, and gives the exit code. `daemon status` gives 0 when a\n * daemon operates on the store, and 1 when none operates. `daemon` gives 0 after a clean stop. An\n * error at start rejects, so the command exits with 1.\n */\nexport const daemonCommand = async (\n line: DaemonLine,\n options: OpenOptions,\n io: DaemonIo,\n): Promise<number> => {\n const { changeCommand, readDaemonState, startDaemon } = await import(\"./daemon.ts\");\n if (line.action === \"status\") {\n const state = readDaemonState(options.dir);\n io.out(\n io.json ? JSON.stringify(state ?? null, null, 2) : describeDaemonStatus(state, options.dir),\n );\n return state?.alive === true ? 0 : 1;\n }\n const daemon = await startDaemon({\n ...options,\n tick: line.tickMs,\n jitter: line.jitterMs,\n budget: line.budgetMs,\n maxPages: line.maxPages,\n grace: line.graceMs,\n events: line.events,\n onChange: line.onChange === undefined ? undefined : changeCommand(line.onChange),\n mcp:\n line.mcp === undefined\n ? undefined\n : {\n port: line.mcp.port,\n host: line.mcp.host,\n allowRemote: line.mcp.allowRemote,\n tokenFile: line.mcp.tokenFile,\n allowUserProvided: io.allowUserProvided,\n },\n signals: true,\n });\n await daemon.done;\n return 0;\n};\n"],"mappings":";;;;;;;AAuBA,MAAa,gBAAgB,OAC3B,MACA,SACA,OACoB;CACpB,MAAM,EAAE,eAAe,iBAAiB,gBAAgB,MAAM,OAAO,wBAAc,CAAA,MAAA,MAAA,EAAA,CAAA;CACnF,IAAI,KAAK,WAAW,UAAU;EAC5B,MAAM,QAAQ,gBAAgB,QAAQ,GAAG;EACzC,GAAG,IACD,GAAG,OAAO,KAAK,UAAU,SAAS,MAAM,MAAM,CAAC,IAAI,qBAAqB,OAAO,QAAQ,GAAG,CAC5F;EACA,OAAO,OAAO,UAAU,OAAO,IAAI;CACrC;CAsBA,OAAM,MArBe,YAAY;EAC/B,GAAG;EACH,MAAM,KAAK;EACX,QAAQ,KAAK;EACb,QAAQ,KAAK;EACb,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,QAAQ,KAAK;EACb,UAAU,KAAK,aAAa,KAAA,IAAY,KAAA,IAAY,cAAc,KAAK,QAAQ;EAC/E,KACE,KAAK,QAAQ,KAAA,IACT,KAAA,IACA;GACE,MAAM,KAAK,IAAI;GACf,MAAM,KAAK,IAAI;GACf,aAAa,KAAK,IAAI;GACtB,WAAW,KAAK,IAAI;GACpB,mBAAmB,GAAG;EACxB;EACN,SAAS;CACX,CAAC,EAAA,CACY;CACb,OAAO;AACT"}
1
+ {"version":3,"file":"cli-daemon-Br17snPs.mjs","names":[],"sources":["../src/cli-daemon.ts"],"sourcesContent":["/*\n * The command `daemon` of the command line: `daemon` runs the daemon until `SIGINT` or `SIGTERM`, and\n * `daemon status` prints the state file. `cli.ts` loads this file only for the command `daemon`.\n */\nimport type { DaemonLine } from \"./cli-line.ts\";\nimport { describeDaemonStatus } from \"./cli-line.ts\";\nimport type { OpenOptions } from \"./open.ts\";\n\n/** These are the parts of the command line that the command `daemon` uses. */\nexport interface DaemonIo {\n /** The value true lets `user_provided` work when the MCP client cannot ask the user. */\n readonly allowUserProvided: boolean;\n /** The value true prints the status as JSON. */\n readonly json: boolean;\n /** This function prints a line on the standard output. */\n readonly out: (text: string) => void;\n}\n\n/**\n * This function does the command `daemon`, and gives the exit code. `daemon status` gives 0 when a\n * daemon operates on the store, and 1 when none operates. `daemon` gives 0 after a clean stop. An\n * error at start rejects, so the command exits with 1.\n */\nexport const daemonCommand = async (\n line: DaemonLine,\n options: OpenOptions,\n io: DaemonIo,\n): Promise<number> => {\n const { changeCommand, readDaemonState, startDaemon } = await import(\"./daemon.ts\");\n if (line.action === \"status\") {\n const state = readDaemonState(options.dir);\n io.out(\n io.json ? JSON.stringify(state ?? null, null, 2) : describeDaemonStatus(state, options.dir),\n );\n return state?.alive === true ? 0 : 1;\n }\n const daemon = await startDaemon({\n ...options,\n tick: line.tickMs,\n jitter: line.jitterMs,\n budget: line.budgetMs,\n maxPages: line.maxPages,\n grace: line.graceMs,\n events: line.events,\n onChange: line.onChange === undefined ? undefined : changeCommand(line.onChange),\n mcp:\n line.mcp === undefined\n ? undefined\n : {\n port: line.mcp.port,\n host: line.mcp.host,\n allowRemote: line.mcp.allowRemote,\n tokenFile: line.mcp.tokenFile,\n allowUserProvided: io.allowUserProvided,\n },\n signals: true,\n });\n await daemon.done;\n return 0;\n};\n"],"mappings":";;;;;;;AAuBA,MAAa,gBAAgB,OAC3B,MACA,SACA,OACoB;CACpB,MAAM,EAAE,eAAe,iBAAiB,gBAAgB,MAAM,OAAO,wBAAc,CAAA,MAAA,MAAA,EAAA,CAAA;CACnF,IAAI,KAAK,WAAW,UAAU;EAC5B,MAAM,QAAQ,gBAAgB,QAAQ,GAAG;EACzC,GAAG,IACD,GAAG,OAAO,KAAK,UAAU,SAAS,MAAM,MAAM,CAAC,IAAI,qBAAqB,OAAO,QAAQ,GAAG,CAC5F;EACA,OAAO,OAAO,UAAU,OAAO,IAAI;CACrC;CAsBA,OAAM,MArBe,YAAY;EAC/B,GAAG;EACH,MAAM,KAAK;EACX,QAAQ,KAAK;EACb,QAAQ,KAAK;EACb,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,QAAQ,KAAK;EACb,UAAU,KAAK,aAAa,KAAA,IAAY,KAAA,IAAY,cAAc,KAAK,QAAQ;EAC/E,KACE,KAAK,QAAQ,KAAA,IACT,KAAA,IACA;GACE,MAAM,KAAK,IAAI;GACf,MAAM,KAAK,IAAI;GACf,aAAa,KAAK,IAAI;GACtB,WAAW,KAAK,IAAI;GACpB,mBAAmB,GAAG;EACxB;EACN,SAAS;CACX,CAAC,EAAA,CACY;CACb,OAAO;AACT"}
@@ -1,5 +1,5 @@
1
1
  import { compareHours, fromCompact } from "./hours.mjs";
2
- import { d as readVenue, o as readProduct } from "./recipe-product-BqDdxZxL.mjs";
2
+ import { d as readVenue, o as readProduct } from "./recipe-product-BuJ_gEwn.mjs";
3
3
  import { a as keptWeekOf, i as describeVenue, o as productCode, r as describeProduct, s as venueCode, t as UsageError } from "./cli.mjs";
4
4
  //#region src/cli-recipes.ts
5
5
  /**
@@ -56,4 +56,4 @@ const productCommand = async (target, line, io) => {
56
56
  //#endregion
57
57
  export { prepareRecipe, productCommand, venueCommand };
58
58
 
59
- //# sourceMappingURL=cli-recipes-InEU5y1L.mjs.map
59
+ //# sourceMappingURL=cli-recipes-Cjq6ohWH.mjs.map
@@ -1 +1 @@
1
- {"version":3,"file":"cli-recipes-InEU5y1L.mjs","names":[],"sources":["../src/cli-recipes.ts"],"sourcesContent":["/*\n * The commands `venue` and `product` of the command line. They read a venue or a product page with\n * the recipes of the package (`readVenue` and `readProduct`). `cli.ts` loads this file only for these\n * two commands, so the other commands do not load the recipes.\n */\nimport type { KeptWeek, RecipeLine, VenueCompare } from \"./cli-line.ts\";\nimport {\n UsageError,\n describeProduct,\n describeVenue,\n keptWeekOf,\n productCode,\n venueCode,\n} from \"./cli-line.ts\";\nimport { compareHours, fromCompact } from \"./hours.ts\";\nimport type { RecipeTarget } from \"./recipe.ts\";\nimport type { ProductOptions } from \"./recipe-product.ts\";\nimport { readProduct } from \"./recipe-product.ts\";\nimport type { VenueOptions } from \"./recipe-venue.ts\";\nimport { readVenue } from \"./recipe-venue.ts\";\n\n/** These are the parts of the command line that `venue` and `product` use. */\nexport interface RecipeIo {\n /** The value true prints the result as JSON. */\n readonly json: boolean;\n /** This function prints a line on the standard output. */\n readonly out: (text: string) => void;\n /** This text tells why rendering is off, when the computer has no browser. */\n readonly renderOff?: string | undefined;\n}\n\n/** These are the inputs of `venue` that the command reads before the store opens. */\nexport interface RecipeInput {\n /** This value is the kept week of `--compare`, or undefined with no `--compare`. */\n readonly kept: KeptWeek | undefined;\n}\n\n/**\n * This function checks the parts of `venue` that need a file or a reader, before the store opens.\n * They are the kept week of `--compare` and the value of `--osm`. An error is a usage error.\n */\nexport const prepareRecipe = (line: RecipeLine): RecipeInput => {\n if (line.osm !== undefined && fromCompact(line.osm, \"osm\") === undefined) {\n throw new UsageError(\n `--osm: the hours reader cannot read \"${line.osm}\". Write an opening_hours value, such as \"Mo-Fr 08:00-18:00; Sa 09:00-13:00\"`,\n );\n }\n return { kept: line.compare === undefined ? undefined : keptWeekOf(line.compare) };\n};\n\n/** This function gives the options of the recipe that `venue` and `product` share. */\nconst sharedOptions = (line: RecipeLine): ProductOptions => ({\n render: line.render,\n maxAge: line.maxAge,\n budget: line.budget,\n onBehalfOf: line.onBehalfOf,\n userProvided: line.userProvided,\n fallback: line.fallback,\n archiveDespiteRobots: line.archiveDespiteRobots,\n});\n\n/**\n * This function does the command `venue`, and gives the exit code. It reads the hours of the venue\n * with `readVenue`. With `--compare`, it compares the week of the site (`from_site`) with the kept\n * week through `compareHours`.\n */\nexport const venueCommand = async (\n target: RecipeTarget,\n line: RecipeLine,\n input: RecipeInput,\n io: RecipeIo,\n): Promise<number> => {\n const options: VenueOptions = {\n ...sharedOptions(line),\n name: line.name,\n requireName: line.requireName,\n location: line.location,\n osm: line.osm,\n pages: line.pages,\n };\n const result = await readVenue(target, line.url, options);\n const site = result.from_site;\n const compare: VenueCompare | undefined =\n input.kept === undefined\n ? undefined\n : {\n file: input.kept.file,\n kept: input.kept.week,\n comparison: site === null ? null : compareHours(input.kept.week, site.week),\n };\n io.out(\n io.json\n ? JSON.stringify(compare === undefined ? result : { ...result, compare }, null, 2)\n : describeVenue(result, compare, io.renderOff).join(\"\\n\"),\n );\n return venueCode(result);\n};\n\n/** This function does the command `product`, and gives the exit code. It reads the page with `readProduct`. */\nexport const productCommand = async (\n target: RecipeTarget,\n line: RecipeLine,\n io: RecipeIo,\n): Promise<number> => {\n const result = await readProduct(target, line.url, sharedOptions(line));\n io.out(\n io.json ? JSON.stringify(result, null, 2) : describeProduct(result, io.renderOff).join(\"\\n\"),\n );\n return productCode(result);\n};\n"],"mappings":";;;;;;;;AAyCA,MAAa,iBAAiB,SAAkC;CAC9D,IAAI,KAAK,QAAQ,KAAA,KAAa,YAAY,KAAK,KAAK,KAAK,MAAM,KAAA,GAC7D,MAAM,IAAI,WACR,wCAAwC,KAAK,IAAI,6EACnD;CAEF,OAAO,EAAE,MAAM,KAAK,YAAY,KAAA,IAAY,KAAA,IAAY,WAAW,KAAK,OAAO,EAAE;AACnF;;AAGA,MAAM,iBAAiB,UAAsC;CAC3D,QAAQ,KAAK;CACb,QAAQ,KAAK;CACb,QAAQ,KAAK;CACb,YAAY,KAAK;CACjB,cAAc,KAAK;CACnB,UAAU,KAAK;CACf,sBAAsB,KAAK;AAC7B;;;;;;AAOA,MAAa,eAAe,OAC1B,QACA,MACA,OACA,OACoB;CACpB,MAAM,UAAwB;EAC5B,GAAG,cAAc,IAAI;EACrB,MAAM,KAAK;EACX,aAAa,KAAK;EAClB,UAAU,KAAK;EACf,KAAK,KAAK;EACV,OAAO,KAAK;CACd;CACA,MAAM,SAAS,MAAM,UAAU,QAAQ,KAAK,KAAK,OAAO;CACxD,MAAM,OAAO,OAAO;CACpB,MAAM,UACJ,MAAM,SAAS,KAAA,IACX,KAAA,IACA;EACE,MAAM,MAAM,KAAK;EACjB,MAAM,MAAM,KAAK;EACjB,YAAY,SAAS,OAAO,OAAO,aAAa,MAAM,KAAK,MAAM,KAAK,IAAI;CAC5E;CACN,GAAG,IACD,GAAG,OACC,KAAK,UAAU,YAAY,KAAA,IAAY,SAAS;EAAE,GAAG;EAAQ;CAAQ,GAAG,MAAM,CAAC,IAC/E,cAAc,QAAQ,SAAS,GAAG,SAAS,CAAC,CAAC,KAAK,IAAI,CAC5D;CACA,OAAO,UAAU,MAAM;AACzB;;AAGA,MAAa,iBAAiB,OAC5B,QACA,MACA,OACoB;CACpB,MAAM,SAAS,MAAM,YAAY,QAAQ,KAAK,KAAK,cAAc,IAAI,CAAC;CACtE,GAAG,IACD,GAAG,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,gBAAgB,QAAQ,GAAG,SAAS,CAAC,CAAC,KAAK,IAAI,CAC7F;CACA,OAAO,YAAY,MAAM;AAC3B"}
1
+ {"version":3,"file":"cli-recipes-Cjq6ohWH.mjs","names":[],"sources":["../src/cli-recipes.ts"],"sourcesContent":["/*\n * The commands `venue` and `product` of the command line. They read a venue or a product page with\n * the recipes of the package (`readVenue` and `readProduct`). `cli.ts` loads this file only for these\n * two commands, so the other commands do not load the recipes.\n */\nimport type { KeptWeek, RecipeLine, VenueCompare } from \"./cli-line.ts\";\nimport {\n UsageError,\n describeProduct,\n describeVenue,\n keptWeekOf,\n productCode,\n venueCode,\n} from \"./cli-line.ts\";\nimport { compareHours, fromCompact } from \"./hours.ts\";\nimport type { RecipeTarget } from \"./recipe.ts\";\nimport type { ProductOptions } from \"./recipe-product.ts\";\nimport { readProduct } from \"./recipe-product.ts\";\nimport type { VenueOptions } from \"./recipe-venue.ts\";\nimport { readVenue } from \"./recipe-venue.ts\";\n\n/** These are the parts of the command line that `venue` and `product` use. */\nexport interface RecipeIo {\n /** The value true prints the result as JSON. */\n readonly json: boolean;\n /** This function prints a line on the standard output. */\n readonly out: (text: string) => void;\n /** This text tells why rendering is off, when the computer has no browser. */\n readonly renderOff?: string | undefined;\n}\n\n/** These are the inputs of `venue` that the command reads before the store opens. */\nexport interface RecipeInput {\n /** This value is the kept week of `--compare`, or undefined with no `--compare`. */\n readonly kept: KeptWeek | undefined;\n}\n\n/**\n * This function checks the parts of `venue` that need a file or a reader, before the store opens.\n * They are the kept week of `--compare` and the value of `--osm`. An error is a usage error.\n */\nexport const prepareRecipe = (line: RecipeLine): RecipeInput => {\n if (line.osm !== undefined && fromCompact(line.osm, \"osm\") === undefined) {\n throw new UsageError(\n `--osm: the hours reader cannot read \"${line.osm}\". Write an opening_hours value, such as \"Mo-Fr 08:00-18:00; Sa 09:00-13:00\"`,\n );\n }\n return { kept: line.compare === undefined ? undefined : keptWeekOf(line.compare) };\n};\n\n/** This function gives the options of the recipe that `venue` and `product` share. */\nconst sharedOptions = (line: RecipeLine): ProductOptions => ({\n render: line.render,\n maxAge: line.maxAge,\n budget: line.budget,\n onBehalfOf: line.onBehalfOf,\n userProvided: line.userProvided,\n fallback: line.fallback,\n archiveDespiteRobots: line.archiveDespiteRobots,\n});\n\n/**\n * This function does the command `venue`, and gives the exit code. It reads the hours of the venue\n * with `readVenue`. With `--compare`, it compares the week of the site (`from_site`) with the kept\n * week through `compareHours`.\n */\nexport const venueCommand = async (\n target: RecipeTarget,\n line: RecipeLine,\n input: RecipeInput,\n io: RecipeIo,\n): Promise<number> => {\n const options: VenueOptions = {\n ...sharedOptions(line),\n name: line.name,\n requireName: line.requireName,\n location: line.location,\n osm: line.osm,\n pages: line.pages,\n };\n const result = await readVenue(target, line.url, options);\n const site = result.from_site;\n const compare: VenueCompare | undefined =\n input.kept === undefined\n ? undefined\n : {\n file: input.kept.file,\n kept: input.kept.week,\n comparison: site === null ? null : compareHours(input.kept.week, site.week),\n };\n io.out(\n io.json\n ? JSON.stringify(compare === undefined ? result : { ...result, compare }, null, 2)\n : describeVenue(result, compare, io.renderOff).join(\"\\n\"),\n );\n return venueCode(result);\n};\n\n/** This function does the command `product`, and gives the exit code. It reads the page with `readProduct`. */\nexport const productCommand = async (\n target: RecipeTarget,\n line: RecipeLine,\n io: RecipeIo,\n): Promise<number> => {\n const result = await readProduct(target, line.url, sharedOptions(line));\n io.out(\n io.json ? JSON.stringify(result, null, 2) : describeProduct(result, io.renderOff).join(\"\\n\"),\n );\n return productCode(result);\n};\n"],"mappings":";;;;;;;;AAyCA,MAAa,iBAAiB,SAAkC;CAC9D,IAAI,KAAK,QAAQ,KAAA,KAAa,YAAY,KAAK,KAAK,KAAK,MAAM,KAAA,GAC7D,MAAM,IAAI,WACR,wCAAwC,KAAK,IAAI,6EACnD;CAEF,OAAO,EAAE,MAAM,KAAK,YAAY,KAAA,IAAY,KAAA,IAAY,WAAW,KAAK,OAAO,EAAE;AACnF;;AAGA,MAAM,iBAAiB,UAAsC;CAC3D,QAAQ,KAAK;CACb,QAAQ,KAAK;CACb,QAAQ,KAAK;CACb,YAAY,KAAK;CACjB,cAAc,KAAK;CACnB,UAAU,KAAK;CACf,sBAAsB,KAAK;AAC7B;;;;;;AAOA,MAAa,eAAe,OAC1B,QACA,MACA,OACA,OACoB;CACpB,MAAM,UAAwB;EAC5B,GAAG,cAAc,IAAI;EACrB,MAAM,KAAK;EACX,aAAa,KAAK;EAClB,UAAU,KAAK;EACf,KAAK,KAAK;EACV,OAAO,KAAK;CACd;CACA,MAAM,SAAS,MAAM,UAAU,QAAQ,KAAK,KAAK,OAAO;CACxD,MAAM,OAAO,OAAO;CACpB,MAAM,UACJ,MAAM,SAAS,KAAA,IACX,KAAA,IACA;EACE,MAAM,MAAM,KAAK;EACjB,MAAM,MAAM,KAAK;EACjB,YAAY,SAAS,OAAO,OAAO,aAAa,MAAM,KAAK,MAAM,KAAK,IAAI;CAC5E;CACN,GAAG,IACD,GAAG,OACC,KAAK,UAAU,YAAY,KAAA,IAAY,SAAS;EAAE,GAAG;EAAQ;CAAQ,GAAG,MAAM,CAAC,IAC/E,cAAc,QAAQ,SAAS,GAAG,SAAS,CAAC,CAAC,KAAK,IAAI,CAC5D;CACA,OAAO,UAAU,MAAM;AACzB;;AAGA,MAAa,iBAAiB,OAC5B,QACA,MACA,OACoB;CACpB,MAAM,SAAS,MAAM,YAAY,QAAQ,KAAK,KAAK,cAAc,IAAI,CAAC;CACtE,GAAG,IACD,GAAG,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,gBAAgB,QAAQ,GAAG,SAAS,CAAC,CAAC,KAAK,IAAI,CAC7F;CACA,OAAO,YAAY,MAAM;AAC3B"}
package/build/cli.mjs CHANGED
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env node
2
- import { i as durationMs, r as VERSION } from "./version-BVrf4OJP.mjs";
2
+ import { i as durationMs, r as VERSION } from "./version-B9CUZfx5.mjs";
3
3
  import { join, resolve } from "node:path";
4
4
  import { readFileSync } from "node:fs";
5
5
  import { isIP } from "node:net";
@@ -1335,7 +1335,7 @@ Exit codes:
1335
1335
  * command line first. The export only reads the store: it makes no store, and it sends no request.
1336
1336
  */
1337
1337
  const exportCommand = async (plan, dir, json) => {
1338
- const result = plan.format === "wacz" ? await (await import("./wacz-BXMxWyfg.mjs").then((n) => n.n)).exportWacz(dir, plan.options) : await (await import("./warc-DmzB-FkC.mjs").then((n) => n.a)).exportWarc(dir, plan.options);
1338
+ const result = plan.format === "wacz" ? await (await import("./wacz-D5bvxsiM.mjs").then((n) => n.n)).exportWacz(dir, plan.options) : await (await import("./warc-DsB3oP78.mjs").then((n) => n.a)).exportWarc(dir, plan.options);
1339
1339
  process.stdout.write(`${json ? JSON.stringify(result, null, 2) : describeExport(result, plan.options.includeRefusals)}\n`);
1340
1340
  return 0;
1341
1341
  };
@@ -1457,7 +1457,7 @@ const runWatchCommand = async (crawler, line, values, limits, json, out) => {
1457
1457
  };
1458
1458
  /** This function gives the text of a diff for a person: the two captures, then the text of `formatDiff`. */
1459
1459
  const describeDiff = async (diff) => {
1460
- const { formatDiff } = await import("./diff-DLyQ2mkg.mjs").then((n) => n.i);
1460
+ const { formatDiff } = await import("./diff-DuD9pinS.mjs").then((n) => n.i);
1461
1461
  return [
1462
1462
  `${diff.doc} ${diff.from.capture} (${diff.from.fetched_at}) -> ${diff.to.capture} (${diff.to.fetched_at})`,
1463
1463
  formatDiff(diff),
@@ -1467,7 +1467,7 @@ const describeDiff = async (diff) => {
1467
1467
  };
1468
1468
  /** This function checks the fields against a stored document, and prints a line for each field. The exit code is 0 when each field is verified or normalized. */
1469
1469
  const runExtract = async (crawler, doc, fields, options, json, out) => {
1470
- const { extractDocument } = await import("./extract-mnKsYeaM.mjs").then((n) => n.i);
1470
+ const { extractDocument } = await import("./extract-BISJyKWN.mjs").then((n) => n.i);
1471
1471
  const claims = fields;
1472
1472
  const result = crawler.extract === void 0 ? await extractDocument(crawler.runtime, doc, claims, options) : await crawler.extract(doc, claims, options);
1473
1473
  if (json) out(JSON.stringify(result, null, 2));
@@ -1570,7 +1570,7 @@ const main = async (args) => {
1570
1570
  return 0;
1571
1571
  }
1572
1572
  if (command === "daemon") {
1573
- const { daemonCommand } = await import("./cli-daemon-CehzGYMQ.mjs");
1573
+ const { daemonCommand } = await import("./cli-daemon-Br17snPs.mjs");
1574
1574
  return daemonCommand(line.daemonLine, options, {
1575
1575
  allowUserProvided: line.allowUserProvided,
1576
1576
  json: values.json === true,
@@ -1579,9 +1579,9 @@ const main = async (args) => {
1579
1579
  }
1580
1580
  if (line.exportPlan !== void 0) return exportCommand(line.exportPlan, dir, values.json === true);
1581
1581
  const recipeLine = line.recipeLine;
1582
- const recipes = recipeLine === void 0 ? void 0 : await import("./cli-recipes-InEU5y1L.mjs");
1582
+ const recipes = recipeLine === void 0 ? void 0 : await import("./cli-recipes-Cjq6ohWH.mjs");
1583
1583
  const recipeInput = recipes === void 0 || recipeLine === void 0 ? void 0 : recipes.prepareRecipe(recipeLine);
1584
- const { openCrawler } = await import("./open-CDFgV-FC.mjs").then((n) => n.a);
1584
+ const { openCrawler } = await import("./open-CsFRWOjf.mjs").then((n) => n.a);
1585
1585
  let renderOff;
1586
1586
  let renderer = options.renderer;
1587
1587
  if (!(command === "fetch" || command === "crawl" || recipeLine !== void 0 || watchLine?.action === "run") || render === "never") renderer = false;
@@ -1,8 +1,8 @@
1
1
  import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
2
2
  import { t as browserMissing } from "./browser-56dBBLec.mjs";
3
- import { i as durationMs, r as VERSION } from "./version-BVrf4OJP.mjs";
3
+ import { i as durationMs, r as VERSION } from "./version-B9CUZfx5.mjs";
4
4
  import { realClock } from "./politeness.mjs";
5
- import { i as openCrawler } from "./open-CDFgV-FC.mjs";
5
+ import { i as openCrawler } from "./open-CsFRWOjf.mjs";
6
6
  import { randomBytes } from "node:crypto";
7
7
  import { dirname, join } from "node:path";
8
8
  import { appendFileSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
@@ -516,4 +516,4 @@ const startDaemon = async (options) => {
516
516
  //#endregion
517
517
  export { daemon_exports as a, startDaemon as c, daemonStateFile as i, DAEMON_STATE_SCHEMA as n, readDaemonState as o, changeCommand as r, scheduleWatches as s, DAEMON_DEFAULTS as t };
518
518
 
519
- //# sourceMappingURL=daemon-1RNbvE5p.mjs.map
519
+ //# sourceMappingURL=daemon-BPswUOgu.mjs.map
@@ -1 +1 @@
1
- {"version":3,"file":"daemon-1RNbvE5p.mjs","names":[],"sources":["../src/daemon.ts"],"sourcesContent":["import { spawn } from \"node:child_process\";\nimport { randomBytes } from \"node:crypto\";\nimport {\n appendFileSync,\n mkdirSync,\n readFileSync,\n renameSync,\n unlinkSync,\n writeFileSync,\n} from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { browserMissing } from \"./browser.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport type { OpenOptions, OpenedCrawler } from \"./open.ts\";\nimport { openCrawler } from \"./open.ts\";\nimport type { Clock } from \"./politeness.ts\";\nimport { realClock } from \"./politeness.ts\";\nimport { VERSION } from \"./version.ts\";\nimport type { WatchEntry, WatchEvent, WatchList, WatchSummary } from \"./watch.ts\";\n\n/*\n * The daemon is a local process that runs for a long time. It does these jobs:\n *\n * - It checks the watch list on a schedule (`scheduleWatches`). Each entry that is due gets its\n * check at its time, with each polite rule of `watch.run`.\n * - It writes each change as one JSON line, and it can start a command of the operator for it.\n * - It can serve the MCP tools over HTTP, with the long tools as MCP tasks. This part loads the MCP\n * SDK only when the daemon serves MCP, through a small seam (`DaemonMcpHost`).\n * - It writes a small state file, which `polite-crawler daemon status` reads.\n */\n\n/** These are the defaults of the daemon. */\nexport const DAEMON_DEFAULTS = {\n /** The longest wait between two ticks: 1 minute. A tick comes sooner when an entry is due sooner. */\n tickMs: 60_000,\n /** The most random time that the daemon adds to each wait: 10 seconds. */\n jitterMs: 10_000,\n /** The budget of the checks of one tick: 10 minutes. No request starts after it. */\n budgetMs: 600_000,\n /** The most checks of one tick. */\n maxPages: 100,\n /** At a stop, the checks in progress get this time to end: 10 seconds. Then they abort. */\n graceMs: 10_000,\n /** The shortest wait between two ticks: 1 second. */\n minGapMs: 1_000,\n /** The shortest tick: 1 second. */\n minTickMs: 1_000,\n /** The status lists this number of the next due entries. */\n dueShown: 10,\n /** An on-change command that runs longer than this time stops: 1 minute. */\n hookTimeoutMs: 60_000,\n} as const;\n\n/** This value is the version of the format of the state file. */\nexport const DAEMON_STATE_SCHEMA = 1;\n\n/** This is a watch entry that is due soon, as the status shows it. */\nexport interface DaemonDue {\n readonly url: string;\n /** This time (ISO 8601) is the time when the entry is due. */\n readonly next_at: string;\n readonly every_ms: number;\n /** This value is the kind of the last check, or null before the first check. */\n readonly last_kind: string | null;\n}\n\n/** This is one tick of the schedule: one run of the watch list. */\nexport interface DaemonTick {\n /** This time (ISO 8601) is the start of the run. */\n readonly started_at: string;\n /** This time (ISO 8601) is the end of the run. */\n readonly ended_at: string;\n /** This value is the summary of the run, or null when the run failed. */\n readonly summary: WatchSummary | null;\n /** This text tells why the run failed, or null. */\n readonly error: string | null;\n /** This time (ISO 8601) is the time of the next tick, or null when the schedule stops. */\n readonly next_at: string | null;\n /** These entries are the next due entries after the run, the first due first. */\n readonly due: readonly DaemonDue[];\n}\n\n/** These options control the schedule of the watch list. */\nexport interface ScheduleOptions {\n /** This value is the longest wait between two ticks. The default is `1m`, and the floor is `1s`. */\n readonly tick?: Duration | undefined;\n /** This value is the most random time that the schedule adds to each wait. The default is `10s`. */\n readonly jitter?: Duration | undefined;\n /** This value is the budget of the checks of one tick. The default is `10m`. */\n readonly budget?: Duration | undefined;\n /** This value is the most checks of one tick. The default is 100. */\n readonly maxPages?: number | undefined;\n /** This clock gives the time and the waits. Give the clock of the crawler. The default is the real clock. */\n readonly clock?: Clock | undefined;\n /** This function gives the random numbers of the jitter. The default is `Math.random`. */\n readonly random?: (() => number) | undefined;\n /** This function gets each event of each check. */\n readonly onEvent?: ((event: WatchEvent) => void) | undefined;\n /** This function gets each tick when its run ends. */\n readonly onTick?: ((tick: DaemonTick) => void) | undefined;\n /** This function gets the error of a run that failed. The schedule goes on at the next tick. */\n readonly onError?: ((error: Error) => void) | undefined;\n}\n\n/** This is the result of a schedule: the counts so far, and the last tick. */\nexport interface ScheduleResult {\n readonly ticks: number;\n /** This value counts the checks that gave an event. */\n readonly checked: number;\n /** This value counts the confirmed changes. */\n readonly changes: number;\n readonly last: DaemonTick | null;\n}\n\n/** A schedule that runs. `stop` ends it, and `done` resolves when it ends. */\nexport interface WatchSchedule {\n /** This promise resolves when the schedule ends. It does not reject. */\n readonly done: Promise<ScheduleResult>;\n /**\n * This method stops the schedule. The run in progress starts no new check. The checks in\n * progress get `grace` (10 seconds by default) to end and write their state. Then they abort,\n * and the run releases their leases. A second call with a shorter grace shortens it.\n */\n stop(grace?: Duration): Promise<ScheduleResult>;\n /** This method gives the counts so far. */\n result(): ScheduleResult;\n}\n\nconst iso = (ms: number): string => new Date(ms).toISOString();\n\n/** This function reads a duration of an option, with a floor. A bad value throws a `RangeError`. */\nconst msOf = (name: string, value: Duration, floor: number): number => {\n let ms: number;\n try {\n ms = durationMs(value);\n } catch (error) {\n throw new RangeError(`${name}: ${(error as Error).message}`, { cause: error });\n }\n if (ms < floor)\n throw new RangeError(`${name} must be ${String(floor)} ms or more, not ${String(ms)}`);\n return ms;\n};\n\nconst pagesOf = (value: number | undefined): number => {\n const pages = value ?? DAEMON_DEFAULTS.maxPages;\n if (!Number.isInteger(pages) || pages < 1)\n throw new RangeError(`maxPages must be an integer of 1 or more, not ${String(pages)}`);\n return pages;\n};\n\nconst errorOf = (error: unknown): Error =>\n error instanceof Error ? error : new Error(String(error));\n\n/** This function gives the next due entries of a list, as the status shows them. */\nconst dueOf = (entries: readonly WatchEntry[]): DaemonDue[] =>\n [...entries]\n .sort((a, b) => Date.parse(a.next_at) - Date.parse(b.next_at) || a.url.localeCompare(b.url))\n .slice(0, DAEMON_DEFAULTS.dueShown)\n .map((entry) => ({\n url: entry.url,\n next_at: entry.next_at,\n every_ms: entry.every_ms,\n last_kind: entry.last_kind,\n }));\n\n/**\n * This function gives the time of the next tick. It is the time when the first entry is due, or\n * the end of its lease. It is not later than one tick, and not sooner than `minGapMs`. A random\n * jitter from 0 to `jitterMs` comes after it, so a check does not come before its time.\n *\n * When the budget or `maxPages` of a run left due entries, they wait for the next tick, so one\n * tick does not exceed its budget. A run that failed also waits one tick.\n */\nexport const nextTickAt = (\n entries: readonly WatchEntry[],\n summary: WatchSummary | null,\n now: number,\n options: { readonly tickMs: number; readonly jitterMs: number; readonly random: () => number },\n): number => {\n let next = now + options.tickMs;\n const leftover = summary === null || summary.stopped !== \"done\";\n for (const entry of entries) {\n const lease = entry.leased_until === null ? 0 : Date.parse(entry.leased_until);\n const due = Math.max(Date.parse(entry.next_at), lease);\n if (due <= now && leftover) continue;\n next = Math.min(next, Math.max(due, now + DAEMON_DEFAULTS.minGapMs));\n }\n const random = Math.min(Math.max(options.random(), 0), 1);\n return next + Math.min(options.jitterMs, Math.floor(random * options.jitterMs));\n};\n\n/**\n * This function checks a watch list on a schedule. The first tick starts at once. Each tick is one\n * `watch.run` with the budget and `maxPages` of a tick. Thus each polite rule of the run applies:\n * one request at a time to each host, the stops of a host, `Retry-After` and robots.txt.\n *\n * After a run, the schedule waits for the next tick (`nextTickAt`). An error of a run goes to\n * `onError`, and the schedule goes on.\n */\nexport const scheduleWatches = (watch: WatchList, options: ScheduleOptions = {}): WatchSchedule => {\n const tickMs = msOf(\"tick\", options.tick ?? DAEMON_DEFAULTS.tickMs, DAEMON_DEFAULTS.minTickMs);\n const jitterMs = msOf(\"jitter\", options.jitter ?? DAEMON_DEFAULTS.jitterMs, 0);\n const budgetMs = msOf(\"budget\", options.budget ?? DAEMON_DEFAULTS.budgetMs, 0);\n const maxPages = pagesOf(options.maxPages);\n const clock = options.clock ?? realClock;\n const random = options.random ?? Math.random;\n // the drain starts no new check, the cancel aborts the checks in progress, the wake ends a wait\n const drain = new AbortController();\n const cancel = new AbortController();\n const wake = new AbortController();\n let stopping = false;\n const counts = { ticks: 0, checked: 0, changes: 0, last: null as DaemonTick | null };\n const result = (): ScheduleResult => ({ ...counts });\n const report = (error: unknown): void => {\n try {\n options.onError?.(errorOf(error));\n } catch {\n // an error of the error handler stops nothing\n }\n };\n const safely = (work: () => void): void => {\n try {\n work();\n } catch (error) {\n report(error);\n }\n };\n\n const loop = async (): Promise<void> => {\n while (!stopping) {\n const started = clock.now();\n let summary: WatchSummary | null = null;\n let failure: string | null = null;\n try {\n const run = watch.run({\n budgetMs,\n maxPages,\n signal: cancel.signal,\n drain: drain.signal,\n });\n for await (const event of run) {\n counts.checked += 1;\n if (event.confirmed) counts.changes += 1;\n safely(() => options.onEvent?.(event));\n }\n summary = await run.summary;\n } catch (error) {\n failure = errorOf(error).message;\n report(error);\n }\n counts.ticks += 1;\n // the list gives the next due entries to the status, also at the last tick\n let entries: WatchEntry[] = [];\n try {\n entries = await watch.list();\n } catch (error) {\n report(error);\n }\n const next = stopping\n ? null\n : nextTickAt(entries, summary, clock.now(), { tickMs, jitterMs, random });\n const tick: DaemonTick = {\n started_at: iso(started),\n ended_at: iso(clock.now()),\n summary,\n error: failure,\n next_at: next === null ? null : iso(next),\n due: dueOf(entries),\n };\n counts.last = tick;\n safely(() => options.onTick?.(tick));\n if (next === null || stopping) break;\n await clock.sleep(Math.max(0, next - clock.now()), wake.signal);\n }\n };\n\n const done = loop().then(result, (error: unknown) => {\n report(error);\n return result();\n });\n return {\n done,\n result,\n stop(grace = DAEMON_DEFAULTS.graceMs) {\n const graceMs = msOf(\"grace\", grace, 0);\n stopping = true;\n drain.abort(new Error(\"the schedule stops\"));\n wake.abort(new Error(\"the schedule stops\"));\n if (graceMs === 0) cancel.abort(new Error(\"the schedule stopped at once\"));\n else {\n // a real timer: the grace is time of the process, not of the clock of the crawler\n const timer = setTimeout(\n () => cancel.abort(new Error(\"the grace time of the stop ended\")),\n graceMs,\n );\n void done.finally(() => clearTimeout(timer));\n }\n return done;\n },\n };\n};\n\n// ── the state file ──────────────────────────────────────────────────────────\n\n/** This is a task of the MCP server, as the status shows it. */\nexport interface DaemonTask {\n readonly id: string;\n /** This value is the name of the tool, for example `crawl_site`. */\n readonly tool: string;\n /** This value is `working`, `input_required`, `completed`, `failed` or `cancelled`. */\n readonly status: string;\n /** This text tells the progress, for example `3 of 10 done`, or null. */\n readonly message: string | null;\n readonly created_at: string;\n readonly updated_at: string;\n}\n\n/** This is the status of a daemon. The state file (`<store>/daemon/state.json`) holds the same JSON. */\nexport interface DaemonStatus {\n /** This value is the version of the format (`DAEMON_STATE_SCHEMA`). */\n readonly schema: number;\n /** This value is the version of polite-crawler. */\n readonly version: string;\n /** This value is the id of the process of the daemon. */\n readonly pid: number;\n readonly store: string;\n /** This value is false after the daemon stopped. */\n readonly running: boolean;\n readonly started_at: string;\n readonly updated_at: string;\n readonly stopped_at: string | null;\n readonly tick_ms: number;\n readonly jitter_ms: number;\n readonly budget_ms: number;\n readonly max_pages: number;\n /** This value counts the ticks. */\n readonly ticks: number;\n /** This value counts the checks. */\n readonly checked: number;\n /** This value counts the confirmed changes. */\n readonly changes: number;\n readonly last_tick: DaemonTick | null;\n /** This time (ISO 8601) is the time of the next tick, or null. */\n readonly next_tick_at: string | null;\n /** These entries are the next due entries at the last tick, the first due first. */\n readonly next_due: readonly DaemonDue[];\n /** These tasks of the MCP server did not end. */\n readonly tasks: readonly DaemonTask[];\n /** This value is the MCP endpoint and the file of its token, or null with no MCP server. */\n readonly mcp: { readonly url: string; readonly token_file: string | null } | null;\n /** This value is the last error: of a tick, of the event log, of an on-change hook or of the MCP server. */\n readonly last_error: { readonly at: string; readonly message: string } | null;\n}\n\n/** This is a status that `readDaemonState` read from a state file. */\nexport interface DaemonState extends DaemonStatus {\n /** This value is true when the daemon runs: the state says so, and its process is alive. */\n readonly alive: boolean;\n}\n\n/** These state files belong to a daemon that runs in this process. */\nconst RUNNING = new Set<string>();\n\n/** This function tells if a process is alive. A process of a different user is alive too. */\nconst processAlive = (pid: number): boolean => {\n if (!Number.isInteger(pid) || pid <= 0) return false;\n try {\n process.kill(pid, 0);\n return true;\n } catch (error) {\n return (error as { code?: unknown }).code === \"EPERM\";\n }\n};\n\n/** This function gives the path of the state file of a store. */\nexport const daemonStateFile = (dir: string): string => join(dir, \"daemon\", \"state.json\");\n\n/**\n * This function reads the state file of a daemon. It gives undefined when the file is not there,\n * or when it is not a state file. `alive` tells if the daemon runs.\n */\nexport const readDaemonState = (\n dir: string,\n file: string = daemonStateFile(dir),\n): DaemonState | undefined => {\n let state: DaemonStatus;\n try {\n state = JSON.parse(readFileSync(file, \"utf8\")) as DaemonStatus;\n } catch {\n return undefined;\n }\n if (typeof state !== \"object\" || state === null || typeof state.pid !== \"number\")\n return undefined;\n const alive =\n state.running === true &&\n (state.pid === process.pid ? RUNNING.has(file) : processAlive(state.pid));\n return { ...state, alive };\n};\n\n/** This function writes a state file for the owner only. A rename makes the change atomic where the system can. */\nconst writeState = (file: string, state: DaemonStatus): void => {\n mkdirSync(dirname(file), { recursive: true, mode: 0o700 });\n const text = `${JSON.stringify(state, null, 2)}\\n`;\n const temp = `${file}.${String(process.pid)}.tmp`;\n writeFileSync(temp, text, { mode: 0o600 });\n try {\n renameSync(temp, file);\n } catch {\n // Windows can refuse the rename while a reader has the file open\n writeFileSync(file, text, { mode: 0o600 });\n try {\n unlinkSync(temp);\n } catch {\n // the next write replaces it\n }\n }\n};\n\n/** This function writes a new random token to a file for the owner only, and gives the token. */\nconst writeToken = (file: string): string => {\n const token = randomBytes(32).toString(\"base64url\");\n mkdirSync(dirname(file), { recursive: true, mode: 0o700 });\n try {\n // a new file gets the mode: an old file can have a wider mode\n unlinkSync(file);\n } catch {\n // no old file\n }\n writeFileSync(file, `${token}\\n`, { mode: 0o600, flag: \"wx\" });\n return token;\n};\n\n// ── the on-change command ───────────────────────────────────────────────────\n\n/** These options start an on-change command. */\nexport interface ChangeCommandOptions {\n /** A command that runs longer than this time stops. The default is `1m`. */\n readonly timeout?: Duration | undefined;\n /** This folder is the working folder of the command. The default is the working folder of the daemon. */\n readonly cwd?: string | undefined;\n}\n\n/**\n * This function gives an `onChange` hook that starts a command of the operator for each change.\n * The command operates in the shell of the system, as the user of the daemon. It gets the event as\n * one JSON line on its standard input, not in its command line. Thus no text of a page goes into\n * the command line. The output of the command goes nowhere. A command that exits with a code that\n * is not 0, or that runs too long, rejects with the end of its standard error.\n *\n * The command is the decision of the operator. Do not take it from a page or from an assistant.\n */\nexport const changeCommand = (\n command: string,\n options: ChangeCommandOptions = {},\n): ((event: WatchEvent) => Promise<void>) => {\n if (command.trim() === \"\") throw new RangeError(\"the on-change command is empty\");\n const timeoutMs = msOf(\"timeout\", options.timeout ?? DAEMON_DEFAULTS.hookTimeoutMs, 1);\n return (event) =>\n new Promise<void>((resolve, reject) => {\n const child = spawn(command, {\n shell: true,\n stdio: [\"pipe\", \"ignore\", \"pipe\"],\n windowsHide: true,\n ...(options.cwd === undefined ? {} : { cwd: options.cwd }),\n });\n let tail = \"\";\n let late = false;\n const timer = setTimeout(() => {\n late = true;\n child.kill();\n }, timeoutMs);\n child.stderr.setEncoding(\"utf8\");\n child.stderr.on(\"data\", (text: string) => {\n tail = (tail + text).slice(-2000);\n });\n // a command that does not read its input closes the pipe early\n child.stdin.on(\"error\", () => undefined);\n child.on(\"error\", (error) => {\n clearTimeout(timer);\n reject(\n new Error(`the on-change command did not start: ${error.message}`, { cause: error }),\n );\n });\n child.on(\"close\", (code, signal) => {\n clearTimeout(timer);\n const said = tail.trim() === \"\" ? \"\" : `: ${tail.trim().slice(-300)}`;\n if (late) {\n reject(\n new Error(\n `the on-change command ran for more than ${String(timeoutMs)} ms, so the daemon stopped it`,\n ),\n );\n } else if (code === 0) resolve();\n else\n reject(\n new Error(\n `the on-change command exited with ${code === null ? String(signal) : `code ${String(code)}`}${said}`,\n ),\n );\n });\n child.stdin.end(`${JSON.stringify(event)}\\n`);\n });\n};\n\n// ── the daemon ──────────────────────────────────────────────────────────────\n\n/**\n * These options serve the MCP tools over HTTP.\n *\n * @beta\n */\nexport interface DaemonMcpOptions {\n /** This value is the port. The default is 9307. The value 0 takes a free port. */\n readonly port?: number | undefined;\n /** This value is the host to bind. The default is `127.0.0.1`. A host that is not loopback needs `allowRemote`. */\n readonly host?: string | undefined;\n /** The value true permits a host that is not loopback. Read the warning in the README first. */\n readonly allowRemote?: boolean | undefined;\n /** This value is the bearer token. The default makes a new token at each start, in `<store>/daemon/mcp-token`. */\n readonly token?: string | undefined;\n /** This file holds the bearer token of the operator. The daemon reads it and writes nothing. */\n readonly tokenFile?: string | undefined;\n /** These tokens get their robots.txt rules applied to each fetch of a tool. The default is `[\"Claude-User\"]`. */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** The value true lets `user_provided` release held-back text when the client cannot ask the user. */\n readonly allowUserProvided?: boolean | undefined;\n /** This text tells why the server renders no page. */\n readonly renderOff?: string | undefined;\n}\n\n/**\n * These functions connect an MCP host to the daemon.\n *\n * @beta\n */\nexport interface DaemonMcpHooks {\n /** This function gives the status of the daemon, for the status route. */\n readonly status: () => DaemonStatus;\n /** This function operates when a task starts, changes or ends. */\n readonly onTasks: () => void;\n /** This function writes a message of the server. */\n readonly log: (line: string) => void;\n}\n\n/**\n * This is the MCP server of the daemon, as the MCP host gives it.\n *\n * @beta\n */\nexport interface DaemonMcpService {\n /** This URL is the MCP endpoint. */\n readonly url: string;\n /** This method gives the tasks that did not end. */\n tasks(): readonly DaemonTask[];\n /** This method stops the server and cancels its tasks. */\n close(): Promise<void>;\n}\n\n/**\n * This is the seam between the daemon and its MCP server. The default host (`mcpHttpHost` of the\n * package polite-crawler-mcp) serves the tools over Streamable HTTP. A different MCP host can\n * replace it.\n *\n * @beta\n */\nexport interface DaemonMcpHost {\n start(\n crawler: OpenedCrawler,\n options: DaemonMcpOptions,\n hooks: DaemonMcpHooks,\n ): Promise<DaemonMcpService>;\n}\n\n/** These options start a daemon on a store. Only `dir` is necessary. */\nexport interface DaemonOptions extends OpenOptions {\n /** This value is the longest wait between two ticks. The default is `1m`, and the floor is `1s`. */\n readonly tick?: Duration | undefined;\n /** This value is the most random time that the daemon adds to each wait. The default is `10s`. */\n readonly jitter?: Duration | undefined;\n /** This value is the budget of the checks of one tick. The default is `10m`. */\n readonly budget?: Duration | undefined;\n /** This value is the most checks of one tick. The default is 100. */\n readonly maxPages?: number | undefined;\n /** At a stop, the checks in progress get this time to end. The default is `10s`. */\n readonly grace?: Duration | undefined;\n /**\n * The daemon writes each change event as one JSON line here: a file, a function, or the standard\n * output (the default). The value false writes no line. A change is an event with `confirmed: true`.\n */\n readonly events?: string | ((line: string) => void) | false | undefined;\n /** This function gets each change event. The daemon starts it in order, one at a time, and logs its error. */\n readonly onChange?: ((event: WatchEvent) => void | Promise<void>) | undefined;\n /**\n * These options serve the MCP tools over HTTP. Without them, the daemon serves no MCP.\n *\n * @beta\n */\n readonly mcp?: DaemonMcpOptions | undefined;\n /**\n * This host serves the MCP tools. The default serves them over Streamable HTTP.\n *\n * @beta\n */\n readonly mcpHost?: DaemonMcpHost | undefined;\n /** The value false installs no handler of `SIGINT` and `SIGTERM`. The default is true. */\n readonly signals?: boolean | undefined;\n /** This file keeps the state of the daemon. The default is `<store>/daemon/state.json`. */\n readonly stateFile?: string | undefined;\n /** This function writes the messages of the daemon. The default writes them to the standard error. */\n readonly log?: ((line: string) => void) | undefined;\n}\n\n/** This is a daemon that runs. */\nexport interface Daemon {\n /** This crawler is the crawler of the daemon. The daemon closes it when it stops. */\n readonly crawler: OpenedCrawler;\n /** This file keeps the state of the daemon. */\n readonly stateFile: string;\n /**\n * This value is the MCP server, or undefined.\n *\n * @beta\n */\n readonly mcp: DaemonMcpService | undefined;\n /** This method gives the status at this time. */\n status(): DaemonStatus;\n /**\n * This method stops the daemon. The checks in progress get the grace time to end, then they\n * abort and the run releases their leases. The MCP server stops and cancels its tasks. A second\n * call with `0` aborts the checks at once.\n */\n stop(grace?: Duration): Promise<DaemonStatus>;\n /** This promise resolves with the last status when the daemon stopped. */\n readonly done: Promise<DaemonStatus>;\n}\n\n/**\n * The default MCP host is `mcpHttpHost` of the package polite-crawler-mcp. The daemon loads the\n * package only when it serves MCP, and an error tells where the MCP server is when it is not there.\n */\nconst defaultHost: DaemonMcpHost = {\n async start(crawler, options, hooks) {\n const { loadMcpPackage } = await import(\"./mcp-package.ts\");\n const { mcpHttpHost } = await loadMcpPackage();\n return mcpHttpHost.start(crawler, options, hooks);\n },\n};\n\n/** This function gives the writer of the change events. */\nconst eventWriter = (events: DaemonOptions[\"events\"]): ((line: string) => void) | undefined => {\n if (events === false) return undefined;\n if (typeof events === \"function\") return events;\n if (events === undefined) return (line) => void process.stdout.write(`${line}\\n`);\n mkdirSync(dirname(events), { recursive: true });\n return (line) => appendFileSync(events, `${line}\\n`, { mode: 0o600 });\n};\n\n/**\n * This function starts a daemon on a store. It opens the crawler, serves MCP when `mcp` is given,\n * and checks the watch list on a schedule (`scheduleWatches`). It writes each change event as one\n * JSON line, and it gives each change to `onChange`. The state file tells `daemon status` what runs.\n *\n * Only one daemon operates on a store: a second start rejects. At `SIGINT` or `SIGTERM`, the daemon\n * stops cleanly (`stop`). A second signal aborts the checks at once.\n */\nexport const startDaemon = async (options: DaemonOptions): Promise<Daemon> => {\n const {\n tick,\n jitter,\n budget,\n maxPages,\n grace,\n events,\n onChange,\n mcp,\n mcpHost,\n signals = true,\n stateFile: givenState,\n log = (line: string) => void process.stderr.write(`${line}\\n`),\n ...open\n } = options;\n const tickMs = msOf(\"tick\", tick ?? DAEMON_DEFAULTS.tickMs, DAEMON_DEFAULTS.minTickMs);\n const jitterMs = msOf(\"jitter\", jitter ?? DAEMON_DEFAULTS.jitterMs, 0);\n const budgetMs = msOf(\"budget\", budget ?? DAEMON_DEFAULTS.budgetMs, 0);\n const pages = pagesOf(maxPages);\n const graceMs = msOf(\"grace\", grace ?? DAEMON_DEFAULTS.graceMs, 0);\n const stateFile = givenState ?? daemonStateFile(open.dir);\n const before = readDaemonState(open.dir, stateFile);\n if (before?.alive === true) {\n throw new Error(\n `a daemon runs on the store ${open.dir} already (process ${String(before.pid)}): stop it first, or remove ${stateFile} when no daemon runs`,\n );\n }\n const writeEvent = eventWriter(events);\n // a missing browser turns rendering off one time, at start\n const renderOff = open.renderer === undefined ? await browserMissing() : undefined;\n const crawler = openCrawler({\n ...open,\n ...(renderOff === undefined ? {} : { renderer: false as const }),\n });\n const clock = open.clock ?? realClock;\n const startedAt = iso(clock.now());\n let lastError: DaemonStatus[\"last_error\"] = null;\n let service: DaemonMcpService | undefined;\n let tokenFile: string | null = null;\n let schedule: WatchSchedule | undefined;\n let nextDue: readonly DaemonDue[] = [];\n let stoppedAt: string | null = null;\n\n const status = (): DaemonStatus => {\n const counts = schedule?.result() ?? { ticks: 0, checked: 0, changes: 0, last: null };\n return {\n schema: DAEMON_STATE_SCHEMA,\n version: VERSION,\n pid: process.pid,\n store: open.dir,\n running: stoppedAt === null,\n started_at: startedAt,\n updated_at: iso(clock.now()),\n stopped_at: stoppedAt,\n tick_ms: tickMs,\n jitter_ms: jitterMs,\n budget_ms: budgetMs,\n max_pages: pages,\n ticks: counts.ticks,\n checked: counts.checked,\n changes: counts.changes,\n last_tick: counts.last,\n next_tick_at: stoppedAt === null ? (counts.last?.next_at ?? startedAt) : null,\n next_due: nextDue,\n tasks: service?.tasks() ?? [],\n mcp: service === undefined ? null : { url: service.url, token_file: tokenFile },\n last_error: lastError,\n };\n };\n const save = (): void => {\n try {\n writeState(stateFile, status());\n } catch (error) {\n log(`polite-crawler: the state file ${stateFile} does not write: ${errorOf(error).message}`);\n }\n };\n const fail = (error: unknown, where: string): void => {\n const message = `${where}: ${errorOf(error).message}`;\n lastError = { at: iso(clock.now()), message };\n log(`polite-crawler: ${message}`);\n };\n\n if (mcp !== undefined) {\n let given: DaemonMcpOptions = { ...mcp, renderOff: mcp.renderOff ?? renderOff };\n try {\n if (mcp.token === undefined && mcp.tokenFile === undefined) {\n // a new token at each start: a token that leaked ends at the next start\n tokenFile = join(dirname(stateFile), \"mcp-token\");\n given = { ...given, token: writeToken(tokenFile) };\n } else tokenFile = mcp.tokenFile ?? null;\n service = await (mcpHost ?? defaultHost).start(crawler, given, {\n status,\n onTasks: save,\n log,\n });\n } catch (error) {\n await crawler.close();\n throw error;\n }\n }\n\n // the hooks run in order, one at a time, so a slow hook does not start many processes\n let hooks: Promise<void> = Promise.resolve();\n const changed = (event: WatchEvent): void => {\n if (writeEvent !== undefined) {\n try {\n writeEvent(JSON.stringify(event));\n } catch (error) {\n fail(error, \"the event log\");\n }\n }\n if (onChange !== undefined) {\n hooks = hooks\n .then(() => onChange(event))\n .catch((error: unknown) => fail(error, \"the on-change hook\"));\n }\n };\n\n RUNNING.add(stateFile);\n const running = scheduleWatches(crawler.watch, {\n tick: tickMs,\n jitter: jitterMs,\n budget: budgetMs,\n maxPages: pages,\n clock,\n random: open.random,\n onEvent: (event) => {\n if (event.confirmed) changed(event);\n },\n onTick: (done) => {\n nextDue = done.due;\n // a run that failed went to onError\n if (done.summary !== null && done.summary.checked > 0) {\n const summary = done.summary;\n log(\n `polite-crawler: ${String(summary.checked)} checked: ${String(summary.confirmed)} changes, ${String(summary.suspected)} suspected, ${String(summary.refused)} refused, ${String(summary.error)} errors. Next tick at ${done.next_at ?? \"the stop\"}.`,\n );\n }\n save();\n },\n onError: (error) => fail(error, \"a tick\"),\n });\n schedule = running;\n save();\n log(\n `polite-crawler ${VERSION}: the daemon runs on the store ${open.dir}, a tick each ${String(tickMs)} ms${service === undefined ? \"\" : `, MCP at ${service.url}${tokenFile === null ? \"\" : ` (the token is in ${tokenFile})`}`}`,\n );\n if (renderOff !== undefined) log(`polite-crawler: rendering is off: ${renderOff}`);\n\n let resolveDone: (final: DaemonStatus) => void = () => undefined;\n const done = new Promise<DaemonStatus>((resolve) => {\n resolveDone = resolve;\n });\n let stopping: Promise<DaemonStatus> | undefined;\n let signalled = 0;\n const onSignal = (signal: NodeJS.Signals): void => {\n signalled += 1;\n log(\n signalled > 1\n ? `polite-crawler: ${signal} again: the checks in progress abort now`\n : `polite-crawler: ${signal}: the daemon stops, and the checks in progress get ${String(graceMs)} ms`,\n );\n void stop(signalled > 1 ? 0 : graceMs);\n };\n const stop = (stopGrace: Duration = graceMs): Promise<DaemonStatus> => {\n const ms = msOf(\"grace\", stopGrace, 0);\n void running.stop(ms);\n stopping ??= (async () => {\n const closing = service?.close().catch((error: unknown) => fail(error, \"the MCP server\"));\n await running.done;\n // the hooks in the queue get the grace too\n let timer: ReturnType<typeof setTimeout> | undefined;\n await Promise.race([\n hooks,\n new Promise<void>((resolve) => {\n timer = setTimeout(resolve, ms);\n }),\n ]);\n clearTimeout(timer);\n await closing;\n stoppedAt = iso(clock.now());\n save();\n RUNNING.delete(stateFile);\n await crawler.close();\n if (signals) {\n process.removeListener(\"SIGINT\", onSignal);\n process.removeListener(\"SIGTERM\", onSignal);\n }\n log(\"polite-crawler: the daemon stopped\");\n const final = status();\n resolveDone(final);\n return final;\n })();\n return stopping;\n };\n if (signals) {\n process.on(\"SIGINT\", onSignal);\n process.on(\"SIGTERM\", onSignal);\n }\n return { crawler, stateFile, mcp: service, status, stop, done };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAiCA,MAAa,kBAAkB;;CAE7B,QAAQ;;CAER,UAAU;;CAEV,UAAU;;CAEV,UAAU;;CAEV,SAAS;;CAET,UAAU;;CAEV,WAAW;;CAEX,UAAU;;CAEV,eAAe;AACjB;;AAGA,MAAa,sBAAsB;AA0EnC,MAAM,OAAO,OAAuB,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;;AAG7D,MAAM,QAAQ,MAAc,OAAiB,UAA0B;CACrE,IAAI;CACJ,IAAI;EACF,KAAK,WAAW,KAAK;CACvB,SAAS,OAAO;EACd,MAAM,IAAI,WAAW,GAAG,KAAK,IAAK,MAAgB,WAAW,EAAE,OAAO,MAAM,CAAC;CAC/E;CACA,IAAI,KAAK,OACP,MAAM,IAAI,WAAW,GAAG,KAAK,WAAW,OAAO,KAAK,EAAE,mBAAmB,OAAO,EAAE,GAAG;CACvF,OAAO;AACT;AAEA,MAAM,WAAW,UAAsC;CACrD,MAAM,QAAQ,SAAS,gBAAgB;CACvC,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GACtC,MAAM,IAAI,WAAW,iDAAiD,OAAO,KAAK,GAAG;CACvF,OAAO;AACT;AAEA,MAAM,WAAW,UACf,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;;AAG1D,MAAM,SAAS,YACb,CAAC,GAAG,OAAO,CAAC,CACT,MAAM,GAAG,MAAM,KAAK,MAAM,EAAE,OAAO,IAAI,KAAK,MAAM,EAAE,OAAO,KAAK,EAAE,IAAI,cAAc,EAAE,GAAG,CAAC,CAAC,CAC3F,MAAM,GAAG,gBAAgB,QAAQ,CAAC,CAClC,KAAK,WAAW;CACf,KAAK,MAAM;CACX,SAAS,MAAM;CACf,UAAU,MAAM;CAChB,WAAW,MAAM;AACnB,EAAE;;;;;;;;;AAUN,MAAa,cACX,SACA,SACA,KACA,YACW;CACX,IAAI,OAAO,MAAM,QAAQ;CACzB,MAAM,WAAW,YAAY,QAAQ,QAAQ,YAAY;CACzD,KAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,QAAQ,MAAM,iBAAiB,OAAO,IAAI,KAAK,MAAM,MAAM,YAAY;EAC7E,MAAM,MAAM,KAAK,IAAI,KAAK,MAAM,MAAM,OAAO,GAAG,KAAK;EACrD,IAAI,OAAO,OAAO,UAAU;EAC5B,OAAO,KAAK,IAAI,MAAM,KAAK,IAAI,KAAK,MAAM,gBAAgB,QAAQ,CAAC;CACrE;CACA,MAAM,SAAS,KAAK,IAAI,KAAK,IAAI,QAAQ,OAAO,GAAG,CAAC,GAAG,CAAC;CACxD,OAAO,OAAO,KAAK,IAAI,QAAQ,UAAU,KAAK,MAAM,SAAS,QAAQ,QAAQ,CAAC;AAChF;;;;;;;;;AAUA,MAAa,mBAAmB,OAAkB,UAA2B,CAAC,MAAqB;CACjG,MAAM,SAAS,KAAK,QAAQ,QAAQ,QAAQ,gBAAgB,QAAQ,gBAAgB,SAAS;CAC7F,MAAM,WAAW,KAAK,UAAU,QAAQ,UAAU,gBAAgB,UAAU,CAAC;CAC7E,MAAM,WAAW,KAAK,UAAU,QAAQ,UAAU,gBAAgB,UAAU,CAAC;CAC7E,MAAM,WAAW,QAAQ,QAAQ,QAAQ;CACzC,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,SAAS,QAAQ,UAAU,KAAK;CAEtC,MAAM,QAAQ,IAAI,gBAAgB;CAClC,MAAM,SAAS,IAAI,gBAAgB;CACnC,MAAM,OAAO,IAAI,gBAAgB;CACjC,IAAI,WAAW;CACf,MAAM,SAAS;EAAE,OAAO;EAAG,SAAS;EAAG,SAAS;EAAG,MAAM;CAA0B;CACnF,MAAM,gBAAgC,EAAE,GAAG,OAAO;CAClD,MAAM,UAAU,UAAyB;EACvC,IAAI;GACF,QAAQ,UAAU,QAAQ,KAAK,CAAC;EAClC,QAAQ,CAER;CACF;CACA,MAAM,UAAU,SAA2B;EACzC,IAAI;GACF,KAAK;EACP,SAAS,OAAO;GACd,OAAO,KAAK;EACd;CACF;CAEA,MAAM,OAAO,YAA2B;EACtC,OAAO,CAAC,UAAU;GAChB,MAAM,UAAU,MAAM,IAAI;GAC1B,IAAI,UAA+B;GACnC,IAAI,UAAyB;GAC7B,IAAI;IACF,MAAM,MAAM,MAAM,IAAI;KACpB;KACA;KACA,QAAQ,OAAO;KACf,OAAO,MAAM;IACf,CAAC;IACD,WAAW,MAAM,SAAS,KAAK;KAC7B,OAAO,WAAW;KAClB,IAAI,MAAM,WAAW,OAAO,WAAW;KACvC,aAAa,QAAQ,UAAU,KAAK,CAAC;IACvC;IACA,UAAU,MAAM,IAAI;GACtB,SAAS,OAAO;IACd,UAAU,QAAQ,KAAK,CAAC,CAAC;IACzB,OAAO,KAAK;GACd;GACA,OAAO,SAAS;GAEhB,IAAI,UAAwB,CAAC;GAC7B,IAAI;IACF,UAAU,MAAM,MAAM,KAAK;GAC7B,SAAS,OAAO;IACd,OAAO,KAAK;GACd;GACA,MAAM,OAAO,WACT,OACA,WAAW,SAAS,SAAS,MAAM,IAAI,GAAG;IAAE;IAAQ;IAAU;GAAO,CAAC;GAC1E,MAAM,OAAmB;IACvB,YAAY,IAAI,OAAO;IACvB,UAAU,IAAI,MAAM,IAAI,CAAC;IACzB;IACA,OAAO;IACP,SAAS,SAAS,OAAO,OAAO,IAAI,IAAI;IACxC,KAAK,MAAM,OAAO;GACpB;GACA,OAAO,OAAO;GACd,aAAa,QAAQ,SAAS,IAAI,CAAC;GACnC,IAAI,SAAS,QAAQ,UAAU;GAC/B,MAAM,MAAM,MAAM,KAAK,IAAI,GAAG,OAAO,MAAM,IAAI,CAAC,GAAG,KAAK,MAAM;EAChE;CACF;CAEA,MAAM,OAAO,KAAK,CAAC,CAAC,KAAK,SAAS,UAAmB;EACnD,OAAO,KAAK;EACZ,OAAO,OAAO;CAChB,CAAC;CACD,OAAO;EACL;EACA;EACA,KAAK,QAAQ,gBAAgB,SAAS;GACpC,MAAM,UAAU,KAAK,SAAS,OAAO,CAAC;GACtC,WAAW;GACX,MAAM,sBAAM,IAAI,MAAM,oBAAoB,CAAC;GAC3C,KAAK,sBAAM,IAAI,MAAM,oBAAoB,CAAC;GAC1C,IAAI,YAAY,GAAG,OAAO,sBAAM,IAAI,MAAM,8BAA8B,CAAC;QACpE;IAEH,MAAM,QAAQ,iBACN,OAAO,sBAAM,IAAI,MAAM,kCAAkC,CAAC,GAChE,OACF;IACA,KAAU,cAAc,aAAa,KAAK,CAAC;GAC7C;GACA,OAAO;EACT;CACF;AACF;;AA6DA,MAAM,0BAAU,IAAI,IAAY;;AAGhC,MAAM,gBAAgB,QAAyB;CAC7C,IAAI,CAAC,OAAO,UAAU,GAAG,KAAK,OAAO,GAAG,OAAO;CAC/C,IAAI;EACF,QAAQ,KAAK,KAAK,CAAC;EACnB,OAAO;CACT,SAAS,OAAO;EACd,OAAQ,MAA6B,SAAS;CAChD;AACF;;AAGA,MAAa,mBAAmB,QAAwB,KAAK,KAAK,UAAU,YAAY;;;;;AAMxF,MAAa,mBACX,KACA,OAAe,gBAAgB,GAAG,MACN;CAC5B,IAAI;CACJ,IAAI;EACF,QAAQ,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;CAC/C,QAAQ;EACN;CACF;CACA,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAO,MAAM,QAAQ,UACtE,OAAO,KAAA;CACT,MAAM,QACJ,MAAM,YAAY,SACjB,MAAM,QAAQ,QAAQ,MAAM,QAAQ,IAAI,IAAI,IAAI,aAAa,MAAM,GAAG;CACzE,OAAO;EAAE,GAAG;EAAO;CAAM;AAC3B;;AAGA,MAAM,cAAc,MAAc,UAA8B;CAC9D,UAAU,QAAQ,IAAI,GAAG;EAAE,WAAW;EAAM,MAAM;CAAM,CAAC;CACzD,MAAM,OAAO,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,EAAE;CAC/C,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,QAAQ,GAAG,EAAE;CAC5C,cAAc,MAAM,MAAM,EAAE,MAAM,IAAM,CAAC;CACzC,IAAI;EACF,WAAW,MAAM,IAAI;CACvB,QAAQ;EAEN,cAAc,MAAM,MAAM,EAAE,MAAM,IAAM,CAAC;EACzC,IAAI;GACF,WAAW,IAAI;EACjB,QAAQ,CAER;CACF;AACF;;AAGA,MAAM,cAAc,SAAyB;CAC3C,MAAM,QAAQ,YAAY,EAAE,CAAC,CAAC,SAAS,WAAW;CAClD,UAAU,QAAQ,IAAI,GAAG;EAAE,WAAW;EAAM,MAAM;CAAM,CAAC;CACzD,IAAI;EAEF,WAAW,IAAI;CACjB,QAAQ,CAER;CACA,cAAc,MAAM,GAAG,MAAM,KAAK;EAAE,MAAM;EAAO,MAAM;CAAK,CAAC;CAC7D,OAAO;AACT;;;;;;;;;;AAqBA,MAAa,iBACX,SACA,UAAgC,CAAC,MACU;CAC3C,IAAI,QAAQ,KAAK,MAAM,IAAI,MAAM,IAAI,WAAW,gCAAgC;CAChF,MAAM,YAAY,KAAK,WAAW,QAAQ,WAAW,gBAAgB,eAAe,CAAC;CACrF,QAAQ,UACN,IAAI,SAAe,SAAS,WAAW;EACrC,MAAM,QAAQ,MAAM,SAAS;GAC3B,OAAO;GACP,OAAO;IAAC;IAAQ;IAAU;GAAM;GAChC,aAAa;GACb,GAAI,QAAQ,QAAQ,KAAA,IAAY,CAAC,IAAI,EAAE,KAAK,QAAQ,IAAI;EAC1D,CAAC;EACD,IAAI,OAAO;EACX,IAAI,OAAO;EACX,MAAM,QAAQ,iBAAiB;GAC7B,OAAO;GACP,MAAM,KAAK;EACb,GAAG,SAAS;EACZ,MAAM,OAAO,YAAY,MAAM;EAC/B,MAAM,OAAO,GAAG,SAAS,SAAiB;GACxC,QAAQ,OAAO,KAAA,CAAM,MAAM,IAAK;EAClC,CAAC;EAED,MAAM,MAAM,GAAG,eAAe,KAAA,CAAS;EACvC,MAAM,GAAG,UAAU,UAAU;GAC3B,aAAa,KAAK;GAClB,OACE,IAAI,MAAM,wCAAwC,MAAM,WAAW,EAAE,OAAO,MAAM,CAAC,CACrF;EACF,CAAC;EACD,MAAM,GAAG,UAAU,MAAM,WAAW;GAClC,aAAa,KAAK;GAClB,MAAM,OAAO,KAAK,KAAK,MAAM,KAAK,KAAK,KAAK,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;GAClE,IAAI,MACF,uBACE,IAAI,MACF,2CAA2C,OAAO,SAAS,EAAE,8BAC/D,CACF;QACK,IAAI,SAAS,GAAG,QAAQ;QAE7B,uBACE,IAAI,MACF,qCAAqC,SAAS,OAAO,OAAO,MAAM,IAAI,QAAQ,OAAO,IAAI,MAAM,MACjG,CACF;EACJ,CAAC;EACD,MAAM,MAAM,IAAI,GAAG,KAAK,UAAU,KAAK,EAAE,GAAG;CAC9C,CAAC;AACL;;;;;AA0IA,MAAM,cAA6B,EACjC,MAAM,MAAM,SAAS,SAAS,OAAO;CACnC,MAAM,EAAE,mBAAmB,MAAM,OAAO;CACxC,MAAM,EAAE,gBAAgB,MAAM,eAAe;CAC7C,OAAO,YAAY,MAAM,SAAS,SAAS,KAAK;AAClD,EACF;;AAGA,MAAM,eAAe,WAA0E;CAC7F,IAAI,WAAW,OAAO,OAAO,KAAA;CAC7B,IAAI,OAAO,WAAW,YAAY,OAAO;CACzC,IAAI,WAAW,KAAA,GAAW,QAAQ,SAAS,KAAK,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;CAChF,UAAU,QAAQ,MAAM,GAAG,EAAE,WAAW,KAAK,CAAC;CAC9C,QAAQ,SAAS,eAAe,QAAQ,GAAG,KAAK,KAAK,EAAE,MAAM,IAAM,CAAC;AACtE;;;;;;;;;AAUA,MAAa,cAAc,OAAO,YAA4C;CAC5E,MAAM,EACJ,MACA,QACA,QACA,UACA,OACA,QACA,UACA,KACA,SACA,UAAU,MACV,WAAW,YACX,OAAO,SAAiB,KAAK,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG,GAC7D,GAAG,SACD;CACJ,MAAM,SAAS,KAAK,QAAQ,QAAQ,gBAAgB,QAAQ,gBAAgB,SAAS;CACrF,MAAM,WAAW,KAAK,UAAU,UAAU,gBAAgB,UAAU,CAAC;CACrE,MAAM,WAAW,KAAK,UAAU,UAAU,gBAAgB,UAAU,CAAC;CACrE,MAAM,QAAQ,QAAQ,QAAQ;CAC9B,MAAM,UAAU,KAAK,SAAS,SAAS,gBAAgB,SAAS,CAAC;CACjE,MAAM,YAAY,cAAc,gBAAgB,KAAK,GAAG;CACxD,MAAM,SAAS,gBAAgB,KAAK,KAAK,SAAS;CAClD,IAAI,QAAQ,UAAU,MACpB,MAAM,IAAI,MACR,8BAA8B,KAAK,IAAI,oBAAoB,OAAO,OAAO,GAAG,EAAE,8BAA8B,UAAU,qBACxH;CAEF,MAAM,aAAa,YAAY,MAAM;CAErC,MAAM,YAAY,KAAK,aAAa,KAAA,IAAY,MAAM,eAAe,IAAI,KAAA;CACzE,MAAM,UAAU,YAAY;EAC1B,GAAG;EACH,GAAI,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,MAAe;CAChE,CAAC;CACD,MAAM,QAAQ,KAAK,SAAS;CAC5B,MAAM,YAAY,IAAI,MAAM,IAAI,CAAC;CACjC,IAAI,YAAwC;CAC5C,IAAI;CACJ,IAAI,YAA2B;CAC/B,IAAI;CACJ,IAAI,UAAgC,CAAC;CACrC,IAAI,YAA2B;CAE/B,MAAM,eAA6B;EACjC,MAAM,SAAS,UAAU,OAAO,KAAK;GAAE,OAAO;GAAG,SAAS;GAAG,SAAS;GAAG,MAAM;EAAK;EACpF,OAAO;GACL,QAAA;GACA,SAAS;GACT,KAAK,QAAQ;GACb,OAAO,KAAK;GACZ,SAAS,cAAc;GACvB,YAAY;GACZ,YAAY,IAAI,MAAM,IAAI,CAAC;GAC3B,YAAY;GACZ,SAAS;GACT,WAAW;GACX,WAAW;GACX,WAAW;GACX,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,SAAS,OAAO;GAChB,WAAW,OAAO;GAClB,cAAc,cAAc,OAAQ,OAAO,MAAM,WAAW,YAAa;GACzE,UAAU;GACV,OAAO,SAAS,MAAM,KAAK,CAAC;GAC5B,KAAK,YAAY,KAAA,IAAY,OAAO;IAAE,KAAK,QAAQ;IAAK,YAAY;GAAU;GAC9E,YAAY;EACd;CACF;CACA,MAAM,aAAmB;EACvB,IAAI;GACF,WAAW,WAAW,OAAO,CAAC;EAChC,SAAS,OAAO;GACd,IAAI,kCAAkC,UAAU,mBAAmB,QAAQ,KAAK,CAAC,CAAC,SAAS;EAC7F;CACF;CACA,MAAM,QAAQ,OAAgB,UAAwB;EACpD,MAAM,UAAU,GAAG,MAAM,IAAI,QAAQ,KAAK,CAAC,CAAC;EAC5C,YAAY;GAAE,IAAI,IAAI,MAAM,IAAI,CAAC;GAAG;EAAQ;EAC5C,IAAI,mBAAmB,SAAS;CAClC;CAEA,IAAI,QAAQ,KAAA,GAAW;EACrB,IAAI,QAA0B;GAAE,GAAG;GAAK,WAAW,IAAI,aAAa;EAAU;EAC9E,IAAI;GACF,IAAI,IAAI,UAAU,KAAA,KAAa,IAAI,cAAc,KAAA,GAAW;IAE1D,YAAY,KAAK,QAAQ,SAAS,GAAG,WAAW;IAChD,QAAQ;KAAE,GAAG;KAAO,OAAO,WAAW,SAAS;IAAE;GACnD,OAAO,YAAY,IAAI,aAAa;GACpC,UAAU,OAAO,WAAW,YAAA,CAAa,MAAM,SAAS,OAAO;IAC7D;IACA,SAAS;IACT;GACF,CAAC;EACH,SAAS,OAAO;GACd,MAAM,QAAQ,MAAM;GACpB,MAAM;EACR;CACF;CAGA,IAAI,QAAuB,QAAQ,QAAQ;CAC3C,MAAM,WAAW,UAA4B;EAC3C,IAAI,eAAe,KAAA,GACjB,IAAI;GACF,WAAW,KAAK,UAAU,KAAK,CAAC;EAClC,SAAS,OAAO;GACd,KAAK,OAAO,eAAe;EAC7B;EAEF,IAAI,aAAa,KAAA,GACf,QAAQ,MACL,WAAW,SAAS,KAAK,CAAC,CAAC,CAC3B,OAAO,UAAmB,KAAK,OAAO,oBAAoB,CAAC;CAElE;CAEA,QAAQ,IAAI,SAAS;CACrB,MAAM,UAAU,gBAAgB,QAAQ,OAAO;EAC7C,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,UAAU;EACV;EACA,QAAQ,KAAK;EACb,UAAU,UAAU;GAClB,IAAI,MAAM,WAAW,QAAQ,KAAK;EACpC;EACA,SAAS,SAAS;GAChB,UAAU,KAAK;GAEf,IAAI,KAAK,YAAY,QAAQ,KAAK,QAAQ,UAAU,GAAG;IACrD,MAAM,UAAU,KAAK;IACrB,IACE,mBAAmB,OAAO,QAAQ,OAAO,EAAE,YAAY,OAAO,QAAQ,SAAS,EAAE,YAAY,OAAO,QAAQ,SAAS,EAAE,cAAc,OAAO,QAAQ,OAAO,EAAE,YAAY,OAAO,QAAQ,KAAK,EAAE,wBAAwB,KAAK,WAAW,WAAW,EACpP;GACF;GACA,KAAK;EACP;EACA,UAAU,UAAU,KAAK,OAAO,QAAQ;CAC1C,CAAC;CACD,WAAW;CACX,KAAK;CACL,IACE,kBAAkB,QAAQ,iCAAiC,KAAK,IAAI,gBAAgB,OAAO,MAAM,EAAE,KAAK,YAAY,KAAA,IAAY,KAAK,YAAY,QAAQ,MAAM,cAAc,OAAO,KAAK,qBAAqB,UAAU,MAC1N;CACA,IAAI,cAAc,KAAA,GAAW,IAAI,qCAAqC,WAAW;CAEjF,IAAI,oBAAmD,KAAA;CACvD,MAAM,OAAO,IAAI,SAAuB,YAAY;EAClD,cAAc;CAChB,CAAC;CACD,IAAI;CACJ,IAAI,YAAY;CAChB,MAAM,YAAY,WAAiC;EACjD,aAAa;EACb,IACE,YAAY,IACR,mBAAmB,OAAO,4CAC1B,mBAAmB,OAAO,qDAAqD,OAAO,OAAO,EAAE,IACrG;EACA,KAAU,YAAY,IAAI,IAAI,OAAO;CACvC;CACA,MAAM,QAAQ,YAAsB,YAAmC;EACrE,MAAM,KAAK,KAAK,SAAS,WAAW,CAAC;EACrC,QAAa,KAAK,EAAE;EACpB,cAAc,YAAY;GACxB,MAAM,UAAU,SAAS,MAAM,CAAC,CAAC,OAAO,UAAmB,KAAK,OAAO,gBAAgB,CAAC;GACxF,MAAM,QAAQ;GAEd,IAAI;GACJ,MAAM,QAAQ,KAAK,CACjB,OACA,IAAI,SAAe,YAAY;IAC7B,QAAQ,WAAW,SAAS,EAAE;GAChC,CAAC,CACH,CAAC;GACD,aAAa,KAAK;GAClB,MAAM;GACN,YAAY,IAAI,MAAM,IAAI,CAAC;GAC3B,KAAK;GACL,QAAQ,OAAO,SAAS;GACxB,MAAM,QAAQ,MAAM;GACpB,IAAI,SAAS;IACX,QAAQ,eAAe,UAAU,QAAQ;IACzC,QAAQ,eAAe,WAAW,QAAQ;GAC5C;GACA,IAAI,oCAAoC;GACxC,MAAM,QAAQ,OAAO;GACrB,YAAY,KAAK;GACjB,OAAO;EACT,EAAA,CAAG;EACH,OAAO;CACT;CACA,IAAI,SAAS;EACX,QAAQ,GAAG,UAAU,QAAQ;EAC7B,QAAQ,GAAG,WAAW,QAAQ;CAChC;CACA,OAAO;EAAE;EAAS;EAAW,KAAK;EAAS;EAAQ;EAAM;CAAK;AAChE"}
1
+ {"version":3,"file":"daemon-BPswUOgu.mjs","names":[],"sources":["../src/daemon.ts"],"sourcesContent":["import { spawn } from \"node:child_process\";\nimport { randomBytes } from \"node:crypto\";\nimport {\n appendFileSync,\n mkdirSync,\n readFileSync,\n renameSync,\n unlinkSync,\n writeFileSync,\n} from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { browserMissing } from \"./browser.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport type { OpenOptions, OpenedCrawler } from \"./open.ts\";\nimport { openCrawler } from \"./open.ts\";\nimport type { Clock } from \"./politeness.ts\";\nimport { realClock } from \"./politeness.ts\";\nimport { VERSION } from \"./version.ts\";\nimport type { WatchEntry, WatchEvent, WatchList, WatchSummary } from \"./watch.ts\";\n\n/*\n * The daemon is a local process that runs for a long time. It does these jobs:\n *\n * - It checks the watch list on a schedule (`scheduleWatches`). Each entry that is due gets its\n * check at its time, with each polite rule of `watch.run`.\n * - It writes each change as one JSON line, and it can start a command of the operator for it.\n * - It can serve the MCP tools over HTTP, with the long tools as MCP tasks. This part loads the MCP\n * SDK only when the daemon serves MCP, through a small seam (`DaemonMcpHost`).\n * - It writes a small state file, which `polite-crawler daemon status` reads.\n */\n\n/** These are the defaults of the daemon. */\nexport const DAEMON_DEFAULTS = {\n /** The longest wait between two ticks: 1 minute. A tick comes sooner when an entry is due sooner. */\n tickMs: 60_000,\n /** The most random time that the daemon adds to each wait: 10 seconds. */\n jitterMs: 10_000,\n /** The budget of the checks of one tick: 10 minutes. No request starts after it. */\n budgetMs: 600_000,\n /** The most checks of one tick. */\n maxPages: 100,\n /** At a stop, the checks in progress get this time to end: 10 seconds. Then they abort. */\n graceMs: 10_000,\n /** The shortest wait between two ticks: 1 second. */\n minGapMs: 1_000,\n /** The shortest tick: 1 second. */\n minTickMs: 1_000,\n /** The status lists this number of the next due entries. */\n dueShown: 10,\n /** An on-change command that runs longer than this time stops: 1 minute. */\n hookTimeoutMs: 60_000,\n} as const;\n\n/** This value is the version of the format of the state file. */\nexport const DAEMON_STATE_SCHEMA = 1;\n\n/** This is a watch entry that is due soon, as the status shows it. */\nexport interface DaemonDue {\n readonly url: string;\n /** This time (ISO 8601) is the time when the entry is due. */\n readonly next_at: string;\n readonly every_ms: number;\n /** This value is the kind of the last check, or null before the first check. */\n readonly last_kind: string | null;\n}\n\n/** This is one tick of the schedule: one run of the watch list. */\nexport interface DaemonTick {\n /** This time (ISO 8601) is the start of the run. */\n readonly started_at: string;\n /** This time (ISO 8601) is the end of the run. */\n readonly ended_at: string;\n /** This value is the summary of the run, or null when the run failed. */\n readonly summary: WatchSummary | null;\n /** This text tells why the run failed, or null. */\n readonly error: string | null;\n /** This time (ISO 8601) is the time of the next tick, or null when the schedule stops. */\n readonly next_at: string | null;\n /** These entries are the next due entries after the run, the first due first. */\n readonly due: readonly DaemonDue[];\n}\n\n/** These options control the schedule of the watch list. */\nexport interface ScheduleOptions {\n /** This value is the longest wait between two ticks. The default is `1m`, and the floor is `1s`. */\n readonly tick?: Duration | undefined;\n /** This value is the most random time that the schedule adds to each wait. The default is `10s`. */\n readonly jitter?: Duration | undefined;\n /** This value is the budget of the checks of one tick. The default is `10m`. */\n readonly budget?: Duration | undefined;\n /** This value is the most checks of one tick. The default is 100. */\n readonly maxPages?: number | undefined;\n /** This clock gives the time and the waits. Give the clock of the crawler. The default is the real clock. */\n readonly clock?: Clock | undefined;\n /** This function gives the random numbers of the jitter. The default is `Math.random`. */\n readonly random?: (() => number) | undefined;\n /** This function gets each event of each check. */\n readonly onEvent?: ((event: WatchEvent) => void) | undefined;\n /** This function gets each tick when its run ends. */\n readonly onTick?: ((tick: DaemonTick) => void) | undefined;\n /** This function gets the error of a run that failed. The schedule goes on at the next tick. */\n readonly onError?: ((error: Error) => void) | undefined;\n}\n\n/** This is the result of a schedule: the counts so far, and the last tick. */\nexport interface ScheduleResult {\n readonly ticks: number;\n /** This value counts the checks that gave an event. */\n readonly checked: number;\n /** This value counts the confirmed changes. */\n readonly changes: number;\n readonly last: DaemonTick | null;\n}\n\n/** A schedule that runs. `stop` ends it, and `done` resolves when it ends. */\nexport interface WatchSchedule {\n /** This promise resolves when the schedule ends. It does not reject. */\n readonly done: Promise<ScheduleResult>;\n /**\n * This method stops the schedule. The run in progress starts no new check. The checks in\n * progress get `grace` (10 seconds by default) to end and write their state. Then they abort,\n * and the run releases their leases. A second call with a shorter grace shortens it.\n */\n stop(grace?: Duration): Promise<ScheduleResult>;\n /** This method gives the counts so far. */\n result(): ScheduleResult;\n}\n\nconst iso = (ms: number): string => new Date(ms).toISOString();\n\n/** This function reads a duration of an option, with a floor. A bad value throws a `RangeError`. */\nconst msOf = (name: string, value: Duration, floor: number): number => {\n let ms: number;\n try {\n ms = durationMs(value);\n } catch (error) {\n throw new RangeError(`${name}: ${(error as Error).message}`, { cause: error });\n }\n if (ms < floor)\n throw new RangeError(`${name} must be ${String(floor)} ms or more, not ${String(ms)}`);\n return ms;\n};\n\nconst pagesOf = (value: number | undefined): number => {\n const pages = value ?? DAEMON_DEFAULTS.maxPages;\n if (!Number.isInteger(pages) || pages < 1)\n throw new RangeError(`maxPages must be an integer of 1 or more, not ${String(pages)}`);\n return pages;\n};\n\nconst errorOf = (error: unknown): Error =>\n error instanceof Error ? error : new Error(String(error));\n\n/** This function gives the next due entries of a list, as the status shows them. */\nconst dueOf = (entries: readonly WatchEntry[]): DaemonDue[] =>\n [...entries]\n .sort((a, b) => Date.parse(a.next_at) - Date.parse(b.next_at) || a.url.localeCompare(b.url))\n .slice(0, DAEMON_DEFAULTS.dueShown)\n .map((entry) => ({\n url: entry.url,\n next_at: entry.next_at,\n every_ms: entry.every_ms,\n last_kind: entry.last_kind,\n }));\n\n/**\n * This function gives the time of the next tick. It is the time when the first entry is due, or\n * the end of its lease. It is not later than one tick, and not sooner than `minGapMs`. A random\n * jitter from 0 to `jitterMs` comes after it, so a check does not come before its time.\n *\n * When the budget or `maxPages` of a run left due entries, they wait for the next tick, so one\n * tick does not exceed its budget. A run that failed also waits one tick.\n */\nexport const nextTickAt = (\n entries: readonly WatchEntry[],\n summary: WatchSummary | null,\n now: number,\n options: { readonly tickMs: number; readonly jitterMs: number; readonly random: () => number },\n): number => {\n let next = now + options.tickMs;\n const leftover = summary === null || summary.stopped !== \"done\";\n for (const entry of entries) {\n const lease = entry.leased_until === null ? 0 : Date.parse(entry.leased_until);\n const due = Math.max(Date.parse(entry.next_at), lease);\n if (due <= now && leftover) continue;\n next = Math.min(next, Math.max(due, now + DAEMON_DEFAULTS.minGapMs));\n }\n const random = Math.min(Math.max(options.random(), 0), 1);\n return next + Math.min(options.jitterMs, Math.floor(random * options.jitterMs));\n};\n\n/**\n * This function checks a watch list on a schedule. The first tick starts at once. Each tick is one\n * `watch.run` with the budget and `maxPages` of a tick. Thus each polite rule of the run applies:\n * one request at a time to each host, the stops of a host, `Retry-After` and robots.txt.\n *\n * After a run, the schedule waits for the next tick (`nextTickAt`). An error of a run goes to\n * `onError`, and the schedule goes on.\n */\nexport const scheduleWatches = (watch: WatchList, options: ScheduleOptions = {}): WatchSchedule => {\n const tickMs = msOf(\"tick\", options.tick ?? DAEMON_DEFAULTS.tickMs, DAEMON_DEFAULTS.minTickMs);\n const jitterMs = msOf(\"jitter\", options.jitter ?? DAEMON_DEFAULTS.jitterMs, 0);\n const budgetMs = msOf(\"budget\", options.budget ?? DAEMON_DEFAULTS.budgetMs, 0);\n const maxPages = pagesOf(options.maxPages);\n const clock = options.clock ?? realClock;\n const random = options.random ?? Math.random;\n // the drain starts no new check, the cancel aborts the checks in progress, the wake ends a wait\n const drain = new AbortController();\n const cancel = new AbortController();\n const wake = new AbortController();\n let stopping = false;\n const counts = { ticks: 0, checked: 0, changes: 0, last: null as DaemonTick | null };\n const result = (): ScheduleResult => ({ ...counts });\n const report = (error: unknown): void => {\n try {\n options.onError?.(errorOf(error));\n } catch {\n // an error of the error handler stops nothing\n }\n };\n const safely = (work: () => void): void => {\n try {\n work();\n } catch (error) {\n report(error);\n }\n };\n\n const loop = async (): Promise<void> => {\n while (!stopping) {\n const started = clock.now();\n let summary: WatchSummary | null = null;\n let failure: string | null = null;\n try {\n const run = watch.run({\n budgetMs,\n maxPages,\n signal: cancel.signal,\n drain: drain.signal,\n });\n for await (const event of run) {\n counts.checked += 1;\n if (event.confirmed) counts.changes += 1;\n safely(() => options.onEvent?.(event));\n }\n summary = await run.summary;\n } catch (error) {\n failure = errorOf(error).message;\n report(error);\n }\n counts.ticks += 1;\n // the list gives the next due entries to the status, also at the last tick\n let entries: WatchEntry[] = [];\n try {\n entries = await watch.list();\n } catch (error) {\n report(error);\n }\n const next = stopping\n ? null\n : nextTickAt(entries, summary, clock.now(), { tickMs, jitterMs, random });\n const tick: DaemonTick = {\n started_at: iso(started),\n ended_at: iso(clock.now()),\n summary,\n error: failure,\n next_at: next === null ? null : iso(next),\n due: dueOf(entries),\n };\n counts.last = tick;\n safely(() => options.onTick?.(tick));\n if (next === null || stopping) break;\n await clock.sleep(Math.max(0, next - clock.now()), wake.signal);\n }\n };\n\n const done = loop().then(result, (error: unknown) => {\n report(error);\n return result();\n });\n return {\n done,\n result,\n stop(grace = DAEMON_DEFAULTS.graceMs) {\n const graceMs = msOf(\"grace\", grace, 0);\n stopping = true;\n drain.abort(new Error(\"the schedule stops\"));\n wake.abort(new Error(\"the schedule stops\"));\n if (graceMs === 0) cancel.abort(new Error(\"the schedule stopped at once\"));\n else {\n // a real timer: the grace is time of the process, not of the clock of the crawler\n const timer = setTimeout(\n () => cancel.abort(new Error(\"the grace time of the stop ended\")),\n graceMs,\n );\n void done.finally(() => clearTimeout(timer));\n }\n return done;\n },\n };\n};\n\n// ── the state file ──────────────────────────────────────────────────────────\n\n/** This is a task of the MCP server, as the status shows it. */\nexport interface DaemonTask {\n readonly id: string;\n /** This value is the name of the tool, for example `crawl_site`. */\n readonly tool: string;\n /** This value is `working`, `input_required`, `completed`, `failed` or `cancelled`. */\n readonly status: string;\n /** This text tells the progress, for example `3 of 10 done`, or null. */\n readonly message: string | null;\n readonly created_at: string;\n readonly updated_at: string;\n}\n\n/** This is the status of a daemon. The state file (`<store>/daemon/state.json`) holds the same JSON. */\nexport interface DaemonStatus {\n /** This value is the version of the format (`DAEMON_STATE_SCHEMA`). */\n readonly schema: number;\n /** This value is the version of polite-crawler. */\n readonly version: string;\n /** This value is the id of the process of the daemon. */\n readonly pid: number;\n readonly store: string;\n /** This value is false after the daemon stopped. */\n readonly running: boolean;\n readonly started_at: string;\n readonly updated_at: string;\n readonly stopped_at: string | null;\n readonly tick_ms: number;\n readonly jitter_ms: number;\n readonly budget_ms: number;\n readonly max_pages: number;\n /** This value counts the ticks. */\n readonly ticks: number;\n /** This value counts the checks. */\n readonly checked: number;\n /** This value counts the confirmed changes. */\n readonly changes: number;\n readonly last_tick: DaemonTick | null;\n /** This time (ISO 8601) is the time of the next tick, or null. */\n readonly next_tick_at: string | null;\n /** These entries are the next due entries at the last tick, the first due first. */\n readonly next_due: readonly DaemonDue[];\n /** These tasks of the MCP server did not end. */\n readonly tasks: readonly DaemonTask[];\n /** This value is the MCP endpoint and the file of its token, or null with no MCP server. */\n readonly mcp: { readonly url: string; readonly token_file: string | null } | null;\n /** This value is the last error: of a tick, of the event log, of an on-change hook or of the MCP server. */\n readonly last_error: { readonly at: string; readonly message: string } | null;\n}\n\n/** This is a status that `readDaemonState` read from a state file. */\nexport interface DaemonState extends DaemonStatus {\n /** This value is true when the daemon runs: the state says so, and its process is alive. */\n readonly alive: boolean;\n}\n\n/** These state files belong to a daemon that runs in this process. */\nconst RUNNING = new Set<string>();\n\n/** This function tells if a process is alive. A process of a different user is alive too. */\nconst processAlive = (pid: number): boolean => {\n if (!Number.isInteger(pid) || pid <= 0) return false;\n try {\n process.kill(pid, 0);\n return true;\n } catch (error) {\n return (error as { code?: unknown }).code === \"EPERM\";\n }\n};\n\n/** This function gives the path of the state file of a store. */\nexport const daemonStateFile = (dir: string): string => join(dir, \"daemon\", \"state.json\");\n\n/**\n * This function reads the state file of a daemon. It gives undefined when the file is not there,\n * or when it is not a state file. `alive` tells if the daemon runs.\n */\nexport const readDaemonState = (\n dir: string,\n file: string = daemonStateFile(dir),\n): DaemonState | undefined => {\n let state: DaemonStatus;\n try {\n state = JSON.parse(readFileSync(file, \"utf8\")) as DaemonStatus;\n } catch {\n return undefined;\n }\n if (typeof state !== \"object\" || state === null || typeof state.pid !== \"number\")\n return undefined;\n const alive =\n state.running === true &&\n (state.pid === process.pid ? RUNNING.has(file) : processAlive(state.pid));\n return { ...state, alive };\n};\n\n/** This function writes a state file for the owner only. A rename makes the change atomic where the system can. */\nconst writeState = (file: string, state: DaemonStatus): void => {\n mkdirSync(dirname(file), { recursive: true, mode: 0o700 });\n const text = `${JSON.stringify(state, null, 2)}\\n`;\n const temp = `${file}.${String(process.pid)}.tmp`;\n writeFileSync(temp, text, { mode: 0o600 });\n try {\n renameSync(temp, file);\n } catch {\n // Windows can refuse the rename while a reader has the file open\n writeFileSync(file, text, { mode: 0o600 });\n try {\n unlinkSync(temp);\n } catch {\n // the next write replaces it\n }\n }\n};\n\n/** This function writes a new random token to a file for the owner only, and gives the token. */\nconst writeToken = (file: string): string => {\n const token = randomBytes(32).toString(\"base64url\");\n mkdirSync(dirname(file), { recursive: true, mode: 0o700 });\n try {\n // a new file gets the mode: an old file can have a wider mode\n unlinkSync(file);\n } catch {\n // no old file\n }\n writeFileSync(file, `${token}\\n`, { mode: 0o600, flag: \"wx\" });\n return token;\n};\n\n// ── the on-change command ───────────────────────────────────────────────────\n\n/** These options start an on-change command. */\nexport interface ChangeCommandOptions {\n /** A command that runs longer than this time stops. The default is `1m`. */\n readonly timeout?: Duration | undefined;\n /** This folder is the working folder of the command. The default is the working folder of the daemon. */\n readonly cwd?: string | undefined;\n}\n\n/**\n * This function gives an `onChange` hook that starts a command of the operator for each change.\n * The command operates in the shell of the system, as the user of the daemon. It gets the event as\n * one JSON line on its standard input, not in its command line. Thus no text of a page goes into\n * the command line. The output of the command goes nowhere. A command that exits with a code that\n * is not 0, or that runs too long, rejects with the end of its standard error.\n *\n * The command is the decision of the operator. Do not take it from a page or from an assistant.\n */\nexport const changeCommand = (\n command: string,\n options: ChangeCommandOptions = {},\n): ((event: WatchEvent) => Promise<void>) => {\n if (command.trim() === \"\") throw new RangeError(\"the on-change command is empty\");\n const timeoutMs = msOf(\"timeout\", options.timeout ?? DAEMON_DEFAULTS.hookTimeoutMs, 1);\n return (event) =>\n new Promise<void>((resolve, reject) => {\n const child = spawn(command, {\n shell: true,\n stdio: [\"pipe\", \"ignore\", \"pipe\"],\n windowsHide: true,\n ...(options.cwd === undefined ? {} : { cwd: options.cwd }),\n });\n let tail = \"\";\n let late = false;\n const timer = setTimeout(() => {\n late = true;\n child.kill();\n }, timeoutMs);\n child.stderr.setEncoding(\"utf8\");\n child.stderr.on(\"data\", (text: string) => {\n tail = (tail + text).slice(-2000);\n });\n // a command that does not read its input closes the pipe early\n child.stdin.on(\"error\", () => undefined);\n child.on(\"error\", (error) => {\n clearTimeout(timer);\n reject(\n new Error(`the on-change command did not start: ${error.message}`, { cause: error }),\n );\n });\n child.on(\"close\", (code, signal) => {\n clearTimeout(timer);\n const said = tail.trim() === \"\" ? \"\" : `: ${tail.trim().slice(-300)}`;\n if (late) {\n reject(\n new Error(\n `the on-change command ran for more than ${String(timeoutMs)} ms, so the daemon stopped it`,\n ),\n );\n } else if (code === 0) resolve();\n else\n reject(\n new Error(\n `the on-change command exited with ${code === null ? String(signal) : `code ${String(code)}`}${said}`,\n ),\n );\n });\n child.stdin.end(`${JSON.stringify(event)}\\n`);\n });\n};\n\n// ── the daemon ──────────────────────────────────────────────────────────────\n\n/**\n * These options serve the MCP tools over HTTP.\n *\n * @beta\n */\nexport interface DaemonMcpOptions {\n /** This value is the port. The default is 9307. The value 0 takes a free port. */\n readonly port?: number | undefined;\n /** This value is the host to bind. The default is `127.0.0.1`. A host that is not loopback needs `allowRemote`. */\n readonly host?: string | undefined;\n /** The value true permits a host that is not loopback. Read the warning in the README first. */\n readonly allowRemote?: boolean | undefined;\n /** This value is the bearer token. The default makes a new token at each start, in `<store>/daemon/mcp-token`. */\n readonly token?: string | undefined;\n /** This file holds the bearer token of the operator. The daemon reads it and writes nothing. */\n readonly tokenFile?: string | undefined;\n /** These tokens get their robots.txt rules applied to each fetch of a tool. The default is `[\"Claude-User\"]`. */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** The value true lets `user_provided` release held-back text when the client cannot ask the user. */\n readonly allowUserProvided?: boolean | undefined;\n /** This text tells why the server renders no page. */\n readonly renderOff?: string | undefined;\n}\n\n/**\n * These functions connect an MCP host to the daemon.\n *\n * @beta\n */\nexport interface DaemonMcpHooks {\n /** This function gives the status of the daemon, for the status route. */\n readonly status: () => DaemonStatus;\n /** This function operates when a task starts, changes or ends. */\n readonly onTasks: () => void;\n /** This function writes a message of the server. */\n readonly log: (line: string) => void;\n}\n\n/**\n * This is the MCP server of the daemon, as the MCP host gives it.\n *\n * @beta\n */\nexport interface DaemonMcpService {\n /** This URL is the MCP endpoint. */\n readonly url: string;\n /** This method gives the tasks that did not end. */\n tasks(): readonly DaemonTask[];\n /** This method stops the server and cancels its tasks. */\n close(): Promise<void>;\n}\n\n/**\n * This is the seam between the daemon and its MCP server. The default host (`mcpHttpHost` of the\n * package polite-crawler-mcp) serves the tools over Streamable HTTP. A different MCP host can\n * replace it.\n *\n * @beta\n */\nexport interface DaemonMcpHost {\n start(\n crawler: OpenedCrawler,\n options: DaemonMcpOptions,\n hooks: DaemonMcpHooks,\n ): Promise<DaemonMcpService>;\n}\n\n/** These options start a daemon on a store. Only `dir` is necessary. */\nexport interface DaemonOptions extends OpenOptions {\n /** This value is the longest wait between two ticks. The default is `1m`, and the floor is `1s`. */\n readonly tick?: Duration | undefined;\n /** This value is the most random time that the daemon adds to each wait. The default is `10s`. */\n readonly jitter?: Duration | undefined;\n /** This value is the budget of the checks of one tick. The default is `10m`. */\n readonly budget?: Duration | undefined;\n /** This value is the most checks of one tick. The default is 100. */\n readonly maxPages?: number | undefined;\n /** At a stop, the checks in progress get this time to end. The default is `10s`. */\n readonly grace?: Duration | undefined;\n /**\n * The daemon writes each change event as one JSON line here: a file, a function, or the standard\n * output (the default). The value false writes no line. A change is an event with `confirmed: true`.\n */\n readonly events?: string | ((line: string) => void) | false | undefined;\n /** This function gets each change event. The daemon starts it in order, one at a time, and logs its error. */\n readonly onChange?: ((event: WatchEvent) => void | Promise<void>) | undefined;\n /**\n * These options serve the MCP tools over HTTP. Without them, the daemon serves no MCP.\n *\n * @beta\n */\n readonly mcp?: DaemonMcpOptions | undefined;\n /**\n * This host serves the MCP tools. The default serves them over Streamable HTTP.\n *\n * @beta\n */\n readonly mcpHost?: DaemonMcpHost | undefined;\n /** The value false installs no handler of `SIGINT` and `SIGTERM`. The default is true. */\n readonly signals?: boolean | undefined;\n /** This file keeps the state of the daemon. The default is `<store>/daemon/state.json`. */\n readonly stateFile?: string | undefined;\n /** This function writes the messages of the daemon. The default writes them to the standard error. */\n readonly log?: ((line: string) => void) | undefined;\n}\n\n/** This is a daemon that runs. */\nexport interface Daemon {\n /** This crawler is the crawler of the daemon. The daemon closes it when it stops. */\n readonly crawler: OpenedCrawler;\n /** This file keeps the state of the daemon. */\n readonly stateFile: string;\n /**\n * This value is the MCP server, or undefined.\n *\n * @beta\n */\n readonly mcp: DaemonMcpService | undefined;\n /** This method gives the status at this time. */\n status(): DaemonStatus;\n /**\n * This method stops the daemon. The checks in progress get the grace time to end, then they\n * abort and the run releases their leases. The MCP server stops and cancels its tasks. A second\n * call with `0` aborts the checks at once.\n */\n stop(grace?: Duration): Promise<DaemonStatus>;\n /** This promise resolves with the last status when the daemon stopped. */\n readonly done: Promise<DaemonStatus>;\n}\n\n/**\n * The default MCP host is `mcpHttpHost` of the package polite-crawler-mcp. The daemon loads the\n * package only when it serves MCP, and an error tells where the MCP server is when it is not there.\n */\nconst defaultHost: DaemonMcpHost = {\n async start(crawler, options, hooks) {\n const { loadMcpPackage } = await import(\"./mcp-package.ts\");\n const { mcpHttpHost } = await loadMcpPackage();\n return mcpHttpHost.start(crawler, options, hooks);\n },\n};\n\n/** This function gives the writer of the change events. */\nconst eventWriter = (events: DaemonOptions[\"events\"]): ((line: string) => void) | undefined => {\n if (events === false) return undefined;\n if (typeof events === \"function\") return events;\n if (events === undefined) return (line) => void process.stdout.write(`${line}\\n`);\n mkdirSync(dirname(events), { recursive: true });\n return (line) => appendFileSync(events, `${line}\\n`, { mode: 0o600 });\n};\n\n/**\n * This function starts a daemon on a store. It opens the crawler, serves MCP when `mcp` is given,\n * and checks the watch list on a schedule (`scheduleWatches`). It writes each change event as one\n * JSON line, and it gives each change to `onChange`. The state file tells `daemon status` what runs.\n *\n * Only one daemon operates on a store: a second start rejects. At `SIGINT` or `SIGTERM`, the daemon\n * stops cleanly (`stop`). A second signal aborts the checks at once.\n */\nexport const startDaemon = async (options: DaemonOptions): Promise<Daemon> => {\n const {\n tick,\n jitter,\n budget,\n maxPages,\n grace,\n events,\n onChange,\n mcp,\n mcpHost,\n signals = true,\n stateFile: givenState,\n log = (line: string) => void process.stderr.write(`${line}\\n`),\n ...open\n } = options;\n const tickMs = msOf(\"tick\", tick ?? DAEMON_DEFAULTS.tickMs, DAEMON_DEFAULTS.minTickMs);\n const jitterMs = msOf(\"jitter\", jitter ?? DAEMON_DEFAULTS.jitterMs, 0);\n const budgetMs = msOf(\"budget\", budget ?? DAEMON_DEFAULTS.budgetMs, 0);\n const pages = pagesOf(maxPages);\n const graceMs = msOf(\"grace\", grace ?? DAEMON_DEFAULTS.graceMs, 0);\n const stateFile = givenState ?? daemonStateFile(open.dir);\n const before = readDaemonState(open.dir, stateFile);\n if (before?.alive === true) {\n throw new Error(\n `a daemon runs on the store ${open.dir} already (process ${String(before.pid)}): stop it first, or remove ${stateFile} when no daemon runs`,\n );\n }\n const writeEvent = eventWriter(events);\n // a missing browser turns rendering off one time, at start\n const renderOff = open.renderer === undefined ? await browserMissing() : undefined;\n const crawler = openCrawler({\n ...open,\n ...(renderOff === undefined ? {} : { renderer: false as const }),\n });\n const clock = open.clock ?? realClock;\n const startedAt = iso(clock.now());\n let lastError: DaemonStatus[\"last_error\"] = null;\n let service: DaemonMcpService | undefined;\n let tokenFile: string | null = null;\n let schedule: WatchSchedule | undefined;\n let nextDue: readonly DaemonDue[] = [];\n let stoppedAt: string | null = null;\n\n const status = (): DaemonStatus => {\n const counts = schedule?.result() ?? { ticks: 0, checked: 0, changes: 0, last: null };\n return {\n schema: DAEMON_STATE_SCHEMA,\n version: VERSION,\n pid: process.pid,\n store: open.dir,\n running: stoppedAt === null,\n started_at: startedAt,\n updated_at: iso(clock.now()),\n stopped_at: stoppedAt,\n tick_ms: tickMs,\n jitter_ms: jitterMs,\n budget_ms: budgetMs,\n max_pages: pages,\n ticks: counts.ticks,\n checked: counts.checked,\n changes: counts.changes,\n last_tick: counts.last,\n next_tick_at: stoppedAt === null ? (counts.last?.next_at ?? startedAt) : null,\n next_due: nextDue,\n tasks: service?.tasks() ?? [],\n mcp: service === undefined ? null : { url: service.url, token_file: tokenFile },\n last_error: lastError,\n };\n };\n const save = (): void => {\n try {\n writeState(stateFile, status());\n } catch (error) {\n log(`polite-crawler: the state file ${stateFile} does not write: ${errorOf(error).message}`);\n }\n };\n const fail = (error: unknown, where: string): void => {\n const message = `${where}: ${errorOf(error).message}`;\n lastError = { at: iso(clock.now()), message };\n log(`polite-crawler: ${message}`);\n };\n\n if (mcp !== undefined) {\n let given: DaemonMcpOptions = { ...mcp, renderOff: mcp.renderOff ?? renderOff };\n try {\n if (mcp.token === undefined && mcp.tokenFile === undefined) {\n // a new token at each start: a token that leaked ends at the next start\n tokenFile = join(dirname(stateFile), \"mcp-token\");\n given = { ...given, token: writeToken(tokenFile) };\n } else tokenFile = mcp.tokenFile ?? null;\n service = await (mcpHost ?? defaultHost).start(crawler, given, {\n status,\n onTasks: save,\n log,\n });\n } catch (error) {\n await crawler.close();\n throw error;\n }\n }\n\n // the hooks run in order, one at a time, so a slow hook does not start many processes\n let hooks: Promise<void> = Promise.resolve();\n const changed = (event: WatchEvent): void => {\n if (writeEvent !== undefined) {\n try {\n writeEvent(JSON.stringify(event));\n } catch (error) {\n fail(error, \"the event log\");\n }\n }\n if (onChange !== undefined) {\n hooks = hooks\n .then(() => onChange(event))\n .catch((error: unknown) => fail(error, \"the on-change hook\"));\n }\n };\n\n RUNNING.add(stateFile);\n const running = scheduleWatches(crawler.watch, {\n tick: tickMs,\n jitter: jitterMs,\n budget: budgetMs,\n maxPages: pages,\n clock,\n random: open.random,\n onEvent: (event) => {\n if (event.confirmed) changed(event);\n },\n onTick: (done) => {\n nextDue = done.due;\n // a run that failed went to onError\n if (done.summary !== null && done.summary.checked > 0) {\n const summary = done.summary;\n log(\n `polite-crawler: ${String(summary.checked)} checked: ${String(summary.confirmed)} changes, ${String(summary.suspected)} suspected, ${String(summary.refused)} refused, ${String(summary.error)} errors. Next tick at ${done.next_at ?? \"the stop\"}.`,\n );\n }\n save();\n },\n onError: (error) => fail(error, \"a tick\"),\n });\n schedule = running;\n save();\n log(\n `polite-crawler ${VERSION}: the daemon runs on the store ${open.dir}, a tick each ${String(tickMs)} ms${service === undefined ? \"\" : `, MCP at ${service.url}${tokenFile === null ? \"\" : ` (the token is in ${tokenFile})`}`}`,\n );\n if (renderOff !== undefined) log(`polite-crawler: rendering is off: ${renderOff}`);\n\n let resolveDone: (final: DaemonStatus) => void = () => undefined;\n const done = new Promise<DaemonStatus>((resolve) => {\n resolveDone = resolve;\n });\n let stopping: Promise<DaemonStatus> | undefined;\n let signalled = 0;\n const onSignal = (signal: NodeJS.Signals): void => {\n signalled += 1;\n log(\n signalled > 1\n ? `polite-crawler: ${signal} again: the checks in progress abort now`\n : `polite-crawler: ${signal}: the daemon stops, and the checks in progress get ${String(graceMs)} ms`,\n );\n void stop(signalled > 1 ? 0 : graceMs);\n };\n const stop = (stopGrace: Duration = graceMs): Promise<DaemonStatus> => {\n const ms = msOf(\"grace\", stopGrace, 0);\n void running.stop(ms);\n stopping ??= (async () => {\n const closing = service?.close().catch((error: unknown) => fail(error, \"the MCP server\"));\n await running.done;\n // the hooks in the queue get the grace too\n let timer: ReturnType<typeof setTimeout> | undefined;\n await Promise.race([\n hooks,\n new Promise<void>((resolve) => {\n timer = setTimeout(resolve, ms);\n }),\n ]);\n clearTimeout(timer);\n await closing;\n stoppedAt = iso(clock.now());\n save();\n RUNNING.delete(stateFile);\n await crawler.close();\n if (signals) {\n process.removeListener(\"SIGINT\", onSignal);\n process.removeListener(\"SIGTERM\", onSignal);\n }\n log(\"polite-crawler: the daemon stopped\");\n const final = status();\n resolveDone(final);\n return final;\n })();\n return stopping;\n };\n if (signals) {\n process.on(\"SIGINT\", onSignal);\n process.on(\"SIGTERM\", onSignal);\n }\n return { crawler, stateFile, mcp: service, status, stop, done };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAiCA,MAAa,kBAAkB;;CAE7B,QAAQ;;CAER,UAAU;;CAEV,UAAU;;CAEV,UAAU;;CAEV,SAAS;;CAET,UAAU;;CAEV,WAAW;;CAEX,UAAU;;CAEV,eAAe;AACjB;;AAGA,MAAa,sBAAsB;AA0EnC,MAAM,OAAO,OAAuB,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;;AAG7D,MAAM,QAAQ,MAAc,OAAiB,UAA0B;CACrE,IAAI;CACJ,IAAI;EACF,KAAK,WAAW,KAAK;CACvB,SAAS,OAAO;EACd,MAAM,IAAI,WAAW,GAAG,KAAK,IAAK,MAAgB,WAAW,EAAE,OAAO,MAAM,CAAC;CAC/E;CACA,IAAI,KAAK,OACP,MAAM,IAAI,WAAW,GAAG,KAAK,WAAW,OAAO,KAAK,EAAE,mBAAmB,OAAO,EAAE,GAAG;CACvF,OAAO;AACT;AAEA,MAAM,WAAW,UAAsC;CACrD,MAAM,QAAQ,SAAS,gBAAgB;CACvC,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GACtC,MAAM,IAAI,WAAW,iDAAiD,OAAO,KAAK,GAAG;CACvF,OAAO;AACT;AAEA,MAAM,WAAW,UACf,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;;AAG1D,MAAM,SAAS,YACb,CAAC,GAAG,OAAO,CAAC,CACT,MAAM,GAAG,MAAM,KAAK,MAAM,EAAE,OAAO,IAAI,KAAK,MAAM,EAAE,OAAO,KAAK,EAAE,IAAI,cAAc,EAAE,GAAG,CAAC,CAAC,CAC3F,MAAM,GAAG,gBAAgB,QAAQ,CAAC,CAClC,KAAK,WAAW;CACf,KAAK,MAAM;CACX,SAAS,MAAM;CACf,UAAU,MAAM;CAChB,WAAW,MAAM;AACnB,EAAE;;;;;;;;;AAUN,MAAa,cACX,SACA,SACA,KACA,YACW;CACX,IAAI,OAAO,MAAM,QAAQ;CACzB,MAAM,WAAW,YAAY,QAAQ,QAAQ,YAAY;CACzD,KAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,QAAQ,MAAM,iBAAiB,OAAO,IAAI,KAAK,MAAM,MAAM,YAAY;EAC7E,MAAM,MAAM,KAAK,IAAI,KAAK,MAAM,MAAM,OAAO,GAAG,KAAK;EACrD,IAAI,OAAO,OAAO,UAAU;EAC5B,OAAO,KAAK,IAAI,MAAM,KAAK,IAAI,KAAK,MAAM,gBAAgB,QAAQ,CAAC;CACrE;CACA,MAAM,SAAS,KAAK,IAAI,KAAK,IAAI,QAAQ,OAAO,GAAG,CAAC,GAAG,CAAC;CACxD,OAAO,OAAO,KAAK,IAAI,QAAQ,UAAU,KAAK,MAAM,SAAS,QAAQ,QAAQ,CAAC;AAChF;;;;;;;;;AAUA,MAAa,mBAAmB,OAAkB,UAA2B,CAAC,MAAqB;CACjG,MAAM,SAAS,KAAK,QAAQ,QAAQ,QAAQ,gBAAgB,QAAQ,gBAAgB,SAAS;CAC7F,MAAM,WAAW,KAAK,UAAU,QAAQ,UAAU,gBAAgB,UAAU,CAAC;CAC7E,MAAM,WAAW,KAAK,UAAU,QAAQ,UAAU,gBAAgB,UAAU,CAAC;CAC7E,MAAM,WAAW,QAAQ,QAAQ,QAAQ;CACzC,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,SAAS,QAAQ,UAAU,KAAK;CAEtC,MAAM,QAAQ,IAAI,gBAAgB;CAClC,MAAM,SAAS,IAAI,gBAAgB;CACnC,MAAM,OAAO,IAAI,gBAAgB;CACjC,IAAI,WAAW;CACf,MAAM,SAAS;EAAE,OAAO;EAAG,SAAS;EAAG,SAAS;EAAG,MAAM;CAA0B;CACnF,MAAM,gBAAgC,EAAE,GAAG,OAAO;CAClD,MAAM,UAAU,UAAyB;EACvC,IAAI;GACF,QAAQ,UAAU,QAAQ,KAAK,CAAC;EAClC,QAAQ,CAER;CACF;CACA,MAAM,UAAU,SAA2B;EACzC,IAAI;GACF,KAAK;EACP,SAAS,OAAO;GACd,OAAO,KAAK;EACd;CACF;CAEA,MAAM,OAAO,YAA2B;EACtC,OAAO,CAAC,UAAU;GAChB,MAAM,UAAU,MAAM,IAAI;GAC1B,IAAI,UAA+B;GACnC,IAAI,UAAyB;GAC7B,IAAI;IACF,MAAM,MAAM,MAAM,IAAI;KACpB;KACA;KACA,QAAQ,OAAO;KACf,OAAO,MAAM;IACf,CAAC;IACD,WAAW,MAAM,SAAS,KAAK;KAC7B,OAAO,WAAW;KAClB,IAAI,MAAM,WAAW,OAAO,WAAW;KACvC,aAAa,QAAQ,UAAU,KAAK,CAAC;IACvC;IACA,UAAU,MAAM,IAAI;GACtB,SAAS,OAAO;IACd,UAAU,QAAQ,KAAK,CAAC,CAAC;IACzB,OAAO,KAAK;GACd;GACA,OAAO,SAAS;GAEhB,IAAI,UAAwB,CAAC;GAC7B,IAAI;IACF,UAAU,MAAM,MAAM,KAAK;GAC7B,SAAS,OAAO;IACd,OAAO,KAAK;GACd;GACA,MAAM,OAAO,WACT,OACA,WAAW,SAAS,SAAS,MAAM,IAAI,GAAG;IAAE;IAAQ;IAAU;GAAO,CAAC;GAC1E,MAAM,OAAmB;IACvB,YAAY,IAAI,OAAO;IACvB,UAAU,IAAI,MAAM,IAAI,CAAC;IACzB;IACA,OAAO;IACP,SAAS,SAAS,OAAO,OAAO,IAAI,IAAI;IACxC,KAAK,MAAM,OAAO;GACpB;GACA,OAAO,OAAO;GACd,aAAa,QAAQ,SAAS,IAAI,CAAC;GACnC,IAAI,SAAS,QAAQ,UAAU;GAC/B,MAAM,MAAM,MAAM,KAAK,IAAI,GAAG,OAAO,MAAM,IAAI,CAAC,GAAG,KAAK,MAAM;EAChE;CACF;CAEA,MAAM,OAAO,KAAK,CAAC,CAAC,KAAK,SAAS,UAAmB;EACnD,OAAO,KAAK;EACZ,OAAO,OAAO;CAChB,CAAC;CACD,OAAO;EACL;EACA;EACA,KAAK,QAAQ,gBAAgB,SAAS;GACpC,MAAM,UAAU,KAAK,SAAS,OAAO,CAAC;GACtC,WAAW;GACX,MAAM,sBAAM,IAAI,MAAM,oBAAoB,CAAC;GAC3C,KAAK,sBAAM,IAAI,MAAM,oBAAoB,CAAC;GAC1C,IAAI,YAAY,GAAG,OAAO,sBAAM,IAAI,MAAM,8BAA8B,CAAC;QACpE;IAEH,MAAM,QAAQ,iBACN,OAAO,sBAAM,IAAI,MAAM,kCAAkC,CAAC,GAChE,OACF;IACA,KAAU,cAAc,aAAa,KAAK,CAAC;GAC7C;GACA,OAAO;EACT;CACF;AACF;;AA6DA,MAAM,0BAAU,IAAI,IAAY;;AAGhC,MAAM,gBAAgB,QAAyB;CAC7C,IAAI,CAAC,OAAO,UAAU,GAAG,KAAK,OAAO,GAAG,OAAO;CAC/C,IAAI;EACF,QAAQ,KAAK,KAAK,CAAC;EACnB,OAAO;CACT,SAAS,OAAO;EACd,OAAQ,MAA6B,SAAS;CAChD;AACF;;AAGA,MAAa,mBAAmB,QAAwB,KAAK,KAAK,UAAU,YAAY;;;;;AAMxF,MAAa,mBACX,KACA,OAAe,gBAAgB,GAAG,MACN;CAC5B,IAAI;CACJ,IAAI;EACF,QAAQ,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;CAC/C,QAAQ;EACN;CACF;CACA,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAO,MAAM,QAAQ,UACtE,OAAO,KAAA;CACT,MAAM,QACJ,MAAM,YAAY,SACjB,MAAM,QAAQ,QAAQ,MAAM,QAAQ,IAAI,IAAI,IAAI,aAAa,MAAM,GAAG;CACzE,OAAO;EAAE,GAAG;EAAO;CAAM;AAC3B;;AAGA,MAAM,cAAc,MAAc,UAA8B;CAC9D,UAAU,QAAQ,IAAI,GAAG;EAAE,WAAW;EAAM,MAAM;CAAM,CAAC;CACzD,MAAM,OAAO,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,EAAE;CAC/C,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,QAAQ,GAAG,EAAE;CAC5C,cAAc,MAAM,MAAM,EAAE,MAAM,IAAM,CAAC;CACzC,IAAI;EACF,WAAW,MAAM,IAAI;CACvB,QAAQ;EAEN,cAAc,MAAM,MAAM,EAAE,MAAM,IAAM,CAAC;EACzC,IAAI;GACF,WAAW,IAAI;EACjB,QAAQ,CAER;CACF;AACF;;AAGA,MAAM,cAAc,SAAyB;CAC3C,MAAM,QAAQ,YAAY,EAAE,CAAC,CAAC,SAAS,WAAW;CAClD,UAAU,QAAQ,IAAI,GAAG;EAAE,WAAW;EAAM,MAAM;CAAM,CAAC;CACzD,IAAI;EAEF,WAAW,IAAI;CACjB,QAAQ,CAER;CACA,cAAc,MAAM,GAAG,MAAM,KAAK;EAAE,MAAM;EAAO,MAAM;CAAK,CAAC;CAC7D,OAAO;AACT;;;;;;;;;;AAqBA,MAAa,iBACX,SACA,UAAgC,CAAC,MACU;CAC3C,IAAI,QAAQ,KAAK,MAAM,IAAI,MAAM,IAAI,WAAW,gCAAgC;CAChF,MAAM,YAAY,KAAK,WAAW,QAAQ,WAAW,gBAAgB,eAAe,CAAC;CACrF,QAAQ,UACN,IAAI,SAAe,SAAS,WAAW;EACrC,MAAM,QAAQ,MAAM,SAAS;GAC3B,OAAO;GACP,OAAO;IAAC;IAAQ;IAAU;GAAM;GAChC,aAAa;GACb,GAAI,QAAQ,QAAQ,KAAA,IAAY,CAAC,IAAI,EAAE,KAAK,QAAQ,IAAI;EAC1D,CAAC;EACD,IAAI,OAAO;EACX,IAAI,OAAO;EACX,MAAM,QAAQ,iBAAiB;GAC7B,OAAO;GACP,MAAM,KAAK;EACb,GAAG,SAAS;EACZ,MAAM,OAAO,YAAY,MAAM;EAC/B,MAAM,OAAO,GAAG,SAAS,SAAiB;GACxC,QAAQ,OAAO,KAAA,CAAM,MAAM,IAAK;EAClC,CAAC;EAED,MAAM,MAAM,GAAG,eAAe,KAAA,CAAS;EACvC,MAAM,GAAG,UAAU,UAAU;GAC3B,aAAa,KAAK;GAClB,OACE,IAAI,MAAM,wCAAwC,MAAM,WAAW,EAAE,OAAO,MAAM,CAAC,CACrF;EACF,CAAC;EACD,MAAM,GAAG,UAAU,MAAM,WAAW;GAClC,aAAa,KAAK;GAClB,MAAM,OAAO,KAAK,KAAK,MAAM,KAAK,KAAK,KAAK,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;GAClE,IAAI,MACF,uBACE,IAAI,MACF,2CAA2C,OAAO,SAAS,EAAE,8BAC/D,CACF;QACK,IAAI,SAAS,GAAG,QAAQ;QAE7B,uBACE,IAAI,MACF,qCAAqC,SAAS,OAAO,OAAO,MAAM,IAAI,QAAQ,OAAO,IAAI,MAAM,MACjG,CACF;EACJ,CAAC;EACD,MAAM,MAAM,IAAI,GAAG,KAAK,UAAU,KAAK,EAAE,GAAG;CAC9C,CAAC;AACL;;;;;AA0IA,MAAM,cAA6B,EACjC,MAAM,MAAM,SAAS,SAAS,OAAO;CACnC,MAAM,EAAE,mBAAmB,MAAM,OAAO;CACxC,MAAM,EAAE,gBAAgB,MAAM,eAAe;CAC7C,OAAO,YAAY,MAAM,SAAS,SAAS,KAAK;AAClD,EACF;;AAGA,MAAM,eAAe,WAA0E;CAC7F,IAAI,WAAW,OAAO,OAAO,KAAA;CAC7B,IAAI,OAAO,WAAW,YAAY,OAAO;CACzC,IAAI,WAAW,KAAA,GAAW,QAAQ,SAAS,KAAK,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;CAChF,UAAU,QAAQ,MAAM,GAAG,EAAE,WAAW,KAAK,CAAC;CAC9C,QAAQ,SAAS,eAAe,QAAQ,GAAG,KAAK,KAAK,EAAE,MAAM,IAAM,CAAC;AACtE;;;;;;;;;AAUA,MAAa,cAAc,OAAO,YAA4C;CAC5E,MAAM,EACJ,MACA,QACA,QACA,UACA,OACA,QACA,UACA,KACA,SACA,UAAU,MACV,WAAW,YACX,OAAO,SAAiB,KAAK,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG,GAC7D,GAAG,SACD;CACJ,MAAM,SAAS,KAAK,QAAQ,QAAQ,gBAAgB,QAAQ,gBAAgB,SAAS;CACrF,MAAM,WAAW,KAAK,UAAU,UAAU,gBAAgB,UAAU,CAAC;CACrE,MAAM,WAAW,KAAK,UAAU,UAAU,gBAAgB,UAAU,CAAC;CACrE,MAAM,QAAQ,QAAQ,QAAQ;CAC9B,MAAM,UAAU,KAAK,SAAS,SAAS,gBAAgB,SAAS,CAAC;CACjE,MAAM,YAAY,cAAc,gBAAgB,KAAK,GAAG;CACxD,MAAM,SAAS,gBAAgB,KAAK,KAAK,SAAS;CAClD,IAAI,QAAQ,UAAU,MACpB,MAAM,IAAI,MACR,8BAA8B,KAAK,IAAI,oBAAoB,OAAO,OAAO,GAAG,EAAE,8BAA8B,UAAU,qBACxH;CAEF,MAAM,aAAa,YAAY,MAAM;CAErC,MAAM,YAAY,KAAK,aAAa,KAAA,IAAY,MAAM,eAAe,IAAI,KAAA;CACzE,MAAM,UAAU,YAAY;EAC1B,GAAG;EACH,GAAI,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,MAAe;CAChE,CAAC;CACD,MAAM,QAAQ,KAAK,SAAS;CAC5B,MAAM,YAAY,IAAI,MAAM,IAAI,CAAC;CACjC,IAAI,YAAwC;CAC5C,IAAI;CACJ,IAAI,YAA2B;CAC/B,IAAI;CACJ,IAAI,UAAgC,CAAC;CACrC,IAAI,YAA2B;CAE/B,MAAM,eAA6B;EACjC,MAAM,SAAS,UAAU,OAAO,KAAK;GAAE,OAAO;GAAG,SAAS;GAAG,SAAS;GAAG,MAAM;EAAK;EACpF,OAAO;GACL,QAAA;GACA,SAAS;GACT,KAAK,QAAQ;GACb,OAAO,KAAK;GACZ,SAAS,cAAc;GACvB,YAAY;GACZ,YAAY,IAAI,MAAM,IAAI,CAAC;GAC3B,YAAY;GACZ,SAAS;GACT,WAAW;GACX,WAAW;GACX,WAAW;GACX,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,SAAS,OAAO;GAChB,WAAW,OAAO;GAClB,cAAc,cAAc,OAAQ,OAAO,MAAM,WAAW,YAAa;GACzE,UAAU;GACV,OAAO,SAAS,MAAM,KAAK,CAAC;GAC5B,KAAK,YAAY,KAAA,IAAY,OAAO;IAAE,KAAK,QAAQ;IAAK,YAAY;GAAU;GAC9E,YAAY;EACd;CACF;CACA,MAAM,aAAmB;EACvB,IAAI;GACF,WAAW,WAAW,OAAO,CAAC;EAChC,SAAS,OAAO;GACd,IAAI,kCAAkC,UAAU,mBAAmB,QAAQ,KAAK,CAAC,CAAC,SAAS;EAC7F;CACF;CACA,MAAM,QAAQ,OAAgB,UAAwB;EACpD,MAAM,UAAU,GAAG,MAAM,IAAI,QAAQ,KAAK,CAAC,CAAC;EAC5C,YAAY;GAAE,IAAI,IAAI,MAAM,IAAI,CAAC;GAAG;EAAQ;EAC5C,IAAI,mBAAmB,SAAS;CAClC;CAEA,IAAI,QAAQ,KAAA,GAAW;EACrB,IAAI,QAA0B;GAAE,GAAG;GAAK,WAAW,IAAI,aAAa;EAAU;EAC9E,IAAI;GACF,IAAI,IAAI,UAAU,KAAA,KAAa,IAAI,cAAc,KAAA,GAAW;IAE1D,YAAY,KAAK,QAAQ,SAAS,GAAG,WAAW;IAChD,QAAQ;KAAE,GAAG;KAAO,OAAO,WAAW,SAAS;IAAE;GACnD,OAAO,YAAY,IAAI,aAAa;GACpC,UAAU,OAAO,WAAW,YAAA,CAAa,MAAM,SAAS,OAAO;IAC7D;IACA,SAAS;IACT;GACF,CAAC;EACH,SAAS,OAAO;GACd,MAAM,QAAQ,MAAM;GACpB,MAAM;EACR;CACF;CAGA,IAAI,QAAuB,QAAQ,QAAQ;CAC3C,MAAM,WAAW,UAA4B;EAC3C,IAAI,eAAe,KAAA,GACjB,IAAI;GACF,WAAW,KAAK,UAAU,KAAK,CAAC;EAClC,SAAS,OAAO;GACd,KAAK,OAAO,eAAe;EAC7B;EAEF,IAAI,aAAa,KAAA,GACf,QAAQ,MACL,WAAW,SAAS,KAAK,CAAC,CAAC,CAC3B,OAAO,UAAmB,KAAK,OAAO,oBAAoB,CAAC;CAElE;CAEA,QAAQ,IAAI,SAAS;CACrB,MAAM,UAAU,gBAAgB,QAAQ,OAAO;EAC7C,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,UAAU;EACV;EACA,QAAQ,KAAK;EACb,UAAU,UAAU;GAClB,IAAI,MAAM,WAAW,QAAQ,KAAK;EACpC;EACA,SAAS,SAAS;GAChB,UAAU,KAAK;GAEf,IAAI,KAAK,YAAY,QAAQ,KAAK,QAAQ,UAAU,GAAG;IACrD,MAAM,UAAU,KAAK;IACrB,IACE,mBAAmB,OAAO,QAAQ,OAAO,EAAE,YAAY,OAAO,QAAQ,SAAS,EAAE,YAAY,OAAO,QAAQ,SAAS,EAAE,cAAc,OAAO,QAAQ,OAAO,EAAE,YAAY,OAAO,QAAQ,KAAK,EAAE,wBAAwB,KAAK,WAAW,WAAW,EACpP;GACF;GACA,KAAK;EACP;EACA,UAAU,UAAU,KAAK,OAAO,QAAQ;CAC1C,CAAC;CACD,WAAW;CACX,KAAK;CACL,IACE,kBAAkB,QAAQ,iCAAiC,KAAK,IAAI,gBAAgB,OAAO,MAAM,EAAE,KAAK,YAAY,KAAA,IAAY,KAAK,YAAY,QAAQ,MAAM,cAAc,OAAO,KAAK,qBAAqB,UAAU,MAC1N;CACA,IAAI,cAAc,KAAA,GAAW,IAAI,qCAAqC,WAAW;CAEjF,IAAI,oBAAmD,KAAA;CACvD,MAAM,OAAO,IAAI,SAAuB,YAAY;EAClD,cAAc;CAChB,CAAC;CACD,IAAI;CACJ,IAAI,YAAY;CAChB,MAAM,YAAY,WAAiC;EACjD,aAAa;EACb,IACE,YAAY,IACR,mBAAmB,OAAO,4CAC1B,mBAAmB,OAAO,qDAAqD,OAAO,OAAO,EAAE,IACrG;EACA,KAAU,YAAY,IAAI,IAAI,OAAO;CACvC;CACA,MAAM,QAAQ,YAAsB,YAAmC;EACrE,MAAM,KAAK,KAAK,SAAS,WAAW,CAAC;EACrC,QAAa,KAAK,EAAE;EACpB,cAAc,YAAY;GACxB,MAAM,UAAU,SAAS,MAAM,CAAC,CAAC,OAAO,UAAmB,KAAK,OAAO,gBAAgB,CAAC;GACxF,MAAM,QAAQ;GAEd,IAAI;GACJ,MAAM,QAAQ,KAAK,CACjB,OACA,IAAI,SAAe,YAAY;IAC7B,QAAQ,WAAW,SAAS,EAAE;GAChC,CAAC,CACH,CAAC;GACD,aAAa,KAAK;GAClB,MAAM;GACN,YAAY,IAAI,MAAM,IAAI,CAAC;GAC3B,KAAK;GACL,QAAQ,OAAO,SAAS;GACxB,MAAM,QAAQ,MAAM;GACpB,IAAI,SAAS;IACX,QAAQ,eAAe,UAAU,QAAQ;IACzC,QAAQ,eAAe,WAAW,QAAQ;GAC5C;GACA,IAAI,oCAAoC;GACxC,MAAM,QAAQ,OAAO;GACrB,YAAY,KAAK;GACjB,OAAO;EACT,EAAA,CAAG;EACH,OAAO;CACT;CACA,IAAI,SAAS;EACX,QAAQ,GAAG,UAAU,QAAQ;EAC7B,QAAQ,GAAG,WAAW,QAAQ;CAChC;CACA,OAAO;EAAE;EAAS;EAAW,KAAK;EAAS;EAAQ;EAAM;CAAK;AAChE"}
@@ -1,5 +1,5 @@
1
1
  import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
2
- import { P as stableStringify, c as isHeldBack, l as isoTime, m as versionAtTime, n as HELD_BACK, s as heldBackMeta } from "./runtime-p5WN2UAA.mjs";
2
+ import { P as stableStringify, c as isHeldBack, l as isoTime, m as versionAtTime, n as HELD_BACK, s as heldBackMeta } from "./runtime-C2VRY89U.mjs";
3
3
  //#region src/diff.ts
4
4
  var diff_exports = /* @__PURE__ */ __exportAll({
5
5
  DIFF_LIMITS: () => DIFF_LIMITS,
@@ -649,4 +649,4 @@ const formatDiff = (diff) => {
649
649
  //#endregion
650
650
  export { formatDiff as a, diff_exports as i, diffDocument as n, diffTexts as r, DIFF_LIMITS as t };
651
651
 
652
- //# sourceMappingURL=diff-DLyQ2mkg.mjs.map
652
+ //# sourceMappingURL=diff-DuD9pinS.mjs.map