polite-crawler 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +555 -8
- package/build/cli-daemon-ZYRdy6EW.mjs +38 -0
- package/build/cli-daemon-ZYRdy6EW.mjs.map +1 -0
- package/build/cli-recipes-Bci6HREA.mjs +59 -0
- package/build/cli-recipes-Bci6HREA.mjs.map +1 -0
- package/build/cli.mjs +673 -26
- package/build/cli.mjs.map +1 -1
- package/build/daemon-DK6U51_v.mjs +515 -0
- package/build/daemon-DK6U51_v.mjs.map +1 -0
- package/build/{open-Ca1lcWR3.d.mts → daemon-wwIWsUmU.d.mts} +919 -17
- package/build/{diff-Dk2eSvi-.mjs → diff-BUM91Rev.mjs} +2 -2
- package/build/{diff-Dk2eSvi-.mjs.map → diff-BUM91Rev.mjs.map} +1 -1
- package/build/{extract-DcwNrTK7.mjs → extract-CefB5Vfn.mjs} +2 -2
- package/build/{extract-DcwNrTK7.mjs.map → extract-CefB5Vfn.mjs.map} +1 -1
- package/build/hours.d.mts +26 -0
- package/build/hours.mjs +48 -1
- package/build/hours.mjs.map +1 -1
- package/build/index.d.mts +491 -11
- package/build/index.mjs +14 -9
- package/build/loopback-C_QYowSf.mjs +22 -0
- package/build/loopback-C_QYowSf.mjs.map +1 -0
- package/build/mcp.d.mts +158 -2
- package/build/mcp.mjs +567 -18
- package/build/mcp.mjs.map +1 -1
- package/build/ollama-BJxkjny4.mjs +160 -0
- package/build/ollama-BJxkjny4.mjs.map +1 -0
- package/build/{open-DnrWgkC0.mjs → open-CbpSCEZc.mjs} +1127 -16
- package/build/open-CbpSCEZc.mjs.map +1 -0
- package/build/recipe-GiFQojmo.mjs +279 -0
- package/build/recipe-GiFQojmo.mjs.map +1 -0
- package/build/recipe-product-Dellqudu.mjs +892 -0
- package/build/recipe-product-Dellqudu.mjs.map +1 -0
- package/build/{runtime-k8mo1_7U.mjs → runtime-D_r2SYge.mjs} +3 -3
- package/build/{runtime-k8mo1_7U.mjs.map → runtime-D_r2SYge.mjs.map} +1 -1
- package/build/transformers-BQMlaGj4.mjs +133 -0
- package/build/transformers-BQMlaGj4.mjs.map +1 -0
- package/build/{duration-iIuMMofi.mjs → version-BPem-XZd.mjs} +10 -10
- package/build/version-BPem-XZd.mjs.map +1 -0
- package/build/{wacz-sfOLT8e9.mjs → wacz-QyzLp4mC.mjs} +3 -3
- package/build/{wacz-sfOLT8e9.mjs.map → wacz-QyzLp4mC.mjs.map} +1 -1
- package/build/{warc-VIPscWxY.mjs → warc-BHcaRKHw.mjs} +2 -2
- package/build/{warc-VIPscWxY.mjs.map → warc-BHcaRKHw.mjs.map} +1 -1
- package/build/webbotauth-CjJdTAfE.mjs.map +1 -1
- package/package.json +10 -2
- package/build/duration-iIuMMofi.mjs.map +0 -1
- package/build/open-DnrWgkC0.mjs.map +0 -1
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { n as describeDaemonStatus } from "./cli.mjs";
|
|
2
|
+
//#region src/cli-daemon.ts
|
|
3
|
+
/**
|
|
4
|
+
* This function does the command `daemon`, and gives the exit code. `daemon status` gives 0 when a
|
|
5
|
+
* daemon operates on the store, and 1 when none operates. `daemon` gives 0 after a clean stop. An
|
|
6
|
+
* error at start rejects, so the command exits with 1.
|
|
7
|
+
*/
|
|
8
|
+
const daemonCommand = async (line, options, io) => {
|
|
9
|
+
const { changeCommand, readDaemonState, startDaemon } = await import("./daemon-DK6U51_v.mjs").then((n) => n.a);
|
|
10
|
+
if (line.action === "status") {
|
|
11
|
+
const state = readDaemonState(options.dir);
|
|
12
|
+
io.out(io.json ? JSON.stringify(state ?? null, null, 2) : describeDaemonStatus(state, options.dir));
|
|
13
|
+
return state?.alive === true ? 0 : 1;
|
|
14
|
+
}
|
|
15
|
+
await (await startDaemon({
|
|
16
|
+
...options,
|
|
17
|
+
tick: line.tickMs,
|
|
18
|
+
jitter: line.jitterMs,
|
|
19
|
+
budget: line.budgetMs,
|
|
20
|
+
maxPages: line.maxPages,
|
|
21
|
+
grace: line.graceMs,
|
|
22
|
+
events: line.events,
|
|
23
|
+
onChange: line.onChange === void 0 ? void 0 : changeCommand(line.onChange),
|
|
24
|
+
mcp: line.mcp === void 0 ? void 0 : {
|
|
25
|
+
port: line.mcp.port,
|
|
26
|
+
host: line.mcp.host,
|
|
27
|
+
allowRemote: line.mcp.allowRemote,
|
|
28
|
+
tokenFile: line.mcp.tokenFile,
|
|
29
|
+
allowUserProvided: io.allowUserProvided
|
|
30
|
+
},
|
|
31
|
+
signals: true
|
|
32
|
+
})).done;
|
|
33
|
+
return 0;
|
|
34
|
+
};
|
|
35
|
+
//#endregion
|
|
36
|
+
export { daemonCommand };
|
|
37
|
+
|
|
38
|
+
//# sourceMappingURL=cli-daemon-ZYRdy6EW.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-daemon-ZYRdy6EW.mjs","names":[],"sources":["../src/cli-daemon.ts"],"sourcesContent":["/*\n * The command `daemon` of the command line: `daemon` runs the daemon until `SIGINT` or `SIGTERM`, and\n * `daemon status` prints the state file. `cli.ts` loads this file only for the command `daemon`.\n */\nimport type { DaemonLine } from \"./cli-line.ts\";\nimport { describeDaemonStatus } from \"./cli-line.ts\";\nimport type { OpenOptions } from \"./open.ts\";\n\n/** These are the parts of the command line that the command `daemon` uses. */\nexport interface DaemonIo {\n /** The value true lets `user_provided` work when the MCP client cannot ask the user. */\n readonly allowUserProvided: boolean;\n /** The value true prints the status as JSON. */\n readonly json: boolean;\n /** This function prints a line on the standard output. */\n readonly out: (text: string) => void;\n}\n\n/**\n * This function does the command `daemon`, and gives the exit code. `daemon status` gives 0 when a\n * daemon operates on the store, and 1 when none operates. `daemon` gives 0 after a clean stop. An\n * error at start rejects, so the command exits with 1.\n */\nexport const daemonCommand = async (\n line: DaemonLine,\n options: OpenOptions,\n io: DaemonIo,\n): Promise<number> => {\n const { changeCommand, readDaemonState, startDaemon } = await import(\"./daemon.ts\");\n if (line.action === \"status\") {\n const state = readDaemonState(options.dir);\n io.out(\n io.json ? JSON.stringify(state ?? null, null, 2) : describeDaemonStatus(state, options.dir),\n );\n return state?.alive === true ? 0 : 1;\n }\n const daemon = await startDaemon({\n ...options,\n tick: line.tickMs,\n jitter: line.jitterMs,\n budget: line.budgetMs,\n maxPages: line.maxPages,\n grace: line.graceMs,\n events: line.events,\n onChange: line.onChange === undefined ? undefined : changeCommand(line.onChange),\n mcp:\n line.mcp === undefined\n ? undefined\n : {\n port: line.mcp.port,\n host: line.mcp.host,\n allowRemote: line.mcp.allowRemote,\n tokenFile: line.mcp.tokenFile,\n allowUserProvided: io.allowUserProvided,\n },\n signals: true,\n });\n await daemon.done;\n return 0;\n};\n"],"mappings":";;;;;;;AAuBA,MAAa,gBAAgB,OAC3B,MACA,SACA,OACoB;CACpB,MAAM,EAAE,eAAe,iBAAiB,gBAAgB,MAAM,OAAO,wBAAc,CAAA,MAAA,MAAA,EAAA,CAAA;CACnF,IAAI,KAAK,WAAW,UAAU;EAC5B,MAAM,QAAQ,gBAAgB,QAAQ,GAAG;EACzC,GAAG,IACD,GAAG,OAAO,KAAK,UAAU,SAAS,MAAM,MAAM,CAAC,IAAI,qBAAqB,OAAO,QAAQ,GAAG,CAC5F;EACA,OAAO,OAAO,UAAU,OAAO,IAAI;CACrC;CAsBA,OAAM,MArBe,YAAY;EAC/B,GAAG;EACH,MAAM,KAAK;EACX,QAAQ,KAAK;EACb,QAAQ,KAAK;EACb,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,QAAQ,KAAK;EACb,UAAU,KAAK,aAAa,KAAA,IAAY,KAAA,IAAY,cAAc,KAAK,QAAQ;EAC/E,KACE,KAAK,QAAQ,KAAA,IACT,KAAA,IACA;GACE,MAAM,KAAK,IAAI;GACf,MAAM,KAAK,IAAI;GACf,aAAa,KAAK,IAAI;GACtB,WAAW,KAAK,IAAI;GACpB,mBAAmB,GAAG;EACxB;EACN,SAAS;CACX,CAAC,EAAA,CACY;CACb,OAAO;AACT"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { compareHours, fromCompact } from "./hours.mjs";
|
|
2
|
+
import { d as readVenue, o as readProduct } from "./recipe-product-Dellqudu.mjs";
|
|
3
|
+
import { a as keptWeekOf, i as describeVenue, o as productCode, r as describeProduct, s as venueCode, t as UsageError } from "./cli.mjs";
|
|
4
|
+
//#region src/cli-recipes.ts
|
|
5
|
+
/**
|
|
6
|
+
* This function checks the parts of `venue` that need a file or a reader, before the store opens.
|
|
7
|
+
* They are the kept week of `--compare` and the value of `--osm`. An error is a usage error.
|
|
8
|
+
*/
|
|
9
|
+
const prepareRecipe = (line) => {
|
|
10
|
+
if (line.osm !== void 0 && fromCompact(line.osm, "osm") === void 0) throw new UsageError(`--osm: the hours reader cannot read "${line.osm}". Write an opening_hours value, such as "Mo-Fr 08:00-18:00; Sa 09:00-13:00"`);
|
|
11
|
+
return { kept: line.compare === void 0 ? void 0 : keptWeekOf(line.compare) };
|
|
12
|
+
};
|
|
13
|
+
/** This function gives the options of the recipe that `venue` and `product` share. */
|
|
14
|
+
const sharedOptions = (line) => ({
|
|
15
|
+
render: line.render,
|
|
16
|
+
maxAge: line.maxAge,
|
|
17
|
+
budget: line.budget,
|
|
18
|
+
onBehalfOf: line.onBehalfOf,
|
|
19
|
+
userProvided: line.userProvided,
|
|
20
|
+
fallback: line.fallback,
|
|
21
|
+
archiveDespiteRobots: line.archiveDespiteRobots
|
|
22
|
+
});
|
|
23
|
+
/**
|
|
24
|
+
* This function does the command `venue`, and gives the exit code. It reads the hours of the venue
|
|
25
|
+
* with `readVenue`. With `--compare`, it compares the week of the site (`from_site`) with the kept
|
|
26
|
+
* week through `compareHours`.
|
|
27
|
+
*/
|
|
28
|
+
const venueCommand = async (target, line, input, io) => {
|
|
29
|
+
const options = {
|
|
30
|
+
...sharedOptions(line),
|
|
31
|
+
name: line.name,
|
|
32
|
+
requireName: line.requireName,
|
|
33
|
+
location: line.location,
|
|
34
|
+
osm: line.osm,
|
|
35
|
+
pages: line.pages
|
|
36
|
+
};
|
|
37
|
+
const result = await readVenue(target, line.url, options);
|
|
38
|
+
const site = result.from_site;
|
|
39
|
+
const compare = input.kept === void 0 ? void 0 : {
|
|
40
|
+
file: input.kept.file,
|
|
41
|
+
kept: input.kept.week,
|
|
42
|
+
comparison: site === null ? null : compareHours(input.kept.week, site.week)
|
|
43
|
+
};
|
|
44
|
+
io.out(io.json ? JSON.stringify(compare === void 0 ? result : {
|
|
45
|
+
...result,
|
|
46
|
+
compare
|
|
47
|
+
}, null, 2) : describeVenue(result, compare, io.renderOff).join("\n"));
|
|
48
|
+
return venueCode(result);
|
|
49
|
+
};
|
|
50
|
+
/** This function does the command `product`, and gives the exit code. It reads the page with `readProduct`. */
|
|
51
|
+
const productCommand = async (target, line, io) => {
|
|
52
|
+
const result = await readProduct(target, line.url, sharedOptions(line));
|
|
53
|
+
io.out(io.json ? JSON.stringify(result, null, 2) : describeProduct(result, io.renderOff).join("\n"));
|
|
54
|
+
return productCode(result);
|
|
55
|
+
};
|
|
56
|
+
//#endregion
|
|
57
|
+
export { prepareRecipe, productCommand, venueCommand };
|
|
58
|
+
|
|
59
|
+
//# sourceMappingURL=cli-recipes-Bci6HREA.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-recipes-Bci6HREA.mjs","names":[],"sources":["../src/cli-recipes.ts"],"sourcesContent":["/*\n * The commands `venue` and `product` of the command line. They read a venue or a product page with\n * the recipes of the package (`readVenue` and `readProduct`). `cli.ts` loads this file only for these\n * two commands, so the other commands do not load the recipes.\n */\nimport type { KeptWeek, RecipeLine, VenueCompare } from \"./cli-line.ts\";\nimport {\n UsageError,\n describeProduct,\n describeVenue,\n keptWeekOf,\n productCode,\n venueCode,\n} from \"./cli-line.ts\";\nimport { compareHours, fromCompact } from \"./hours.ts\";\nimport type { RecipeTarget } from \"./recipe.ts\";\nimport type { ProductOptions } from \"./recipe-product.ts\";\nimport { readProduct } from \"./recipe-product.ts\";\nimport type { VenueOptions } from \"./recipe-venue.ts\";\nimport { readVenue } from \"./recipe-venue.ts\";\n\n/** These are the parts of the command line that `venue` and `product` use. */\nexport interface RecipeIo {\n /** The value true prints the result as JSON. */\n readonly json: boolean;\n /** This function prints a line on the standard output. */\n readonly out: (text: string) => void;\n /** This text tells why rendering is off, when the computer has no browser. */\n readonly renderOff?: string | undefined;\n}\n\n/** These are the inputs of `venue` that the command reads before the store opens. */\nexport interface RecipeInput {\n /** This value is the kept week of `--compare`, or undefined with no `--compare`. */\n readonly kept: KeptWeek | undefined;\n}\n\n/**\n * This function checks the parts of `venue` that need a file or a reader, before the store opens.\n * They are the kept week of `--compare` and the value of `--osm`. An error is a usage error.\n */\nexport const prepareRecipe = (line: RecipeLine): RecipeInput => {\n if (line.osm !== undefined && fromCompact(line.osm, \"osm\") === undefined) {\n throw new UsageError(\n `--osm: the hours reader cannot read \"${line.osm}\". Write an opening_hours value, such as \"Mo-Fr 08:00-18:00; Sa 09:00-13:00\"`,\n );\n }\n return { kept: line.compare === undefined ? undefined : keptWeekOf(line.compare) };\n};\n\n/** This function gives the options of the recipe that `venue` and `product` share. */\nconst sharedOptions = (line: RecipeLine): ProductOptions => ({\n render: line.render,\n maxAge: line.maxAge,\n budget: line.budget,\n onBehalfOf: line.onBehalfOf,\n userProvided: line.userProvided,\n fallback: line.fallback,\n archiveDespiteRobots: line.archiveDespiteRobots,\n});\n\n/**\n * This function does the command `venue`, and gives the exit code. It reads the hours of the venue\n * with `readVenue`. With `--compare`, it compares the week of the site (`from_site`) with the kept\n * week through `compareHours`.\n */\nexport const venueCommand = async (\n target: RecipeTarget,\n line: RecipeLine,\n input: RecipeInput,\n io: RecipeIo,\n): Promise<number> => {\n const options: VenueOptions = {\n ...sharedOptions(line),\n name: line.name,\n requireName: line.requireName,\n location: line.location,\n osm: line.osm,\n pages: line.pages,\n };\n const result = await readVenue(target, line.url, options);\n const site = result.from_site;\n const compare: VenueCompare | undefined =\n input.kept === undefined\n ? undefined\n : {\n file: input.kept.file,\n kept: input.kept.week,\n comparison: site === null ? null : compareHours(input.kept.week, site.week),\n };\n io.out(\n io.json\n ? JSON.stringify(compare === undefined ? result : { ...result, compare }, null, 2)\n : describeVenue(result, compare, io.renderOff).join(\"\\n\"),\n );\n return venueCode(result);\n};\n\n/** This function does the command `product`, and gives the exit code. It reads the page with `readProduct`. */\nexport const productCommand = async (\n target: RecipeTarget,\n line: RecipeLine,\n io: RecipeIo,\n): Promise<number> => {\n const result = await readProduct(target, line.url, sharedOptions(line));\n io.out(\n io.json ? JSON.stringify(result, null, 2) : describeProduct(result, io.renderOff).join(\"\\n\"),\n );\n return productCode(result);\n};\n"],"mappings":";;;;;;;;AAyCA,MAAa,iBAAiB,SAAkC;CAC9D,IAAI,KAAK,QAAQ,KAAA,KAAa,YAAY,KAAK,KAAK,KAAK,MAAM,KAAA,GAC7D,MAAM,IAAI,WACR,wCAAwC,KAAK,IAAI,6EACnD;CAEF,OAAO,EAAE,MAAM,KAAK,YAAY,KAAA,IAAY,KAAA,IAAY,WAAW,KAAK,OAAO,EAAE;AACnF;;AAGA,MAAM,iBAAiB,UAAsC;CAC3D,QAAQ,KAAK;CACb,QAAQ,KAAK;CACb,QAAQ,KAAK;CACb,YAAY,KAAK;CACjB,cAAc,KAAK;CACnB,UAAU,KAAK;CACf,sBAAsB,KAAK;AAC7B;;;;;;AAOA,MAAa,eAAe,OAC1B,QACA,MACA,OACA,OACoB;CACpB,MAAM,UAAwB;EAC5B,GAAG,cAAc,IAAI;EACrB,MAAM,KAAK;EACX,aAAa,KAAK;EAClB,UAAU,KAAK;EACf,KAAK,KAAK;EACV,OAAO,KAAK;CACd;CACA,MAAM,SAAS,MAAM,UAAU,QAAQ,KAAK,KAAK,OAAO;CACxD,MAAM,OAAO,OAAO;CACpB,MAAM,UACJ,MAAM,SAAS,KAAA,IACX,KAAA,IACA;EACE,MAAM,MAAM,KAAK;EACjB,MAAM,MAAM,KAAK;EACjB,YAAY,SAAS,OAAO,OAAO,aAAa,MAAM,KAAK,MAAM,KAAK,IAAI;CAC5E;CACN,GAAG,IACD,GAAG,OACC,KAAK,UAAU,YAAY,KAAA,IAAY,SAAS;EAAE,GAAG;EAAQ;CAAQ,GAAG,MAAM,CAAC,IAC/E,cAAc,QAAQ,SAAS,GAAG,SAAS,CAAC,CAAC,KAAK,IAAI,CAC5D;CACA,OAAO,UAAU,MAAM;AACzB;;AAGA,MAAa,iBAAiB,OAC5B,QACA,MACA,OACoB;CACpB,MAAM,SAAS,MAAM,YAAY,QAAQ,KAAK,KAAK,cAAc,IAAI,CAAC;CACtE,GAAG,IACD,GAAG,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,gBAAgB,QAAQ,GAAG,SAAS,CAAC,CAAC,KAAK,IAAI,CAC7F;CACA,OAAO,YAAY,MAAM;AAC3B"}
|