polite-crawler 0.1.2 → 0.1.3

package/build/cli-daemon--cTmq0Z5.mjs

@@ -0,0 +1,38 @@
+import { t as describeDaemonStatus } from "./cli.mjs";
+//#region src/cli-daemon.ts
+/**
+* This function does the command `daemon`, and gives the exit code. `daemon status` gives 0 when a
+* daemon operates on the store, and 1 when none operates. `daemon` gives 0 after a clean stop. An
+* error at start rejects, so the command exits with 1.
+*/
+const daemonCommand = async (line, options, io) => {
+	const { changeCommand, readDaemonState, startDaemon } = await import("./daemon-fZTE24P5.mjs").then((n) => n.a);
+	if (line.action === "status") {
+		const state = readDaemonState(options.dir);
+		io.out(io.json ? JSON.stringify(state ?? null, null, 2) : describeDaemonStatus(state, options.dir));
+		return state?.alive === true ? 0 : 1;
+	}
+	await (await startDaemon({
+		...options,
+		tick: line.tickMs,
+		jitter: line.jitterMs,
+		budget: line.budgetMs,
+		maxPages: line.maxPages,
+		grace: line.graceMs,
+		events: line.events,
+		onChange: line.onChange === void 0 ? void 0 : changeCommand(line.onChange),
+		mcp: line.mcp === void 0 ? void 0 : {
+			port: line.mcp.port,
+			host: line.mcp.host,
+			allowRemote: line.mcp.allowRemote,
+			tokenFile: line.mcp.tokenFile,
+			allowUserProvided: io.allowUserProvided
+		},
+		signals: true
+	})).done;
+	return 0;
+};
+//#endregion
+export { daemonCommand };
+
+//# sourceMappingURL=cli-daemon--cTmq0Z5.mjs.map

package/build/cli-daemon--cTmq0Z5.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-daemon--cTmq0Z5.mjs","names":[],"sources":["../src/cli-daemon.ts"],"sourcesContent":["/*\n * The command `daemon` of the command line: `daemon` runs the daemon until `SIGINT` or `SIGTERM`, and\n * `daemon status` prints the state file. `cli.ts` loads this file only for the command `daemon`.\n */\nimport type { DaemonLine } from \"./cli-line.ts\";\nimport { describeDaemonStatus } from \"./cli-line.ts\";\nimport type { OpenOptions } from \"./open.ts\";\n\n/** These are the parts of the command line that the command `daemon` uses. */\nexport interface DaemonIo {\n  /** The value true lets `user_provided` work when the MCP client cannot ask the user. */\n  readonly allowUserProvided: boolean;\n  /** The value true prints the status as JSON. */\n  readonly json: boolean;\n  /** This function prints a line on the standard output. */\n  readonly out: (text: string) => void;\n}\n\n/**\n * This function does the command `daemon`, and gives the exit code. `daemon status` gives 0 when a\n * daemon operates on the store, and 1 when none operates. `daemon` gives 0 after a clean stop. An\n * error at start rejects, so the command exits with 1.\n */\nexport const daemonCommand = async (\n  line: DaemonLine,\n  options: OpenOptions,\n  io: DaemonIo,\n): Promise<number> => {\n  const { changeCommand, readDaemonState, startDaemon } = await import(\"./daemon.ts\");\n  if (line.action === \"status\") {\n    const state = readDaemonState(options.dir);\n    io.out(\n      io.json ? JSON.stringify(state ?? null, null, 2) : describeDaemonStatus(state, options.dir),\n    );\n    return state?.alive === true ? 0 : 1;\n  }\n  const daemon = await startDaemon({\n    ...options,\n    tick: line.tickMs,\n    jitter: line.jitterMs,\n    budget: line.budgetMs,\n    maxPages: line.maxPages,\n    grace: line.graceMs,\n    events: line.events,\n    onChange: line.onChange === undefined ? undefined : changeCommand(line.onChange),\n    mcp:\n      line.mcp === undefined\n        ? undefined\n        : {\n            port: line.mcp.port,\n            host: line.mcp.host,\n            allowRemote: line.mcp.allowRemote,\n            tokenFile: line.mcp.tokenFile,\n            allowUserProvided: io.allowUserProvided,\n          },\n    signals: true,\n  });\n  await daemon.done;\n  return 0;\n};\n"],"mappings":";;;;;;;AAuBA,MAAa,gBAAgB,OAC3B,MACA,SACA,OACoB;CACpB,MAAM,EAAE,eAAe,iBAAiB,gBAAgB,MAAM,OAAO,wBAAc,CAAA,MAAA,MAAA,EAAA,CAAA;CACnF,IAAI,KAAK,WAAW,UAAU;EAC5B,MAAM,QAAQ,gBAAgB,QAAQ,GAAG;EACzC,GAAG,IACD,GAAG,OAAO,KAAK,UAAU,SAAS,MAAM,MAAM,CAAC,IAAI,qBAAqB,OAAO,QAAQ,GAAG,CAC5F;EACA,OAAO,OAAO,UAAU,OAAO,IAAI;CACrC;CAsBA,OAAM,MArBe,YAAY;EAC/B,GAAG;EACH,MAAM,KAAK;EACX,QAAQ,KAAK;EACb,QAAQ,KAAK;EACb,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,QAAQ,KAAK;EACb,UAAU,KAAK,aAAa,KAAA,IAAY,KAAA,IAAY,cAAc,KAAK,QAAQ;EAC/E,KACE,KAAK,QAAQ,KAAA,IACT,KAAA,IACA;GACE,MAAM,KAAK,IAAI;GACf,MAAM,KAAK,IAAI;GACf,aAAa,KAAK,IAAI;GACtB,WAAW,KAAK,IAAI;GACpB,mBAAmB,GAAG;EACxB;EACN,SAAS;CACX,CAAC,EAAA,CACY;CACb,OAAO;AACT"}

package/build/cli.mjs

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
-import { i as VERSION, t as durationMs } from "./duration-iIuMMofi.mjs";
+import { i as durationMs, r as VERSION } from "./version-ntbY-6wh.mjs";
+import { t as isLoopbackHost } from "./loopback-C_QYowSf.mjs";
 import { join, resolve } from "node:path";
 import { readFileSync } from "node:fs";
 import { parseArgs } from "node:util";
@@ -80,30 +81,62 @@ const OPTIONS = {
 	"archive-despite-robots": { type: "boolean" },
 	"sign-key": { type: "string" },
 	"signature-agent": { type: "string" },
+	tick: { type: "string" },
+	jitter: { type: "string" },
+	"tick-budget": { type: "string" },
+	"tick-pages": { type: "string" },
+	grace: { type: "string" },
+	events: { type: "string" },
+	"on-change": { type: "string" },
+	"mcp-http": { type: "boolean" },
+	"mcp-port": { type: "string" },
+	"mcp-host": { type: "string" },
+	"mcp-token-file": { type: "string" },
+	"allow-remote": { type: "boolean" },
 	field: {
 		type: "string",
 		multiple: true
 	},
 	capture: { type: "string" },
+	embedder: { type: "string" },
+	"embed-model": { type: "string" },
+	"ollama-url": { type: "string" },
+	"allow-remote-ollama": { type: "boolean" },
+	"max-docs": { type: "string" },
 	...EXPORT_OPTIONS
 };
+/** These options belong to `daemon`, and not to `daemon status`. */
+const DAEMON_RUN_OPTIONS = [
+	"tick",
+	"jitter",
+	"tick-budget",
+	"tick-pages",
+	"grace",
+	"events",
+	"on-change",
+	"mcp-http",
+	"mcp-port",
+	"mcp-host",
+	"mcp-token-file",
+	"allow-remote"
+];
 /** These options belong to one command, or to the commands that the text names. */
 const OWNER = {
 	render: "fetch crawl watch",
 	"max-age": "fetch crawl discover",
 	limit: "find",
-	kind: "find",
+	kind: "find embed",
 	rank: "find",
 	from: "read diff",
 	length: "read",
 	"user-provided": "read extract",
-	"allow-user-provided": "mcp",
+	"allow-user-provided": "mcp daemon",
 	...Object.fromEntries(Object.keys(EXPORT_OPTIONS).map((option) => [option, "export"])),
 	"max-urls": "discover",
 	feed: "discover",
 	"max-pages": "crawl watch",
 	"max-depth": "crawl",
-	budget: "crawl watch",
+	budget: "crawl watch embed",
 	include: "crawl",
 	exclude: "crawl",
 	sitemaps: "crawl",
@@ -116,10 +149,16 @@ const OWNER = {
 	label: "watch",
 	"archive-fallback": "fetch",
 	"archive-despite-robots": "fetch",
-	"sign-key": "fetch discover crawl watch mcp key-directory",
-	"signature-agent": "fetch discover crawl watch mcp key-directory",
+	"sign-key": "fetch discover crawl watch mcp key-directory daemon",
+	"signature-agent": "fetch discover crawl watch mcp key-directory daemon",
+	...Object.fromEntries(DAEMON_RUN_OPTIONS.map((option) => [option, "daemon"])),
 	field: "extract",
-	capture: "extract"
+	capture: "extract",
+	embedder: "find embed mcp",
+	"embed-model": "find embed mcp",
+	"ollama-url": "find embed mcp",
+	"allow-remote-ollama": "find embed mcp",
+	"max-docs": "embed"
 };
 /** These are the commands, with the smallest and the largest count of arguments. */
 const COMMANDS = {
@@ -178,6 +217,11 @@ const COMMANDS = {
 		max: Infinity,
 		wants: "watch wants add, remove, list or run"
 	},
+	daemon: {
+		min: 0,
+		max: 1,
+		wants: "daemon takes no argument, or status"
+	},
 	"key-directory": {
 		min: 0,
 		max: 0,
@@ -187,6 +231,11 @@ const COMMANDS = {
 		min: 1,
 		max: 1,
 		wants: "extract wants one document id and --field name=value"
+	},
+	embed: {
+		min: 0,
+		max: 0,
+		wants: "embed takes no argument: give the model with --embedder"
 	}
 };
 const parse = (args) => {
@@ -424,6 +473,102 @@ const describeEvent = (event) => {
 	].join("\n");
 };
 const describeWatchSummary = (summary) => `${String(summary.checked)} checked: ${String(summary.new)} new, ${String(summary.changed)} changed, ${String(summary.unchanged)} unchanged, ${String(summary.gone)} gone, ${String(summary.refused)} refused, ${String(summary.error)} errors. ${String(summary.suspected)} suspected, ${String(summary.confirmed)} confirmed, ${String(summary.due)} still due. Stopped (${summary.stopped}): ${summary.reason}.`;
+/** This function reads a file name of the command line: a leading `~` is the home folder. */
+const fileOf = (text) => text === "~" ? homedir() : /^~[\\/]/.test(text) ? resolve(homedir(), text.slice(2)) : resolve(text);
+/** This function reads a duration of `daemon`, with a floor in milliseconds. */
+const daemonDuration = (name, text, floor) => {
+	if (text === void 0) return void 0;
+	let ms;
+	try {
+		ms = durationMs(text);
+	} catch (error) {
+		throw new UsageError(`--${name}: ${error.message}`, { cause: error });
+	}
+	if (ms < floor) throw new UsageError(floor === 0 ? `--${name} wants 0 or more` : `--${name} wants ${String(floor / 1e3)}s or more`);
+	return ms;
+};
+/** These options of `daemon` belong to its MCP server, so they need `--mcp-http`. */
+const MCP_OPTIONS = [
+	"mcp-port",
+	"mcp-host",
+	"mcp-token-file",
+	"allow-remote"
+];
+/** This function checks the options of `daemon` and of `daemon status`. */
+const daemonLineOf = (values, rest) => {
+	const [action] = rest;
+	if (action !== void 0 && action !== "status") throw new UsageError(`daemon takes no argument, or status, not "${action}"`);
+	if (action === "status") {
+		const run = [
+			...DAEMON_RUN_OPTIONS,
+			"allow-user-provided",
+			"sign-key",
+			"signature-agent"
+		];
+		for (const option of run) if (values[option] !== void 0) throw new UsageError(`--${option} is an option of daemon, not of daemon status`);
+	}
+	if (values["mcp-http"] !== true) {
+		for (const option of MCP_OPTIONS) if (values[option] !== void 0) throw new UsageError(`--${option} goes with --mcp-http`);
+	}
+	const host = (values["mcp-host"] ?? "127.0.0.1").trim();
+	if (host === "") throw new UsageError("--mcp-host wants a host");
+	const allowRemote = values["allow-remote"] === true;
+	if (values["mcp-http"] === true && !isLoopbackHost(host) && !allowRemote) throw new UsageError(`--mcp-host ${host} is not a loopback address: add --allow-remote to serve other computers, after you read the warning in the README`);
+	const port = whole("mcp-port", values["mcp-port"], 9307, 0);
+	if (port > 65535) throw new UsageError("--mcp-port wants a port from 0 to 65535");
+	const onChange = values["on-change"];
+	if (onChange !== void 0 && onChange.trim() === "") throw new UsageError("--on-change wants a command");
+	const events = values.events;
+	if (events !== void 0 && events.trim() === "") throw new UsageError("--events wants a file");
+	const tokenFile = values["mcp-token-file"];
+	if (tokenFile !== void 0 && tokenFile.trim() === "") throw new UsageError("--mcp-token-file wants a file");
+	return {
+		action: action === "status" ? "status" : "run",
+		tickMs: daemonDuration("tick", values.tick, 1e3),
+		jitterMs: daemonDuration("jitter", values.jitter, 0),
+		budgetMs: daemonDuration("tick-budget", values["tick-budget"], 1e3),
+		maxPages: values["tick-pages"] === void 0 ? void 0 : whole("tick-pages", values["tick-pages"], 1, 1),
+		graceMs: daemonDuration("grace", values.grace, 0),
+		events: events === void 0 ? void 0 : fileOf(events),
+		onChange,
+		mcp: values["mcp-http"] === true ? {
+			port,
+			host,
+			allowRemote,
+			tokenFile: tokenFile === void 0 ? void 0 : fileOf(tokenFile)
+		} : void 0
+	};
+};
+/** This function gives the first line of `daemon status`: the daemon runs, stopped, or stopped with no clean stop. */
+const daemonHead = (state) => state.alive ? `the daemon runs (process ${String(state.pid)}) since ${state.started_at}, on the store ${state.store}` : state.running ? `the daemon of process ${String(state.pid)} does not run: it stopped with no clean stop. Its state is of ${state.updated_at}` : `the daemon stopped at ${state.stopped_at ?? state.updated_at}`;
+/** This function gives the line of the last tick of `daemon status`. */
+const lastTick = (state) => {
+	const tick = state.last_tick;
+	if (tick === null) return "last tick: none yet";
+	if (tick.summary === null) return `last tick: ${tick.ended_at}, failed: ${tick.error ?? "error"}`;
+	const summary = tick.summary;
+	return `last tick: ${tick.ended_at}, ${String(summary.checked)} checked, ${String(summary.confirmed)} changes, ${String(summary.refused)} refused, ${String(summary.error)} errors (${summary.stopped})`;
+};
+/** This function gives the text of `daemon status`. */
+const describeDaemonStatus = (state, dir) => {
+	if (state === void 0) return `no daemon has run on the store ${dir}`;
+	const lines = [daemonHead(state), lastTick(state)];
+	if (state.alive && state.next_tick_at !== null) lines.push(`next tick: ${state.next_tick_at}`);
+	lines.push(`${String(state.ticks)} ticks, ${String(state.checked)} checks, ${String(state.changes)} changes`);
+	if (state.next_due.length === 0) lines.push("next due: no watch");
+	else {
+		lines.push("next due:");
+		for (const due of state.next_due) lines.push(`  ${due.next_at}  ${due.url}  (every ${everyText(due.every_ms)}, last ${due.last_kind ?? "none"})`);
+	}
+	if (state.mcp !== null) {
+		const where = state.mcp.token_file === null ? "" : ` (the token is in ${state.mcp.token_file})`;
+		lines.push(`MCP: ${state.mcp.url}${where}`);
+		lines.push(state.tasks.length === 0 ? "tasks: none running" : `tasks: ${String(state.tasks.length)} running`);
+		for (const task of state.tasks) lines.push(`  ${task.id}  ${task.tool}  ${task.status}${task.message === null ? "" : `: ${task.message}`}`);
+	}
+	lines.push(state.last_error === null ? "last error: none" : `last error: ${state.last_error.at}: ${state.last_error.message}`);
+	return lines.join("\n");
+};
 /** These are the kinds of a field (`FIELD_KINDS` of `extract.ts`). The command line knows them before it loads the crawler. */
 const KINDS = [
 	"text",
@@ -463,6 +608,33 @@ const fieldsOf = (given) => {
 	}
 	return fields;
 };
+/**
+* This function reads the embedder of the command line, or of the environment for the commands
+* that use one. `--embed-model` and `--ollama-url` need `--embedder`.
+*/
+const embedderOf = (values, env, command) => {
+	const uses = OWNER["embedder"].split(" ").includes(command);
+	const fromEnv = (name) => uses ? env[name] || void 0 : void 0;
+	const name = values.embedder ?? fromEnv("POLITE_CRAWLER_EMBEDDER");
+	const model = values["embed-model"] ?? fromEnv("POLITE_CRAWLER_EMBED_MODEL");
+	const url = values["ollama-url"] ?? fromEnv("POLITE_CRAWLER_OLLAMA_URL");
+	const allowRemote = values["allow-remote-ollama"] === true || uses && envFlag(env, "POLITE_CRAWLER_ALLOW_REMOTE_OLLAMA");
+	if (name === void 0) {
+		if (values["embed-model"] !== void 0 || values["ollama-url"] !== void 0) throw new UsageError("--embed-model and --ollama-url go with --embedder transformers or ollama");
+		return;
+	}
+	if (name !== "transformers" && name !== "ollama") throw new UsageError(`--embedder is transformers or ollama, not "${name}"`);
+	if (model !== void 0 && model.trim() === "") throw new UsageError("--embed-model wants a model");
+	if (values["ollama-url"] !== void 0 && name !== "ollama") throw new UsageError("--ollama-url goes with --embedder ollama");
+	return {
+		kind: name,
+		model,
+		url: name === "ollama" ? url : void 0,
+		allowRemote
+	};
+};
+/** This function gives the text of `embed`: the counts and the reason of the stop. */
+const describeEmbed = (result) => [`${String(result.embedded)} texts got vectors (${String(result.chunks)} chunks); ${String(result.reused)} documents had vectors; ${String(result.held_back)} held back, so not sent to the model`, `${String(result.docs)} documents examined in ${String(result.ms)} ms with ${result.embedder}. Stopped (${result.stopped})${result.stopped === "done" ? "" : ": start embed again for the rest"}.`].join("\n");
 /** This function gives the pages of `fetch`: a rejected fetch is an error of its URL only. A page that needs a browser names the reason when rendering is off. */
 const fetchedPages = (urls, settled, renderOff) => settled.map((result, at) => {
 	const url = urls[at];
@@ -477,7 +649,7 @@ const fetchedPages = (urls, settled, renderOff) => settled.map((result, at) => {
 	} : page;
 });
 /** This function gives the text of the results of `find`. An archived copy says so. */
-const describeFound = (found) => found.length === 0 ? "nothing stored matches" : found.map((hit) => `${hit.doc}  ${hit.title}${hit.archived === void 0 ? "" : `  [archived copy, captured ${hit.archived.timestamp}]`}\n  ${hit.snippet.replace(/\s+/g, " ")}`).join("\n");
+const describeFound = (found) => found.length === 0 ? "nothing stored matches" : found.map((hit) => `${hit.doc}  ${hit.title}${hit.archived === void 0 ? "" : `  [archived copy, captured ${hit.archived.timestamp}]`}${hit.passage === void 0 ? "" : `  [passage ${String(hit.passage.start)}–${String(hit.passage.end)}]`}\n  ${hit.snippet.replace(/\s+/g, " ")}`).join("\n");
 /** This function gives the text of `read`. With `--at`, the head names the capture and its fetch time. */
 const describeRead = (read, at) => {
 	const version = at === void 0 ? "" : ` ${read.capture} (${read.fetched_at})`;
@@ -577,7 +749,11 @@ const readLine = (args, env) => {
 	const render = values.render ?? "auto";
 	if (render !== "auto" && render !== "never" && render !== "always") throw new UsageError(`--render is auto, never or always, not "${render}"`);
 	const rank = values.rank ?? "exact";
-	if (rank !== "exact" && rank !== "fast") throw new UsageError(`--rank is exact or fast, not "${rank}"`);
+	if (rank !== "exact" && rank !== "fast" && rank !== "hybrid") throw new UsageError(`--rank is exact, fast or hybrid, not "${rank}"`);
+	const embedder = embedderOf(values, env, command);
+	if (rank === "hybrid" && embedder === void 0) throw new UsageError("--rank hybrid wants --embedder transformers or ollama");
+	if (command === "embed" && embedder === void 0) throw new UsageError("embed wants --embedder transformers or ollama");
+	const maxDocs = values["max-docs"] === void 0 ? void 0 : whole("max-docs", values["max-docs"], 1, 1);
 	const limit = whole("limit", values.limit, 10, 1);
 	const from = command === "read" ? whole("from", values.from, 0, 0) : 0;
 	const length = whole("length", values.length, 4e3, 1);
@@ -587,11 +763,12 @@ const readLine = (args, env) => {
 	const diffTo = versionOf("to", values.to);
 	const watchLine = command === "watch" ? watchLineOf(values, rest) : void 0;
 	const extractFields = command === "extract" ? fieldsOf(values.field) : void 0;
+	const daemonLine = command === "daemon" ? daemonLineOf(values, rest) : void 0;
 	const dir = storeDir(values.dir ?? (env["POLITE_CRAWLER_DIR"] || void 0));
 	const allowPrivate = values["allow-private"] === true || envFlag(env, "POLITE_CRAWLER_ALLOW_PRIVATE");
-	const allowUserProvided = values["allow-user-provided"] === true || command === "mcp" && envFlag(env, "POLITE_CRAWLER_ALLOW_USER_PROVIDED");
+	const allowUserProvided = values["allow-user-provided"] === true || (command === "mcp" || daemonLine?.action === "run") && envFlag(env, "POLITE_CRAWLER_ALLOW_USER_PROVIDED");
 	if (values["archive-despite-robots"] === true && values["archive-fallback"] !== true) throw new UsageError("--archive-despite-robots works only with --archive-fallback");
-	const signs = OWNER["sign-key"].split(" ").includes(command) && (watchLine === void 0 || watchLine.action === "run");
+	const signs = OWNER["sign-key"].split(" ").includes(command) && (watchLine === void 0 || watchLine.action === "run") && (daemonLine === void 0 || daemonLine.action === "run");
 	const signKey = values["sign-key"] ?? (signs ? env["POLITE_CRAWLER_SIGN_KEY"] || void 0 : void 0);
 	const signatureAgent = values["signature-agent"] ?? (signs ? env["POLITE_CRAWLER_SIGNATURE_AGENT"] || void 0 : void 0);
 	if (command === "key-directory") {
@@ -619,6 +796,7 @@ const readLine = (args, env) => {
 		diffFrom,
 		diffTo,
 		watchLine,
+		daemonLine,
 		extractFields,
 		exportPlan: command === "export" ? exportPlan(values) : void 0,
 		dir,
@@ -626,7 +804,9 @@ const readLine = (args, env) => {
 		allowPrivate,
 		allowUserProvided,
 		signKey,
-		signatureAgent
+		signatureAgent,
+		embedder,
+		maxDocs
 	};
 };
 //#endregion
@@ -656,9 +836,13 @@ Usage:
   polite-crawler watch remove <url...> stop the watch of these pages
   polite-crawler watch list            print the watch list
   polite-crawler watch run             check the pages that are due, within a budget
+  polite-crawler daemon                check the watch list on a schedule, until SIGINT or SIGTERM;
+                                       with --mcp-http, serve the MCP tools over local HTTP too
+  polite-crawler daemon status         print the state of the daemon of the store
   polite-crawler key-directory         print the key directory (JWKS) of --sign-key, to host
   polite-crawler extract <doc> --field name=value[:kind]...
                                        check proposed field values against a stored page
+  polite-crawler embed --embedder <name>  give vectors to the stored pages, for find --rank hybrid
 
 Options:
   --dir <path>       the store (default $POLITE_CRAWLER_DIR, or ~/.polite-crawler)
@@ -670,6 +854,13 @@ Options:
                      ($POLITE_CRAWLER_SIGN_KEY)
   --signature-agent <url>  the https origin of your key directory, or the URL of a
                      JSON Web Key Set on a path, for --sign-key ($POLITE_CRAWLER_SIGNATURE_AGENT)
+  --embedder <name>  the local model of hybrid search, for find, embed and mcp: transformers
+                     (transformers.js in this process) or ollama ($POLITE_CRAWLER_EMBEDDER)
+  --embed-model <m>  the model: Xenova/bge-small-en-v1.5 (transformers) or nomic-embed-text
+                     (ollama) by default ($POLITE_CRAWLER_EMBED_MODEL)
+  --ollama-url <url> the Ollama server, http://127.0.0.1:11434 by default ($POLITE_CRAWLER_OLLAMA_URL)
+  --allow-remote-ollama  permit an Ollama server on a different computer: the page text goes
+                     to it ($POLITE_CRAWLER_ALLOW_REMOTE_OLLAMA=1)
   --json             print JSON
   -v, --version      print the version
   -h, --help         print this help
@@ -683,7 +874,8 @@ Options:
   find     --limit <n>       how many results (default 10)
            --kind <kind>     only this kind of document, such as page or pdf
            --rank <mode>     exact (default): BM25 over each match; fast: leave common words
-                             out of the one-word part, faster for a long question
+                             out of the one-word part, faster for a long question; hybrid: BM25
+                             and the meaning of the text, fused (needs --embedder)
   read     --from <n>        where the range starts (default 0)
            --length <n>      how many characters (default 4000)
            --user-provided   you gave this URL yourself: read a page that refuses AI input (logged)
@@ -720,17 +912,38 @@ Options:
              --render <mode> as for fetch
   watch run  --budget <time>  the longest time (default 10m); no request starts after it
              --max-pages <n>  how many pages to check (default 100)
+  daemon   --tick <time>     the longest wait between two ticks: 1s or more (default 1m)
+           --jitter <time>   a random wait from 0 to this after each tick (default 10s)
+           --tick-budget <time>  the longest time of the checks of one tick (default 10m)
+           --tick-pages <n>  how many pages one tick checks at most (default 100)
+           --grace <time>    at a stop, the checks in progress get this time to end (default 10s)
+           --events <file>   write each change as one JSON line to this file (default: the
+                             standard output)
+           --on-change <command>  start this command of yours for each change: it gets the
+                             event as one JSON line on its standard input
+           --mcp-http        serve the MCP tools over HTTP; each request needs the bearer token
+           --mcp-port <n>    the port (default 9307)
+           --mcp-host <host> the host to bind (default 127.0.0.1)
+           --mcp-token-file <file>  read the token from this file (default: a new token at each
+                             start, in <store>/daemon/mcp-token)
+           --allow-remote    permit a --mcp-host that is not loopback: read the README first
+           --allow-user-provided  as for mcp
+  daemon status              exit code 0 when a daemon runs on the store, 1 when none runs
   extract  --field <name=value[:kind]>  a value to check, never made up by the crawler (repeat it);
                              kind: text (default), number, money, date, url, email, phone,
                              or json-ld:<path> such as json-ld:Product.offers.price
            --capture <id>    check the text of this capture, not the current text
            --user-provided   as for read
+  embed    --budget <time>   the longest time (default 10m); no embedding starts after it
+           --max-docs <n>    how many texts get vectors at most
+           --kind <kind>     only this kind of document
 
 Exit codes:
   0  done
   1  not done: a page not fetched, no match, a quote not found, a document unknown or
      held back, a watch check that failed or that is still due, a field value that extract
-     did not find or that is not clear, or an error
+     did not find or that is not clear, embed that stopped before each page had vectors,
+     or an error
      (fetch: a page with an archived copy counts as done)
   2  the command line is wrong
 `;
@@ -739,7 +952,7 @@ Exit codes:
 * command line first. The export only reads the store: it makes no store, and it sends no request.
 */
 const exportCommand = async (plan, dir, json) => {
-	const result = plan.format === "wacz" ? await (await import("./wacz-sfOLT8e9.mjs").then((n) => n.n)).exportWacz(dir, plan.options) : await (await import("./warc-VIPscWxY.mjs").then((n) => n.a)).exportWarc(dir, plan.options);
+	const result = plan.format === "wacz" ? await (await import("./wacz-BFNG42Kt.mjs").then((n) => n.n)).exportWacz(dir, plan.options) : await (await import("./warc-CVS0t1fW.mjs").then((n) => n.a)).exportWarc(dir, plan.options);
 	process.stdout.write(`${json ? JSON.stringify(result, null, 2) : describeExport(result, plan.options.includeRefusals)}\n`);
 	return 0;
 };
@@ -861,7 +1074,7 @@ const runWatchCommand = async (crawler, line, values, limits, json, out) => {
 };
 /** This function gives the text of a diff for a person: the two captures, then the text of `formatDiff`. */
 const describeDiff = async (diff) => {
-	const { formatDiff } = await import("./diff-Dk2eSvi-.mjs").then((n) => n.i);
+	const { formatDiff } = await import("./diff-NQcQqKe9.mjs").then((n) => n.i);
 	return [
 		`${diff.doc}  ${diff.from.capture} (${diff.from.fetched_at}) -> ${diff.to.capture} (${diff.to.fetched_at})`,
 		formatDiff(diff),
@@ -871,13 +1084,54 @@ const describeDiff = async (diff) => {
 };
 /** This function checks the fields against a stored document, and prints a line for each field. The exit code is 0 when each field is verified or normalized. */
 const runExtract = async (crawler, doc, fields, options, json, out) => {
-	const { extractDocument } = await import("./extract-DcwNrTK7.mjs").then((n) => n.i);
+	const { extractDocument } = await import("./extract-ByFfc8Zk.mjs").then((n) => n.i);
 	const claims = fields;
 	const result = crawler.extract === void 0 ? await extractDocument(crawler.runtime, doc, claims, options) : await crawler.extract(doc, claims, options);
 	if (json) out(JSON.stringify(result, null, 2));
 	else for (const line of describeExtract(result)) out(line);
 	return extractCode(result);
 };
+/**
+* This function makes the embedder of the command line. It loads no model: the model loads at the
+* first text. The transformers embedder keeps the files of its model in `<store>/models`.
+*/
+const embedderFor = async (choice, dir) => {
+	try {
+		if (choice.kind === "ollama") {
+			const { ollamaEmbedder } = await import("./ollama-BJxkjny4.mjs").then((n) => n.a);
+			return ollamaEmbedder({
+				...choice.model === void 0 ? {} : { model: choice.model },
+				...choice.url === void 0 ? {} : { baseUrl: choice.url },
+				allowRemote: choice.allowRemote
+			});
+		}
+		const { transformersEmbedder } = await import("./transformers-BQMlaGj4.mjs").then((n) => n.i);
+		return transformersEmbedder({
+			...choice.model === void 0 ? {} : { model: choice.model },
+			cacheDir: join(dir, "models")
+		});
+	} catch (error) {
+		throw new UsageError(`--embedder ${choice.kind}: ${error.message}`, { cause: error });
+	}
+};
+/** This function gives vectors to the stored pages within the budget. The exit code is 0 when each page has vectors. */
+const runEmbed = async (crawler, limits, options, json, out) => {
+	const controller = new AbortController();
+	const stop = () => controller.abort(/* @__PURE__ */ new Error("interrupted"));
+	process.once("SIGINT", stop);
+	try {
+		const result = await crawler.embed({
+			budget: limits.budgetMs,
+			signal: controller.signal,
+			...options.maxDocs === void 0 ? {} : { maxDocs: options.maxDocs },
+			...options.kind === void 0 ? {} : { kind: options.kind }
+		});
+		out(json ? JSON.stringify(result, null, 2) : describeEmbed(result));
+		return result.stopped === "done" ? 0 : 1;
+	} finally {
+		process.removeListener("SIGINT", stop);
+	}
+};
 /** This function does the command of the command line, and gives the exit code. */
 const main = async (args) => {
 	const out = (text) => void process.stdout.write(`${text}\n`);
@@ -891,6 +1145,7 @@ const main = async (args) => {
 		return 0;
 	}
 	const { command, rest, values, render, rank, limit, from, length, maxAge, crawlLimits, at, diffFrom, diffTo, watchLine, extractFields, dir, signKey, signatureAgent } = line;
+	const embedder = line.embedder === void 0 ? void 0 : await embedderFor(line.embedder, dir);
 	if (command === "key-directory") return keyDirectoryCommand(keyFile(signKey), signatureAgent);
 	let signer;
 	if (signKey !== void 0 && signatureAgent !== void 0) {
@@ -911,7 +1166,8 @@ const main = async (args) => {
 		contact: line.contact,
 		...values["no-browser"] === true ? { renderer: false } : {},
 		...line.allowPrivate ? { egress: { allowPrivate: true } } : {},
-		...signer === void 0 ? {} : { signer }
+		...signer === void 0 ? {} : { signer },
+		...embedder === void 0 ? {} : { embedder }
 	};
 	if (command === "mcp") {
 		const { startMcpServer } = await import("./mcp.mjs");
@@ -921,8 +1177,16 @@ const main = async (args) => {
 		});
 		return 0;
 	}
+	if (command === "daemon") {
+		const { daemonCommand } = await import("./cli-daemon--cTmq0Z5.mjs");
+		return daemonCommand(line.daemonLine, options, {
+			allowUserProvided: line.allowUserProvided,
+			json: values.json === true,
+			out
+		});
+	}
 	if (line.exportPlan !== void 0) return exportCommand(line.exportPlan, dir, values.json === true);
-	const { openCrawler } = await import("./open-DnrWgkC0.mjs").then((n) => n.a);
+	const { openCrawler } = await import("./open-C4fhXJgD.mjs").then((n) => n.a);
 	let renderOff;
 	let renderer = options.renderer;
 	if (!(command === "fetch" || command === "crawl" || watchLine?.action === "run") || render === "never") renderer = false;
@@ -1005,6 +1269,10 @@ const main = async (args) => {
 				userProvided: values["user-provided"] === true,
 				at
 			}, values.json === true, out);
+			case "embed": return await runEmbed(crawler, crawlLimits, {
+				maxDocs: line.maxDocs,
+				kind: values.kind
+			}, values.json === true, out);
 			default: {
 				const counts = await crawler.reindex();
 				print(counts, () => describeReindex(counts));
@@ -1024,6 +1292,6 @@ main(process.argv.slice(2)).then((code) => {
 	process.exitCode = usage ? 2 : 1;
 });
 //#endregion
-export {};
+export { describeDaemonStatus as t };
 
 //# sourceMappingURL=cli.mjs.map