polite-crawler-mcp 0.0.0-bootstrap.0 → 0.2.0

package/build/index.mjs

@@ -0,0 +1,266 @@
+import { a as INSTRUCTIONS, i as startMcpServer, n as createMcpServer, o as crawlContent, s as crawlTools, t as crawlHostOptions } from "./server-BYGFTvt_.mjs";
+import { n as VERSION, t as SERVER_NAME } from "./version-B9xqi8IR.mjs";
+import { McpTaskStore, McpTaskStore as McpTaskStore$1, createMcpHost } from "@procedures/mcp-server";
+import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
+import { readFileSync, statSync } from "node:fs";
+import { createServer } from "node:http";
+import { isIP } from "node:net";
+//#region src/loopback.ts
+/**
+* This function tells if a host name or an address is a loopback host: `localhost`, an address in
+* `127.0.0.0/8`, or `::1`. The MCP server over HTTP binds to a loopback host unless the operator
+* permits a different host. The function imports only Node.js, so the command line uses it before
+* the store opens.
+*/
+const isLoopbackHost = (host) => {
+	const name = host.trim().toLowerCase().replace(/^\[(.*)\]$/, "$1");
+	if (name === "localhost" || name.endsWith(".localhost")) return true;
+	const mapped = name.startsWith("::ffff:") ? name.slice(7) : name;
+	if (isIP(mapped) === 4) return mapped.startsWith("127.");
+	if (isIP(name) === 6) return /^(?:0{0,4}:){7}0{0,3}1$|^::1$/.test(name);
+	return false;
+};
+/** This function gives a host for a URL: an IPv6 address goes in brackets. */
+const urlHost = (host) => isIP(host) === 6 ? `[${host}]` : host;
+//#endregion
+//#region src/http.ts
+/** These are the defaults of the MCP server over HTTP. */
+const MCP_HTTP_DEFAULTS = {
+	/** The port of the server. */
+	port: 9307,
+	/** The host of the server: the IPv4 loopback address. */
+	host: "127.0.0.1",
+	/** The path of the MCP endpoint. */
+	path: "/mcp",
+	/** The path of the status of the daemon. */
+	statusPath: "/status",
+	/** The most sessions at one time. A new session above the limit closes the session that was idle longest. */
+	maxSessions: 32,
+	/** The largest body of a request that starts a session: 4 MiB. */
+	maxBodyBytes: 4194304,
+	/** The shortest token: 16 characters. */
+	minTokenLength: 16
+};
+const digest = (text) => createHash("sha256").update(text, "utf8").digest();
+/**
+* This function compares a token with the token of the server in constant time. It compares the
+* SHA-256 digests, so the time does not depend on the length or the text of either token.
+*/
+const sameToken = (given, token) => timingSafeEqual(digest(given), digest(token));
+/** This function makes a random token: 32 bytes in base64url, 43 characters. */
+const makeToken = () => randomBytes(32).toString("base64url");
+/** A token is printable ASCII with no space, so a header holds it. */
+const TOKEN = /^[\x21-\x7e]+$/;
+/** This function checks a token. A token that is short or that a header cannot hold throws an error. */
+const checkToken = (token, source) => {
+	const trimmed = token.trim();
+	if (!TOKEN.test(trimmed) || trimmed.length < MCP_HTTP_DEFAULTS.minTokenLength) throw new Error(`${source}: the token must be ${String(MCP_HTTP_DEFAULTS.minTokenLength)} or more printable characters with no space`);
+	return trimmed;
+};
+/** This function reads a token file. It gives a warning when other users can read the file. */
+const tokenOf = (options, log) => {
+	if (options.token !== void 0) return checkToken(options.token, "token");
+	if (options.tokenFile === void 0) return makeToken();
+	let text;
+	try {
+		text = readFileSync(options.tokenFile, "utf8");
+	} catch (error) {
+		const code = error.code;
+		throw new Error(`the token file ${options.tokenFile} does not open (${typeof code === "string" ? code : "error"})`, { cause: error });
+	}
+	if (process.platform !== "win32" && (statSync(options.tokenFile).mode & 63) !== 0) log(`polite-crawler: other users can read the token file ${options.tokenFile}: chmod 600 it`);
+	return checkToken(text, `the token file ${options.tokenFile}`);
+};
+const json = (response, status, body, headers = {}) => {
+	if (response.headersSent) return;
+	response.writeHead(status, {
+		"content-type": "application/json",
+		...headers
+	});
+	response.end(JSON.stringify(body));
+};
+/** This function gives the answer of a refusal, as a JSON-RPC error with no id. */
+const refuse = (response, status, message, headers = {}) => json(response, status, {
+	jsonrpc: "2.0",
+	error: {
+		code: -32e3,
+		message
+	},
+	id: null
+}, headers);
+/**
+* The number of open sessions of the HTTP face. The state of the face gives it when the shared host
+* has the field. Else the count of its Streamable HTTP endpoint gives it.
+*/
+const sessionsOf = (face) => {
+	const state = face.getState();
+	if (typeof state.sessions === "number") return state.sessions;
+	const endpoint = face.http;
+	return typeof endpoint?.sessions === "number" ? endpoint.sessions : 0;
+};
+/**
+* This function serves an MCP host over HTTP. `makeHost` makes the host with its HTTP face mounted
+* at `/mcp`, with the task store of the server. Each request needs `Authorization: Bearer <token>`.
+*/
+const serveMcpHttp = async (makeHost, options = {}) => {
+	const log = options.log ?? (() => void 0);
+	const host = options.host ?? MCP_HTTP_DEFAULTS.host;
+	const port = options.port ?? MCP_HTTP_DEFAULTS.port;
+	if (!Number.isInteger(port) || port < 0 || port > 65535) throw new RangeError(`the port must be an integer from 0 to 65535, not ${String(port)}`);
+	const loopback = isLoopbackHost(host);
+	if (!loopback && options.allowRemote !== true) throw new Error(`the host ${host} is not a loopback address: give allowRemote (--allow-remote) to serve other computers`);
+	const token = tokenOf(options, log);
+	const tasks = options.tasks ?? new McpTaskStore$1();
+	const mcp = makeHost({
+		token,
+		tasks
+	});
+	const face = mcp.transports[0];
+	if (face === void 0) throw new Error("the MCP host has no HTTP face");
+	await mcp.start();
+	let bound = port;
+	let closing;
+	/** A loopback server accepts only a loopback name and its port in `Host`. */
+	const hostAllowed = (header) => {
+		if (!loopback) return true;
+		if (header === void 0) return false;
+		let parsed;
+		try {
+			parsed = new URL(`http://${header}`);
+		} catch {
+			return false;
+		}
+		return isLoopbackHost(parsed.hostname) && (parsed.port === "" || Number(parsed.port) === bound);
+	};
+	/** An `Origin` header must name the host of the request: a page of a different site gets 403. */
+	const originAllowed = (origin, header) => {
+		if (origin === void 0) return true;
+		try {
+			return new URL(origin).host === header;
+		} catch {
+			return false;
+		}
+	};
+	const authorized = (header) => {
+		const match = /^Bearer\s+(\S+)\s*$/i.exec(header ?? "");
+		return match !== null && sameToken(match[1], token);
+	};
+	const handle = (request, response) => {
+		const hostHeader = request.headers.host;
+		if (!hostAllowed(hostHeader) || !originAllowed(request.headers.origin, hostHeader)) {
+			refuse(response, 403, "the Host or the Origin header is not of this server");
+			return;
+		}
+		if (!authorized(request.headers.authorization)) {
+			refuse(response, 401, "the request needs the bearer token of the server", { "www-authenticate": "Bearer realm=\"polite-crawler\"" });
+			return;
+		}
+		if (new URL(request.url ?? "/", "http://localhost").pathname === MCP_HTTP_DEFAULTS.statusPath && options.status !== void 0) {
+			if (request.method !== "GET") refuse(response, 405, "the status takes GET");
+			else json(response, 200, options.status(), { "cache-control": "no-store" });
+			return;
+		}
+		if (!face.handleHttp(request, response)) refuse(response, 404, `no such path: the MCP endpoint is ${MCP_HTTP_DEFAULTS.path}`);
+	};
+	const http = createServer((request, response) => {
+		try {
+			handle(request, response);
+		} catch (error) {
+			log(`polite-crawler: an MCP request failed: ${error instanceof Error ? error.message : String(error)}`);
+			if (!response.headersSent) refuse(response, 500, "the server could not handle the request");
+			else response.end();
+		}
+	});
+	try {
+		await new Promise((resolve, reject) => {
+			const failed = (error) => {
+				reject(error.code === "EADDRINUSE" ? new Error(`the port ${String(port)} of ${host} is in use: give a different port`, { cause: error }) : error);
+			};
+			http.once("error", failed);
+			http.listen(port, host, () => {
+				http.removeListener("error", failed);
+				resolve();
+			});
+		});
+	} catch (error) {
+		await mcp.stop();
+		await tasks.close();
+		throw error;
+	}
+	bound = http.address().port;
+	const url = `http://${urlHost(host)}:${String(bound)}${MCP_HTTP_DEFAULTS.path}`;
+	if (!loopback) log(`polite-crawler: the MCP server listens on ${host}, not on a loopback address: other computers can reach it`);
+	return {
+		url,
+		host,
+		port: bound,
+		token,
+		tasks,
+		sessions: () => sessionsOf(face),
+		close() {
+			closing ??= (async () => {
+				await tasks.close();
+				await mcp.stop();
+				await new Promise((resolve) => {
+					http.close(() => resolve());
+					http.closeAllConnections();
+				});
+			})();
+			return closing;
+		}
+	};
+};
+/**
+* This function serves the tools of a crawler over the Streamable HTTP transport of MCP. The
+* sessions share one MCP host with the nine tools of the server on the standard input and output.
+* The envelope, the elicitation and the opt-ins are the same. `crawl_site` and `crawl_discover` are
+* MCP tasks. Each request needs the bearer token. The function does not close the crawler.
+*/
+const startMcpHttpServer = (crawler, options = {}) => {
+	const { onBehalfOf, allowUserProvided, renderOff, ...http } = options;
+	return serveMcpHttp(({ tasks }) => createMcpHost({
+		...crawlHostOptions(crawler, {
+			onBehalfOf,
+			allowUserProvided,
+			renderOff,
+			tasks
+		}),
+		faces: { http: {
+			mount: true,
+			path: MCP_HTTP_DEFAULTS.path,
+			allowedHosts: "*",
+			maxSessions: MCP_HTTP_DEFAULTS.maxSessions,
+			maxBodyBytes: MCP_HTTP_DEFAULTS.maxBodyBytes
+		} }
+	}), http);
+};
+/**
+* This host serves the tools of the daemon over HTTP. It is the seam between the daemon and its MCP
+* transport: a different MCP host can replace it with `DaemonOptions.mcpHost`.
+*/
+const mcpHttpHost = { async start(crawler, options, hooks) {
+	const tasks = new McpTaskStore$1({ onChange: hooks.onTasks });
+	const served = await startMcpHttpServer(crawler, {
+		...options,
+		tasks,
+		status: hooks.status,
+		log: hooks.log
+	});
+	const running = () => tasks.running().map((task) => ({
+		id: task.id,
+		tool: task.tool,
+		status: task.status,
+		message: task.message,
+		created_at: task.createdAt,
+		updated_at: task.updatedAt
+	}));
+	return {
+		url: served.url,
+		tasks: running,
+		close: () => served.close()
+	};
+} };
+//#endregion
+export { INSTRUCTIONS, MCP_HTTP_DEFAULTS, McpTaskStore, SERVER_NAME, VERSION, checkToken, crawlContent, crawlHostOptions, crawlTools, createMcpServer, makeToken, mcpHttpHost, sameToken, serveMcpHttp, startMcpHttpServer, startMcpServer };
+
+//# sourceMappingURL=index.mjs.map

package/build/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":["McpTaskStore"],"sources":["../src/loopback.ts","../src/http.ts"],"sourcesContent":["import { isIP } from \"node:net\";\n\n/**\n * This function tells if a host name or an address is a loopback host: `localhost`, an address in\n * `127.0.0.0/8`, or `::1`. The MCP server over HTTP binds to a loopback host unless the operator\n * permits a different host. The function imports only Node.js, so the command line uses it before\n * the store opens.\n */\nexport const isLoopbackHost = (host: string): boolean => {\n  const name = host\n    .trim()\n    .toLowerCase()\n    .replace(/^\\[(.*)\\]$/, \"$1\");\n  if (name === \"localhost\" || name.endsWith(\".localhost\")) return true;\n  const mapped = name.startsWith(\"::ffff:\") ? name.slice(7) : name;\n  if (isIP(mapped) === 4) return mapped.startsWith(\"127.\");\n  if (isIP(name) === 6) return /^(?:0{0,4}:){7}0{0,3}1$|^::1$/.test(name);\n  return false;\n};\n\n/** This function gives a host for a URL: an IPv6 address goes in brackets. */\nexport const urlHost = (host: string): string => (isIP(host) === 6 ? `[${host}]` : host);\n","import { createHash, randomBytes, timingSafeEqual } from \"node:crypto\";\nimport { readFileSync, statSync } from \"node:fs\";\nimport type { IncomingMessage, ServerResponse } from \"node:http\";\nimport { createServer } from \"node:http\";\nimport type { AddressInfo } from \"node:net\";\nimport type { McpHost, McpServerTransport } from \"@procedures/mcp-server\";\nimport { McpTaskStore, createMcpHost } from \"@procedures/mcp-server\";\nimport type { DaemonMcpHost, DaemonTask, PoliteCrawler } from \"polite-crawler\";\nimport { isLoopbackHost, urlHost } from \"./loopback.ts\";\nimport type { CrawlToolOptions } from \"./tools.ts\";\nimport { crawlHostOptions } from \"./server.ts\";\n\n/*\n * This file serves the tools over the Streamable HTTP transport of MCP. The listener is the\n * crawler's own: it checks the host, the origin and the token, and it serves the status of the\n * daemon. The MCP endpoint is the HTTP face of the shared MCP host, mounted at /mcp: the face keeps\n * the sessions and the tasks.\n *\n * - Each request needs the bearer token. The check compares the SHA-256 digests of the two tokens\n *   in constant time.\n * - The server binds to a loopback host by default. A different host needs `allowRemote`.\n * - A server on a loopback host accepts only a loopback name in the `Host` header, and an `Origin`\n *   header must name the same host. Thus a web page cannot reach the server by DNS rebinding.\n */\n\n/** These are the defaults of the MCP server over HTTP. */\nexport const MCP_HTTP_DEFAULTS = {\n  /** The port of the server. */\n  port: 9307,\n  /** The host of the server: the IPv4 loopback address. */\n  host: \"127.0.0.1\",\n  /** The path of the MCP endpoint. */\n  path: \"/mcp\",\n  /** The path of the status of the daemon. */\n  statusPath: \"/status\",\n  /** The most sessions at one time. A new session above the limit closes the session that was idle longest. */\n  maxSessions: 32,\n  /** The largest body of a request that starts a session: 4 MiB. */\n  maxBodyBytes: 4_194_304,\n  /** The shortest token: 16 characters. */\n  minTokenLength: 16,\n} as const;\n\n/** These options serve MCP over HTTP. */\nexport interface McpHttpOptions {\n  /** This value is the port. The default is 9307. The value 0 takes a free port. */\n  readonly port?: number | undefined;\n  /** This value is the host to bind. The default is `127.0.0.1`. A host that is not loopback needs `allowRemote`. */\n  readonly host?: string | undefined;\n  /** The value true permits a host that is not loopback. Then other computers can try the token. */\n  readonly allowRemote?: boolean | undefined;\n  /** This value is the bearer token. The default reads `tokenFile`, or makes a random token. */\n  readonly token?: string | undefined;\n  /** This file holds the bearer token, when `token` is not given. */\n  readonly tokenFile?: string | undefined;\n  /** This function gives the JSON of `GET /status`. Without it, the path gives 404. */\n  readonly status?: (() => unknown) | undefined;\n  /** This store keeps the tasks of each session. The default is a new `McpTaskStore`. */\n  readonly tasks?: McpTaskStore | undefined;\n  /** This function writes the messages of the server. The default writes nothing. */\n  readonly log?: ((line: string) => void) | undefined;\n}\n\n/** This is a server that serves MCP over HTTP. */\nexport interface McpHttpServer {\n  /** This URL is the MCP endpoint, for example `http://127.0.0.1:9307/mcp`. */\n  readonly url: string;\n  readonly host: string;\n  readonly port: number;\n  /** This value is the bearer token. Give it to the client, and keep it secret. */\n  readonly token: string;\n  /** This store keeps the tasks of the long tools. */\n  readonly tasks: McpTaskStore;\n  /** This method counts the open sessions. */\n  sessions(): number;\n  /** This method closes each session, cancels each task and stops the server. A second close does nothing. */\n  close(): Promise<void>;\n}\n\nconst digest = (text: string): Buffer => createHash(\"sha256\").update(text, \"utf8\").digest();\n\n/**\n * This function compares a token with the token of the server in constant time. It compares the\n * SHA-256 digests, so the time does not depend on the length or the text of either token.\n */\nexport const sameToken = (given: string, token: string): boolean =>\n  timingSafeEqual(digest(given), digest(token));\n\n/** This function makes a random token: 32 bytes in base64url, 43 characters. */\nexport const makeToken = (): string => randomBytes(32).toString(\"base64url\");\n\n/** A token is printable ASCII with no space, so a header holds it. */\nconst TOKEN = /^[\\x21-\\x7e]+$/;\n\n/** This function checks a token. A token that is short or that a header cannot hold throws an error. */\nexport const checkToken = (token: string, source: string): string => {\n  const trimmed = token.trim();\n  if (!TOKEN.test(trimmed) || trimmed.length < MCP_HTTP_DEFAULTS.minTokenLength) {\n    throw new Error(\n      `${source}: the token must be ${String(MCP_HTTP_DEFAULTS.minTokenLength)} or more printable characters with no space`,\n    );\n  }\n  return trimmed;\n};\n\n/** This function reads a token file. It gives a warning when other users can read the file. */\nconst tokenOf = (options: McpHttpOptions, log: (line: string) => void): string => {\n  if (options.token !== undefined) return checkToken(options.token, \"token\");\n  if (options.tokenFile === undefined) return makeToken();\n  let text: string;\n  try {\n    text = readFileSync(options.tokenFile, \"utf8\");\n  } catch (error) {\n    const code = (error as { code?: unknown }).code;\n    throw new Error(\n      `the token file ${options.tokenFile} does not open (${typeof code === \"string\" ? code : \"error\"})`,\n      { cause: error },\n    );\n  }\n  if (process.platform !== \"win32\" && (statSync(options.tokenFile).mode & 0o077) !== 0)\n    log(`polite-crawler: other users can read the token file ${options.tokenFile}: chmod 600 it`);\n  return checkToken(text, `the token file ${options.tokenFile}`);\n};\n\nconst json = (response: ServerResponse, status: number, body: unknown, headers = {}): void => {\n  if (response.headersSent) return;\n  response.writeHead(status, { \"content-type\": \"application/json\", ...headers });\n  response.end(JSON.stringify(body));\n};\n\n/** This function gives the answer of a refusal, as a JSON-RPC error with no id. */\nconst refuse = (response: ServerResponse, status: number, message: string, headers = {}): void =>\n  json(response, status, { jsonrpc: \"2.0\", error: { code: -32000, message }, id: null }, headers);\n\n/**\n * The number of open sessions of the HTTP face. The state of the face gives it when the shared host\n * has the field. Else the count of its Streamable HTTP endpoint gives it.\n */\nconst sessionsOf = (face: McpServerTransport): number => {\n  const state = face.getState() as { sessions?: unknown };\n  if (typeof state.sessions === \"number\") return state.sessions;\n  const endpoint = (face as unknown as { http?: { sessions?: unknown } }).http;\n  return typeof endpoint?.sessions === \"number\" ? endpoint.sessions : 0;\n};\n\n/**\n * This function serves an MCP host over HTTP. `makeHost` makes the host with its HTTP face mounted\n * at `/mcp`, with the task store of the server. Each request needs `Authorization: Bearer <token>`.\n */\nexport const serveMcpHttp = async (\n  makeHost: (face: { readonly token: string; readonly tasks: McpTaskStore }) => McpHost,\n  options: McpHttpOptions = {},\n): Promise<McpHttpServer> => {\n  const log = options.log ?? (() => undefined);\n  const host = options.host ?? MCP_HTTP_DEFAULTS.host;\n  const port = options.port ?? MCP_HTTP_DEFAULTS.port;\n  if (!Number.isInteger(port) || port < 0 || port > 65_535)\n    throw new RangeError(`the port must be an integer from 0 to 65535, not ${String(port)}`);\n  const loopback = isLoopbackHost(host);\n  if (!loopback && options.allowRemote !== true) {\n    throw new Error(\n      `the host ${host} is not a loopback address: give allowRemote (--allow-remote) to serve other computers`,\n    );\n  }\n  const token = tokenOf(options, log);\n  const tasks = options.tasks ?? new McpTaskStore();\n  const mcp = makeHost({ token, tasks });\n  const face = mcp.transports[0];\n  if (face === undefined) throw new Error(\"the MCP host has no HTTP face\");\n  await mcp.start();\n  let bound = port;\n  let closing: Promise<void> | undefined;\n\n  /** A loopback server accepts only a loopback name and its port in `Host`. */\n  const hostAllowed = (header: string | undefined): boolean => {\n    if (!loopback) return true;\n    if (header === undefined) return false;\n    let parsed: URL;\n    try {\n      parsed = new URL(`http://${header}`);\n    } catch {\n      return false;\n    }\n    return isLoopbackHost(parsed.hostname) && (parsed.port === \"\" || Number(parsed.port) === bound);\n  };\n\n  /** An `Origin` header must name the host of the request: a page of a different site gets 403. */\n  const originAllowed = (origin: string | undefined, header: string | undefined): boolean => {\n    if (origin === undefined) return true;\n    try {\n      return new URL(origin).host === header;\n    } catch {\n      return false;\n    }\n  };\n\n  const authorized = (header: string | undefined): boolean => {\n    const match = /^Bearer\\s+(\\S+)\\s*$/i.exec(header ?? \"\");\n    return match !== null && sameToken(match[1]!, token);\n  };\n\n  const handle = (request: IncomingMessage, response: ServerResponse): void => {\n    const hostHeader = request.headers.host;\n    if (!hostAllowed(hostHeader) || !originAllowed(request.headers.origin, hostHeader)) {\n      refuse(response, 403, \"the Host or the Origin header is not of this server\");\n      return;\n    }\n    if (!authorized(request.headers.authorization)) {\n      refuse(response, 401, \"the request needs the bearer token of the server\", {\n        \"www-authenticate\": 'Bearer realm=\"polite-crawler\"',\n      });\n      return;\n    }\n    const path = new URL(request.url ?? \"/\", \"http://localhost\").pathname;\n    if (path === MCP_HTTP_DEFAULTS.statusPath && options.status !== undefined) {\n      if (request.method !== \"GET\") refuse(response, 405, \"the status takes GET\");\n      else json(response, 200, options.status(), { \"cache-control\": \"no-store\" });\n      return;\n    }\n    // the face answers its path: the sessions, the initialize rule, the body limit and the tasks\n    if (!face.handleHttp(request, response)) {\n      refuse(response, 404, `no such path: the MCP endpoint is ${MCP_HTTP_DEFAULTS.path}`);\n    }\n  };\n\n  const http = createServer((request, response) => {\n    try {\n      handle(request, response);\n    } catch (error) {\n      log(\n        `polite-crawler: an MCP request failed: ${error instanceof Error ? error.message : String(error)}`,\n      );\n      if (!response.headersSent) refuse(response, 500, \"the server could not handle the request\");\n      else response.end();\n    }\n  });\n  try {\n    await new Promise<void>((resolve, reject) => {\n      const failed = (error: NodeJS.ErrnoException): void => {\n        reject(\n          error.code === \"EADDRINUSE\"\n            ? new Error(`the port ${String(port)} of ${host} is in use: give a different port`, {\n                cause: error,\n              })\n            : error,\n        );\n      };\n      http.once(\"error\", failed);\n      http.listen(port, host, () => {\n        http.removeListener(\"error\", failed);\n        resolve();\n      });\n    });\n  } catch (error) {\n    await mcp.stop();\n    await tasks.close();\n    throw error;\n  }\n  bound = (http.address() as AddressInfo).port;\n  const url = `http://${urlHost(host)}:${String(bound)}${MCP_HTTP_DEFAULTS.path}`;\n  if (!loopback)\n    log(\n      `polite-crawler: the MCP server listens on ${host}, not on a loopback address: other computers can reach it`,\n    );\n  return {\n    url,\n    host,\n    port: bound,\n    token,\n    tasks,\n    sessions: () => sessionsOf(face),\n    close() {\n      closing ??= (async () => {\n        await tasks.close();\n        await mcp.stop();\n        await new Promise<void>((resolve) => {\n          http.close(() => resolve());\n          http.closeAllConnections();\n        });\n      })();\n      return closing;\n    },\n  };\n};\n\n/** These options serve the tools over HTTP: the options of the tools, and of the HTTP server. */\nexport interface StartMcpHttpOptions extends McpHttpOptions, CrawlToolOptions {}\n\n/**\n * This function serves the tools of a crawler over the Streamable HTTP transport of MCP. The\n * sessions share one MCP host with the nine tools of the server on the standard input and output.\n * The envelope, the elicitation and the opt-ins are the same. `crawl_site` and `crawl_discover` are\n * MCP tasks. Each request needs the bearer token. The function does not close the crawler.\n */\nexport const startMcpHttpServer = (\n  crawler: PoliteCrawler,\n  options: StartMcpHttpOptions = {},\n): Promise<McpHttpServer> => {\n  const { onBehalfOf, allowUserProvided, renderOff, ...http } = options;\n  return serveMcpHttp(\n    ({ tasks }) =>\n      createMcpHost({\n        ...crawlHostOptions(crawler, { onBehalfOf, allowUserProvided, renderOff, tasks }),\n        faces: {\n          // The listener checks the host, the origin and the token of each request, as in 0.1.4.\n          // Thus the face checks no token and no host name: the checks have one owner.\n          http: {\n            mount: true,\n            path: MCP_HTTP_DEFAULTS.path,\n            allowedHosts: \"*\",\n            maxSessions: MCP_HTTP_DEFAULTS.maxSessions,\n            maxBodyBytes: MCP_HTTP_DEFAULTS.maxBodyBytes,\n          },\n        },\n      }),\n    http,\n  );\n};\n\n/**\n * This host serves the tools of the daemon over HTTP. It is the seam between the daemon and its MCP\n * transport: a different MCP host can replace it with `DaemonOptions.mcpHost`.\n */\nexport const mcpHttpHost: DaemonMcpHost = {\n  async start(crawler, options, hooks) {\n    const tasks = new McpTaskStore({ onChange: hooks.onTasks });\n    const served = await startMcpHttpServer(crawler, {\n      ...options,\n      tasks,\n      status: hooks.status,\n      log: hooks.log,\n    });\n    const running = (): DaemonTask[] =>\n      tasks.running().map((task) => ({\n        id: task.id,\n        tool: task.tool,\n        status: task.status,\n        message: task.message,\n        created_at: task.createdAt,\n        updated_at: task.updatedAt,\n      }));\n    return { url: served.url, tasks: running, close: () => served.close() };\n  },\n};\n"],"mappings":";;;;;;;;;;;;;;AAQA,MAAa,kBAAkB,SAA0B;CACvD,MAAM,OAAO,KACV,KAAK,CAAC,CACN,YAAY,CAAC,CACb,QAAQ,cAAc,IAAI;CAC7B,IAAI,SAAS,eAAe,KAAK,SAAS,YAAY,GAAG,OAAO;CAChE,MAAM,SAAS,KAAK,WAAW,SAAS,IAAI,KAAK,MAAM,CAAC,IAAI;CAC5D,IAAI,KAAK,MAAM,MAAM,GAAG,OAAO,OAAO,WAAW,MAAM;CACvD,IAAI,KAAK,IAAI,MAAM,GAAG,OAAO,gCAAgC,KAAK,IAAI;CACtE,OAAO;AACT;;AAGA,MAAa,WAAW,SAA0B,KAAK,IAAI,MAAM,IAAI,IAAI,KAAK,KAAK;;;;ACKnF,MAAa,oBAAoB;;CAE/B,MAAM;;CAEN,MAAM;;CAEN,MAAM;;CAEN,YAAY;;CAEZ,aAAa;;CAEb,cAAc;;CAEd,gBAAgB;AAClB;AAsCA,MAAM,UAAU,SAAyB,WAAW,QAAQ,CAAC,CAAC,OAAO,MAAM,MAAM,CAAC,CAAC,OAAO;;;;;AAM1F,MAAa,aAAa,OAAe,UACvC,gBAAgB,OAAO,KAAK,GAAG,OAAO,KAAK,CAAC;;AAG9C,MAAa,kBAA0B,YAAY,EAAE,CAAC,CAAC,SAAS,WAAW;;AAG3E,MAAM,QAAQ;;AAGd,MAAa,cAAc,OAAe,WAA2B;CACnE,MAAM,UAAU,MAAM,KAAK;CAC3B,IAAI,CAAC,MAAM,KAAK,OAAO,KAAK,QAAQ,SAAS,kBAAkB,gBAC7D,MAAM,IAAI,MACR,GAAG,OAAO,sBAAsB,OAAO,kBAAkB,cAAc,EAAE,4CAC3E;CAEF,OAAO;AACT;;AAGA,MAAM,WAAW,SAAyB,QAAwC;CAChF,IAAI,QAAQ,UAAU,KAAA,GAAW,OAAO,WAAW,QAAQ,OAAO,OAAO;CACzE,IAAI,QAAQ,cAAc,KAAA,GAAW,OAAO,UAAU;CACtD,IAAI;CACJ,IAAI;EACF,OAAO,aAAa,QAAQ,WAAW,MAAM;CAC/C,SAAS,OAAO;EACd,MAAM,OAAQ,MAA6B;EAC3C,MAAM,IAAI,MACR,kBAAkB,QAAQ,UAAU,kBAAkB,OAAO,SAAS,WAAW,OAAO,QAAQ,IAChG,EAAE,OAAO,MAAM,CACjB;CACF;CACA,IAAI,QAAQ,aAAa,YAAY,SAAS,QAAQ,SAAS,CAAC,CAAC,OAAO,QAAW,GACjF,IAAI,uDAAuD,QAAQ,UAAU,eAAe;CAC9F,OAAO,WAAW,MAAM,kBAAkB,QAAQ,WAAW;AAC/D;AAEA,MAAM,QAAQ,UAA0B,QAAgB,MAAe,UAAU,CAAC,MAAY;CAC5F,IAAI,SAAS,aAAa;CAC1B,SAAS,UAAU,QAAQ;EAAE,gBAAgB;EAAoB,GAAG;CAAQ,CAAC;CAC7E,SAAS,IAAI,KAAK,UAAU,IAAI,CAAC;AACnC;;AAGA,MAAM,UAAU,UAA0B,QAAgB,SAAiB,UAAU,CAAC,MACpF,KAAK,UAAU,QAAQ;CAAE,SAAS;CAAO,OAAO;EAAE,MAAM;EAAQ;CAAQ;CAAG,IAAI;AAAK,GAAG,OAAO;;;;;AAMhG,MAAM,cAAc,SAAqC;CACvD,MAAM,QAAQ,KAAK,SAAS;CAC5B,IAAI,OAAO,MAAM,aAAa,UAAU,OAAO,MAAM;CACrD,MAAM,WAAY,KAAsD;CACxE,OAAO,OAAO,UAAU,aAAa,WAAW,SAAS,WAAW;AACtE;;;;;AAMA,MAAa,eAAe,OAC1B,UACA,UAA0B,CAAC,MACA;CAC3B,MAAM,MAAM,QAAQ,cAAc,KAAA;CAClC,MAAM,OAAO,QAAQ,QAAQ,kBAAkB;CAC/C,MAAM,OAAO,QAAQ,QAAQ,kBAAkB;CAC/C,IAAI,CAAC,OAAO,UAAU,IAAI,KAAK,OAAO,KAAK,OAAO,OAChD,MAAM,IAAI,WAAW,oDAAoD,OAAO,IAAI,GAAG;CACzF,MAAM,WAAW,eAAe,IAAI;CACpC,IAAI,CAAC,YAAY,QAAQ,gBAAgB,MACvC,MAAM,IAAI,MACR,YAAY,KAAK,uFACnB;CAEF,MAAM,QAAQ,QAAQ,SAAS,GAAG;CAClC,MAAM,QAAQ,QAAQ,SAAS,IAAIA,eAAa;CAChD,MAAM,MAAM,SAAS;EAAE;EAAO;CAAM,CAAC;CACrC,MAAM,OAAO,IAAI,WAAW;CAC5B,IAAI,SAAS,KAAA,GAAW,MAAM,IAAI,MAAM,+BAA+B;CACvE,MAAM,IAAI,MAAM;CAChB,IAAI,QAAQ;CACZ,IAAI;;CAGJ,MAAM,eAAe,WAAwC;EAC3D,IAAI,CAAC,UAAU,OAAO;EACtB,IAAI,WAAW,KAAA,GAAW,OAAO;EACjC,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,UAAU,QAAQ;EACrC,QAAQ;GACN,OAAO;EACT;EACA,OAAO,eAAe,OAAO,QAAQ,MAAM,OAAO,SAAS,MAAM,OAAO,OAAO,IAAI,MAAM;CAC3F;;CAGA,MAAM,iBAAiB,QAA4B,WAAwC;EACzF,IAAI,WAAW,KAAA,GAAW,OAAO;EACjC,IAAI;GACF,OAAO,IAAI,IAAI,MAAM,CAAC,CAAC,SAAS;EAClC,QAAQ;GACN,OAAO;EACT;CACF;CAEA,MAAM,cAAc,WAAwC;EAC1D,MAAM,QAAQ,uBAAuB,KAAK,UAAU,EAAE;EACtD,OAAO,UAAU,QAAQ,UAAU,MAAM,IAAK,KAAK;CACrD;CAEA,MAAM,UAAU,SAA0B,aAAmC;EAC3E,MAAM,aAAa,QAAQ,QAAQ;EACnC,IAAI,CAAC,YAAY,UAAU,KAAK,CAAC,cAAc,QAAQ,QAAQ,QAAQ,UAAU,GAAG;GAClF,OAAO,UAAU,KAAK,qDAAqD;GAC3E;EACF;EACA,IAAI,CAAC,WAAW,QAAQ,QAAQ,aAAa,GAAG;GAC9C,OAAO,UAAU,KAAK,oDAAoD,EACxE,oBAAoB,kCACtB,CAAC;GACD;EACF;EAEA,IADa,IAAI,IAAI,QAAQ,OAAO,KAAK,kBAAkB,CAAC,CAAC,aAChD,kBAAkB,cAAc,QAAQ,WAAW,KAAA,GAAW;GACzE,IAAI,QAAQ,WAAW,OAAO,OAAO,UAAU,KAAK,sBAAsB;QACrE,KAAK,UAAU,KAAK,QAAQ,OAAO,GAAG,EAAE,iBAAiB,WAAW,CAAC;GAC1E;EACF;EAEA,IAAI,CAAC,KAAK,WAAW,SAAS,QAAQ,GACpC,OAAO,UAAU,KAAK,qCAAqC,kBAAkB,MAAM;CAEvF;CAEA,MAAM,OAAO,cAAc,SAAS,aAAa;EAC/C,IAAI;GACF,OAAO,SAAS,QAAQ;EAC1B,SAAS,OAAO;GACd,IACE,0CAA0C,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GACjG;GACA,IAAI,CAAC,SAAS,aAAa,OAAO,UAAU,KAAK,yCAAyC;QACrF,SAAS,IAAI;EACpB;CACF,CAAC;CACD,IAAI;EACF,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,UAAU,UAAuC;IACrD,OACE,MAAM,SAAS,eACX,IAAI,MAAM,YAAY,OAAO,IAAI,EAAE,MAAM,KAAK,oCAAoC,EAChF,OAAO,MACT,CAAC,IACD,KACN;GACF;GACA,KAAK,KAAK,SAAS,MAAM;GACzB,KAAK,OAAO,MAAM,YAAY;IAC5B,KAAK,eAAe,SAAS,MAAM;IACnC,QAAQ;GACV,CAAC;EACH,CAAC;CACH,SAAS,OAAO;EACd,MAAM,IAAI,KAAK;EACf,MAAM,MAAM,MAAM;EAClB,MAAM;CACR;CACA,QAAS,KAAK,QAAQ,CAAC,CAAiB;CACxC,MAAM,MAAM,UAAU,QAAQ,IAAI,EAAE,GAAG,OAAO,KAAK,IAAI,kBAAkB;CACzE,IAAI,CAAC,UACH,IACE,6CAA6C,KAAK,0DACpD;CACF,OAAO;EACL;EACA;EACA,MAAM;EACN;EACA;EACA,gBAAgB,WAAW,IAAI;EAC/B,QAAQ;GACN,aAAa,YAAY;IACvB,MAAM,MAAM,MAAM;IAClB,MAAM,IAAI,KAAK;IACf,MAAM,IAAI,SAAe,YAAY;KACnC,KAAK,YAAY,QAAQ,CAAC;KAC1B,KAAK,oBAAoB;IAC3B,CAAC;GACH,EAAA,CAAG;GACH,OAAO;EACT;CACF;AACF;;;;;;;AAWA,MAAa,sBACX,SACA,UAA+B,CAAC,MACL;CAC3B,MAAM,EAAE,YAAY,mBAAmB,WAAW,GAAG,SAAS;CAC9D,OAAO,cACJ,EAAE,YACD,cAAc;EACZ,GAAG,iBAAiB,SAAS;GAAE;GAAY;GAAmB;GAAW;EAAM,CAAC;EAChF,OAAO,EAGL,MAAM;GACJ,OAAO;GACP,MAAM,kBAAkB;GACxB,cAAc;GACd,aAAa,kBAAkB;GAC/B,cAAc,kBAAkB;EAClC,EACF;CACF,CAAC,GACH,IACF;AACF;;;;;AAMA,MAAa,cAA6B,EACxC,MAAM,MAAM,SAAS,SAAS,OAAO;CACnC,MAAM,QAAQ,IAAIA,eAAa,EAAE,UAAU,MAAM,QAAQ,CAAC;CAC1D,MAAM,SAAS,MAAM,mBAAmB,SAAS;EAC/C,GAAG;EACH;EACA,QAAQ,MAAM;EACd,KAAK,MAAM;CACb,CAAC;CACD,MAAM,gBACJ,MAAM,QAAQ,CAAC,CAAC,KAAK,UAAU;EAC7B,IAAI,KAAK;EACT,MAAM,KAAK;EACX,QAAQ,KAAK;EACb,SAAS,KAAK;EACd,YAAY,KAAK;EACjB,YAAY,KAAK;CACnB,EAAE;CACJ,OAAO;EAAE,KAAK,OAAO;EAAK,OAAO;EAAS,aAAa,OAAO,MAAM;CAAE;AACxE,EACF"}