polite-crawler 0.1.2 → 0.1.4

package/build/loopback-C_QYowSf.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"loopback-C_QYowSf.mjs","names":[],"sources":["../src/loopback.ts"],"sourcesContent":["import { isIP } from \"node:net\";\n\n/**\n * This function tells if a host name or an address is a loopback host: `localhost`, an address in\n * `127.0.0.0/8`, or `::1`. The MCP server over HTTP binds to a loopback host unless the operator\n * permits a different host. The function imports only Node.js, so the command line uses it before\n * the store opens.\n */\nexport const isLoopbackHost = (host: string): boolean => {\n  const name = host\n    .trim()\n    .toLowerCase()\n    .replace(/^\\[(.*)\\]$/, \"$1\");\n  if (name === \"localhost\" || name.endsWith(\".localhost\")) return true;\n  const mapped = name.startsWith(\"::ffff:\") ? name.slice(7) : name;\n  if (isIP(mapped) === 4) return mapped.startsWith(\"127.\");\n  if (isIP(name) === 6) return /^(?:0{0,4}:){7}0{0,3}1$|^::1$/.test(name);\n  return false;\n};\n\n/** This function gives a host for a URL: an IPv6 address goes in brackets. */\nexport const urlHost = (host: string): string => (isIP(host) === 6 ? `[${host}]` : host);\n"],"mappings":";;;;;;;;AAQA,MAAa,kBAAkB,SAA0B;CACvD,MAAM,OAAO,KACV,KAAK,CAAC,CACN,YAAY,CAAC,CACb,QAAQ,cAAc,IAAI;CAC7B,IAAI,SAAS,eAAe,KAAK,SAAS,YAAY,GAAG,OAAO;CAChE,MAAM,SAAS,KAAK,WAAW,SAAS,IAAI,KAAK,MAAM,CAAC,IAAI;CAC5D,IAAI,KAAK,MAAM,MAAM,GAAG,OAAO,OAAO,WAAW,MAAM;CACvD,IAAI,KAAK,IAAI,MAAM,GAAG,OAAO,gCAAgC,KAAK,IAAI;CACtE,OAAO;AACT;;AAGA,MAAa,WAAW,SAA0B,KAAK,IAAI,MAAM,IAAI,IAAI,KAAK,KAAK"}

package/build/mcp.d.mts

@@ -1,5 +1,141 @@
-import { a as PoliteCrawler, r as OpenOptions } from "./open-Ca1lcWR3.mjs";
+import { E as OpenOptions, O as PoliteCrawler, s as DaemonMcpHost } from "./daemon-wwIWsUmU.mjs";
 import { McpServer, McpServer as McpServer$1 } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { Request, RequestId, Result, Task } from "@modelcontextprotocol/sdk/types.js";
+import { CreateTaskOptions, TaskStore } from "@modelcontextprotocol/sdk/experimental/tasks/interfaces.js";
+import "@modelcontextprotocol/sdk/shared/protocol.js";
+//#region src/mcp-tasks.d.ts
+/** These are the limits of the tasks of one MCP server. */
+export declare const TASK_LIMITS: {
+  /** A task and its result stay for this time after the task starts or ends: 1 hour by default. */
+  readonly ttlMs: 3_600_000;
+  /** The longest time that a client can ask for: 1 day. */
+  readonly maxTtlMs: 86_400_000;
+  /** The time that the server proposes between two polls of the status: 1 second. */
+  readonly pollMs: 1_000;
+  /** The most tasks that run at one time. A new task above the limit is an error. */
+  readonly maxRunning: 8;
+};
+/** This is a task of the server, as the status of the daemon shows it. */
+interface TaskInfo {
+  /** This id is the task id that the client got. */
+  readonly id: string;
+  /** This value is the name of the tool, for example `crawl_site`. */
+  readonly tool: string;
+  /** This value is `working`, `input_required`, `completed`, `failed` or `cancelled`. */
+  readonly status: string;
+  /** This text tells the progress, for example `3 of 10 done`, or null. */
+  readonly message: string | null;
+  readonly created_at: string;
+  readonly updated_at: string;
+}
+/** These options make a task store. */
+interface CrawlTaskStoreOptions {
+  /** This value is the most tasks that run at one time. The default is 8. */
+  readonly maxRunning?: number | undefined;
+  /** This function operates when a task starts, changes its status or ends. */
+  readonly onChange?: (() => void) | undefined;
+}
+/**
+ * This store keeps the tasks of the long tools in memory. It wraps the `InMemoryTaskStore` of the
+ * SDK, which binds each task to its session. It also keeps an abort signal for each task: the
+ * cancel of a task (`tasks/cancel`) aborts the work of its tool. One store serves each session of
+ * one server.
+ */
+export declare class CrawlTaskStore implements TaskStore {
+  private readonly options;
+  private readonly store;
+  private readonly jobs;
+  private readonly maxRunning;
+  private closed;
+  constructor(options?: CrawlTaskStoreOptions);
+  private changed;
+  /** This method removes the jobs whose tasks the SDK store removed at the end of their time. */
+  private prune;
+  createTask(params: CreateTaskOptions, requestId: RequestId, request: Request, sessionId?: string): Promise<Task>;
+  getTask(taskId: string, sessionId?: string): Promise<Task | null>;
+  storeTaskResult(taskId: string, status: "completed" | "failed", result: Result, sessionId?: string): Promise<void>;
+  getTaskResult(taskId: string, sessionId?: string): Promise<Result>;
+  updateTaskStatus(taskId: string, status: Task["status"], statusMessage?: string, sessionId?: string): Promise<void>;
+  listTasks(cursor?: string, sessionId?: string): Promise<{
+    tasks: Task[];
+    nextCursor?: string;
+  }>;
+  /** This method gives the abort signal of a task. A cancel of the task aborts it. */
+  signal(taskId: string): AbortSignal;
+  /** This method gives the reason that a task did not start, or undefined for a task that started. */
+  refusal(taskId: string): string | undefined;
+  /** This method tells if the client asked for the task. False is the call of a client with no tasks. */
+  augmented(taskId: string): boolean;
+  /** This method gives the tasks that did not end, the oldest first. */
+  running(): TaskInfo[];
+  /** This method gives the sessions that have a task in progress. */
+  busySessions(): Set<string>;
+  /** This method cancels the tasks of a session that closed. Their work stops. */
+  cancelSession(sessionId: string, message?: string): Promise<void>;
+  /** This method cancels each task that runs, and removes all the tasks. A second close does nothing. */
+  close(): Promise<void>;
+}
+//#endregion
+//#region src/mcp-http.d.ts
+/** These are the defaults of the MCP server over HTTP. */
+export declare const MCP_HTTP_DEFAULTS: {
+  /** The port of the server. */
+  readonly port: 9307;
+  /** The host of the server: the IPv4 loopback address. */
+  readonly host: "127.0.0.1";
+  /** The path of the MCP endpoint. */
+  readonly path: "/mcp";
+  /** The path of the status of the daemon. */
+  readonly statusPath: "/status";
+  /** The most sessions at one time. A new session above the limit closes the session that was idle longest. */
+  readonly maxSessions: 32;
+  /** The largest body of a request that starts a session: 4 MiB. */
+  readonly maxBodyBytes: 4_194_304;
+  /** The shortest token: 16 characters. */
+  readonly minTokenLength: 16;
+};
+/** These options serve MCP over HTTP. */
+interface McpHttpOptions {
+  /** This value is the port. The default is 9307. The value 0 takes a free port. */
+  readonly port?: number | undefined;
+  /** This value is the host to bind. The default is `127.0.0.1`. A host that is not loopback needs `allowRemote`. */
+  readonly host?: string | undefined;
+  /** The value true permits a host that is not loopback. Then other computers can try the token. */
+  readonly allowRemote?: boolean | undefined;
+  /** This value is the bearer token. The default reads `tokenFile`, or makes a random token. */
+  readonly token?: string | undefined;
+  /** This file holds the bearer token, when `token` is not given. */
+  readonly tokenFile?: string | undefined;
+  /** This function gives the JSON of `GET /status`. Without it, the path gives 404. */
+  readonly status?: (() => unknown) | undefined;
+  /** This store keeps the tasks of each session. The default is a new `CrawlTaskStore`. */
+  readonly tasks?: CrawlTaskStore | undefined;
+  /** This function writes the messages of the server. The default writes nothing. */
+  readonly log?: ((line: string) => void) | undefined;
+}
+/** This is a server that serves MCP over HTTP. */
+interface McpHttpServer {
+  /** This URL is the MCP endpoint, for example `http://127.0.0.1:9307/mcp`. */
+  readonly url: string;
+  readonly host: string;
+  readonly port: number;
+  /** This value is the bearer token. Give it to the client, and keep it secret. */
+  readonly token: string;
+  /** This store keeps the tasks of the long tools. */
+  readonly tasks: CrawlTaskStore;
+  /** This method counts the open sessions. */
+  sessions(): number;
+  /** This method closes each session, cancels each task and stops the server. A second close does nothing. */
+  close(): Promise<void>;
+}
+/**
+ * This function compares a token with the token of the server in constant time. It compares the
+ * SHA-256 digests, so the time does not depend on the length or the text of either token.
+ */
+export declare const sameToken: (given: string, token: string) => boolean;
+/** This function makes a random token: 32 bytes in base64url, 43 characters. */
+export declare const makeToken: () => string;
+//#endregion
 //#region src/mcp.d.ts
 /** These options change the tools of the MCP server. */
 export interface CrawlServerOptions {
@@ -16,6 +152,12 @@ export interface CrawlServerOptions {
   readonly allowUserProvided?: boolean | undefined;
   /** This text tells why the server renders no page. The result of a page that needs JavaScript shows it. */
   readonly renderOff?: string | undefined;
+  /**
+   * This store makes the long tools (`crawl_site` and `crawl_discover`) MCP tasks. A client that
+   * asks for a task gets a task id at once. A client with no tasks gets the result in the call. The
+   * server over HTTP gives the store. Without it, the long tools are normal tools.
+   */
+  readonly tasks?: CrawlTaskStore | undefined;
 }
 /** These options start the MCP server on the standard input and output. */
 export interface StartMcpOptions extends OpenOptions, CrawlServerOptions {
@@ -38,6 +180,20 @@ export declare const createMcpServer: (crawler: PoliteCrawler, options?: CrawlSe
  * The server stops when its standard input closes, or at `SIGINT` or `SIGTERM`.
  */
 export declare const startMcpServer: (options: StartMcpOptions) => Promise<void>;
+/** These options serve the tools over HTTP: the options of the tools, and of the HTTP server. */
+export interface StartMcpHttpOptions extends McpHttpOptions, Omit<CrawlServerOptions, "tasks"> {}
+/**
+ * This function serves the tools of a crawler over the Streamable HTTP transport of MCP. Each
+ * session gets an MCP server with the nine tools of the server on the standard input and output.
+ * The envelope, the elicitation and the opt-ins are the same. `crawl_site` and `crawl_discover` are MCP tasks.
+ * Each request needs the bearer token. The function does not close the crawler.
+ */
+export declare const startMcpHttpServer: (crawler: PoliteCrawler, options?: StartMcpHttpOptions) => Promise<McpHttpServer>;
+/**
+ * This host serves the tools of the daemon over HTTP. It is the seam between the daemon and its MCP
+ * transport: a different MCP host can replace it with `DaemonOptions.mcpHost`.
+ */
+export declare const mcpHttpHost: DaemonMcpHost;
 //#endregion
-export type { McpServer };
+export type { CrawlTaskStoreOptions, McpHttpOptions, McpHttpServer, McpServer, TaskInfo };
 //# sourceMappingURL=mcp.d.mts.map