pipework 0.212.0 → 0.243.0

package/dist/infra/crawl/handle.d.ts

@@ -0,0 +1,149 @@
+import type { Handler } from '../../request/di/types.js';
+import type { Manifold } from '../../core/pipework.js';
+import type { Queue } from '../../async/jobs/queue.js';
+import type { FetchTarget } from '../fetch/handle.js';
+import type { Storage } from '../storage/handle.js';
+import type { Socket } from 'node:net';
+export type CrawlScope = 'host' | {
+    readonly domain: string;
+};
+export type CrawlState = 'awaiting_approval' | 'running' | 'done' | 'stopped' | 'failed';
+export type CrawlStoppedBy = 'pages' | 'depth' | 'bytes' | 'wall' | 'robots_unreachable';
+/** Limits of one crawl; each is optional and the default is in the field's description. */
+export interface CrawlLimits {
+    /** Requests in flight at once to one host. Default 1. */
+    readonly perHostConcurrency?: number;
+    /** Least milliseconds between requests to a host; the gap used is the larger of this and the host's `Crawl-delay`. Default 1000. */
+    readonly minGapMs?: number;
+    /** Most pages and files requested (robots.txt not counted). Default 1000. */
+    readonly maxPages?: number;
+    /** Most links deep from the seed. Default 10. */
+    readonly maxDepth?: number;
+    /** Most body bytes fetched in all. Default 1 GiB. */
+    readonly maxBytes?: number;
+    /** Most bytes of one response; a larger one is refused as `too_large`. Default 25 MiB. */
+    readonly maxBytesPerResponse?: number;
+    /** Most milliseconds the crawl runs, counted from its start. Default one hour. */
+    readonly maxWallMs?: number;
+    /** Time limit of one request. Default 30 s. */
+    readonly requestTimeoutMs?: number;
+    /** Most redirects followed in a chain. Default 5. */
+    readonly maxRedirects?: number;
+    /** Time limit of one `onPage` call. Default 5000. */
+    readonly pageHookMs?: number;
+}
+export type ResolvedCrawlLimits = Required<CrawlLimits>;
+export interface CrawlPage {
+    readonly url: string;
+    /** The page's html, already limited to `maxBytesPerResponse`. */
+    readonly html: string;
+    /** Aborted when the hook has overrun `pageHookMs`. */
+    readonly signal: AbortSignal;
+}
+export interface CrawlConfig {
+    /** The app's manifold; its Blueprint must have a `crawl` block. */
+    readonly instance: Manifold;
+    /** Names the site and its job type (`pipework.crawl.<name>`): letters, digits, `_`, `.` and `-`, starting with a letter. */
+    readonly name: string;
+    /** The queue the crawl job runs on; its worker surface must run `site.job`. */
+    readonly queue: Pick<Queue, 'enqueue'>;
+    /** Which resources are listed. In-scope `text/html` is followed for links whatever this says; any other type this refuses is not read. */
+    readonly accept: (resource: {
+        url: string;
+        contentType: string;
+    }) => boolean;
+    /** Called with each followed page's html. A hook that throws or overruns `pageHookMs` is recorded against the URL and the crawl goes on. */
+    readonly onPage?: (page: CrawlPage) => Promise<void> | void;
+    /** When given, each listed resource's bytes are put here (the job's tenant) and the row records the sha256 and file id. */
+    readonly store?: Storage;
+    /** Sent on every request; its product token (the leading letters) picks the robots.txt group. */
+    readonly userAgent: string;
+    /** For tests: replaces DNS and the socket, as in `untrustedFetch.get`. */
+    readonly network?: {
+        readonly resolve?: (hostname: string) => Promise<string[]>;
+        readonly connect?: (target: FetchTarget) => Socket;
+    };
+    /** Record the full URL in each request's trace step. Default true: crawled links are public. */
+    readonly traceUrls?: boolean;
+    /** Database holding the crawl tables; default the default database. */
+    readonly database?: string;
+    /** Attempts the queue gives the job: a job that dies or throws is retried and carries on from the queued URLs. Default 5. */
+    readonly maxAttempts?: number;
+    /** First wait before a robots.txt retry, doubling each time. Default 2000. */
+    readonly robotsRetryMs?: number;
+}
+export interface CrawlStartOptions {
+    readonly seed: string;
+    /** `'host'` (default): the seed's exact host. `{ domain: 'example.gov.uk' }`: that host and every host under it. */
+    readonly scope?: CrawlScope;
+    readonly limits?: CrawlLimits;
+}
+export interface CrawlCounts {
+    readonly queued: number;
+    readonly fetching: number;
+    readonly done: number;
+    readonly refused: number;
+    readonly listed: number;
+    readonly unchanged: number;
+    /** Body bytes fetched. */
+    readonly bytes: number;
+}
+export interface CrawlStatus {
+    readonly state: CrawlState;
+    /** What the crawl stays inside. A `'host'` scope moves to the new host when the seed's own redirect chain leaves the seed's host (example.gov.uk to www.example.gov.uk). */
+    readonly scope: {
+        readonly host: string;
+    } | {
+        readonly domain: string;
+    };
+    /** Present when `state` is `stopped`. */
+    readonly stoppedBy?: CrawlStoppedBy;
+    /** Present when `state` is `failed`. */
+    readonly error?: string;
+    readonly counts: CrawlCounts;
+}
+export interface CrawlResource {
+    readonly url: string;
+    /** The page that linked to it; null for the seed. */
+    readonly from: string | null;
+    readonly status: number;
+    readonly contentType: string | null;
+    readonly bytes: number;
+    readonly etag: string | null;
+    readonly lastModified: string | null;
+    readonly sha256: string | null;
+    readonly fileId: string | null;
+    /** True when the server answered a conditional request with 304; the other fields are those of the earlier crawl. */
+    readonly unchanged: boolean;
+}
+export interface CrawlRefusal {
+    readonly url: string;
+    readonly from: string | null;
+    /** `scope`, `robots`, `depth`, `page_hook`, or the `untrustedFetch.get` reason (`address`, `too_large`, `rejected`, `status`, ...). */
+    readonly reason: string;
+    readonly message: string;
+}
+export interface CrawlSite {
+    /** Add to a worker surface's `jobs`. */
+    readonly job: Handler<Record<string, unknown>, unknown>;
+    /** Job type of `job`. */
+    readonly jobType: string;
+    /**
+     * Records a crawl and, unless the Blueprint's `crawl.requireApproval` is true, queues its job. Needs a Flow. Throws
+     * `CrawlSeedError` for a seed that is not http(s) or not inside its own scope. One audit record: who, seed, scope, limits.
+     */
+    start(options: CrawlStartOptions): Promise<{
+        id: string;
+    }>;
+    /** Queues the job of a crawl that is `awaiting_approval`; one audit record. Throws `ConflictError` for a crawl in another state. */
+    approve(id: string): Promise<void>;
+    status(id: string): Promise<CrawlStatus>;
+    /** The listed resources, in the order they were found. */
+    resources(id: string): Promise<CrawlResource[]>;
+    /** Every URL the crawl declined to fetch or could not, and every `onPage` failure, with the reason. */
+    refusals(id: string): Promise<CrawlRefusal[]>;
+}
+export declare const DEFAULT_LIMITS: ResolvedCrawlLimits;
+/** Creates a site crawler; see `crawl.create`. */
+export declare function createCrawl(config: CrawlConfig): CrawlSite;
+//# sourceMappingURL=handle.d.ts.map

package/dist/infra/crawl/handle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"handle.d.ts","sourceRoot":"","sources":["../../../src/infra/crawl/handle.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAA;AAOxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACtD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAA;AAEtD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAA;AAMtC,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAA;AAC7D,MAAM,MAAM,UAAU,GAAG,mBAAmB,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAA;AACxF,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,oBAAoB,CAAA;AAExF,2FAA2F;AAC3F,MAAM,WAAW,WAAW;IAC1B,yDAAyD;IACzD,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;IACpC,oIAAoI;IACpI,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,6EAA6E;IAC7E,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,iDAAiD;IACjD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,qDAAqD;IACrD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,0FAA0F;IAC1F,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAA;IACrC,kFAAkF;IAClF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,+CAA+C;IAC/C,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,qDAAqD;IACrD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B,qDAAqD;IACrD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED,MAAM,MAAM,mBAAmB,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAA;AAEvD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,iEAAiE;IACjE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;CAC7B;AAED,MAAM,WAAW,WAAW;IAC1B,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,4HAA4H;IAC5H,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;IACtC,0IAA0I;IAC1I,QAAQ,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAA;IAC5E,4IAA4I;IAC5I,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IAC3D,2HAA2H;IAC3H,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAA;IACxB,iGAAiG;IACjG,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,CAAC,EAAE;QACjB,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QAC1D,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,MAAM,CAAA;KACnD,CAAA;IACD,gGAAgG;IAChG,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAA;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,6HAA6H;IAC7H,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAChC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,oHAAoH;IACpH,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAA;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,0BAA0B;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAA;IAC1B,4KAA4K;IAC5K,QAAQ,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;IACvE,yCAAyC;IACzC,QAAQ,CAAC,SAAS,CAAC,EAAE,cAAc,CAAA;IACnC,wCAAwC;IACxC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,qHAAqH;IACrH,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;CAC5B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,uIAAuI;IACvI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,SAAS;IACxB,wCAAwC;IACxC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAA;IACvD,yBAAyB;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB;;;OAGG;IACH,KAAK,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC1D,oIAAoI;IACpI,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAClC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;IACxC,0DAA0D;IAC1D,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;IAC/C,uGAAuG;IACvG,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;CAC9C;AAED,eAAO,MAAM,cAAc,EAAE,mBAG5B,CAAA;AAyED,kDAAkD;AAClD,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,CAqc1D"}