pipework 0.212.0 → 0.242.0

package/dist/infra/crawl/handle.js

@@ -0,0 +1,550 @@
+import { Readable } from 'node:stream';
+import { sql } from '../../data/db/sql.js';
+import { pipeSystem } from '../../data/db/db.js';
+import { fitting } from '../../request/di/index.js';
+import { getContext } from '../../request/context/store.js';
+import { resolveActorId } from '../../data/write/audit/actor.js';
+import { schema } from '../../data/schema/namespace.js';
+import { __flowStep } from '../../trace/runtime/flow-step.js';
+import { getBaseLogger } from '../../core/logging/index.js';
+import { ConflictError, NotFoundError } from '../../core/errors/index.js';
+import { untrustedGet } from '../fetch/handle.js';
+import { auditCrawl } from './audit.js';
+import { CrawlNotEnabledError, CrawlSeedError } from './errors.js';
+import { extractLinks } from './links.js';
+import { ALLOW_ALL, DISALLOW_ALL, parseRobots } from './robots.js';
+export const DEFAULT_LIMITS = {
+    perHostConcurrency: 1, minGapMs: 1000, maxPages: 1000, maxDepth: 10, maxBytes: 1024 ** 3, maxBytesPerResponse: 25 * 1024 * 1024,
+    maxWallMs: 3_600_000, requestTimeoutMs: 30_000, maxRedirects: 5, pageHookMs: 5000,
+};
+const NAME_RE = /^[A-Za-z][A-Za-z0-9_.-]*$/;
+const ROBOTS_MAX_BYTES = 500 * 1024;
+const ROBOTS_TRIES = 4;
+const barred = (reason, message) => ({ ...DISALLOW_ALL, refusal: { reason, message } });
+const HTML_TYPES = new Set(['text/html', 'application/xhtml+xml']);
+const num = (v) => Number(v ?? 0);
+const str = (v) => (v === null || v === undefined ? null : String(v));
+const baseType = (contentType) => (contentType ?? '').split(';')[0].trim().toLowerCase();
+const describeError = (err) => (err instanceof Error ? err.message : String(err)).slice(0, 500);
+function sleep(ms, signal) {
+    return new Promise(resolve => {
+        if (ms <= 0 || signal.aborted) {
+            resolve();
+            return;
+        }
+        const done = () => { clearTimeout(timer); signal.removeEventListener('abort', done); resolve(); };
+        const timer = setTimeout(done, ms);
+        signal.addEventListener('abort', done, { once: true });
+    });
+}
+function inScope(scope, hostname) {
+    if (scope.kind === 'host')
+        return hostname === scope.host;
+    return hostname === scope.domain || hostname.endsWith('.' + scope.domain);
+}
+function resolveLimits(given) {
+    const out = { ...DEFAULT_LIMITS };
+    for (const [key, value] of Object.entries(given ?? {})) {
+        if (!(key in DEFAULT_LIMITS))
+            throw new CrawlSeedError(`unknown limit "${key}" (limits: ${Object.keys(DEFAULT_LIMITS).join(', ')}).`);
+        if (value === undefined)
+            continue;
+        const min = key === 'minGapMs' || key === 'maxRedirects' || key === 'maxDepth' ? 0 : 1;
+        if (typeof value !== 'number' || !Number.isFinite(value) || value < min)
+            throw new CrawlSeedError(`limit ${key} must be a number of at least ${String(min)}, got ${String(value)}.`);
+        out[key] = value;
+    }
+    return out;
+}
+function parseSeed(seed, scope) {
+    let url;
+    try {
+        url = new URL(seed);
+    }
+    catch {
+        throw new CrawlSeedError(`the seed "${seed}" is not a URL.`);
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:')
+        throw new CrawlSeedError(`the seed must be an http or https URL, got ${url.protocol}`);
+    url.hash = '';
+    let stored;
+    if (scope === undefined || scope === 'host') {
+        stored = { kind: 'host', host: url.hostname };
+    }
+    else if (typeof scope === 'object' && typeof scope.domain === 'string' && scope.domain.trim() !== '') {
+        stored = { kind: 'domain', domain: scope.domain.trim().toLowerCase().replace(/^\.+|\.+$/g, '') };
+    }
+    else {
+        throw new CrawlSeedError(`scope must be 'host' or { domain: 'example.gov.uk' }.`);
+    }
+    if (!inScope(stored, url.hostname)) {
+        throw new CrawlSeedError(`the seed's host ${url.hostname} is not inside its scope (${stored.kind === 'host' ? stored.host : stored.domain}).`);
+    }
+    return { url, stored };
+}
+/** Creates a site crawler; see `crawl.create`. */
+export function createCrawl(config) {
+    const blueprint = config.instance.config.raw.crawl;
+    if (blueprint === undefined)
+        throw new CrawlNotEnabledError();
+    if (typeof config.name !== 'string' || !NAME_RE.test(config.name)) {
+        throw new Error(`[pipework] crawl.create: name "${String(config.name)}" must start with a letter and use letters, digits, "_", "." or "-".\n`);
+    }
+    if (typeof config.userAgent !== 'string' || config.userAgent.trim() === '')
+        throw new Error('[pipework] crawl.create: userAgent is required: sites read it to decide what you may fetch.\n');
+    const requireApproval = blueprint.requireApproval === true;
+    const jobType = `pipework.crawl.${config.name}`;
+    const database = config.database;
+    const db = () => pipeSystem(database);
+    const robotsRetryMs = config.robotsRetryMs ?? 2000;
+    const urlIsSafeToTrace = config.traceUrls !== false;
+    const enqueue = async (id) => {
+        await config.queue.enqueue(db(), { type: jobType, payload: { crawlId: id }, liveKey: id, continueFlow: true, maxAttempts: config.maxAttempts ?? 5 });
+    };
+    const startStep = __flowStep({ ref: 'pipework/crawl:start', capture: [null] }, async (options) => {
+        const { url, stored } = parseSeed(options.seed, options.scope);
+        const limits = resolveLimits(options.limits);
+        const flow = getContext();
+        const actor = flow === undefined ? null : resolveActorId(flow);
+        const state = requireApproval ? 'awaiting_approval' : 'running';
+        const rows = await db().execute(sql `
+      INSERT INTO pipework_crawl (site, seed, scope, limits, state, tenant_id, started_by, run_started_at)
+      VALUES (${config.name}, ${url.href}, ${JSON.stringify(stored)}::jsonb, ${JSON.stringify(limits)}::jsonb, ${state},
+              ${flow?.tenant ?? null}, ${actor}, ${requireApproval ? null : sql `now()`})
+      RETURNING id::text AS id`);
+        const id = rows[0]['id'];
+        await db().execute(sql `INSERT INTO pipework_crawl_url (crawl_id, url, depth, seed_chain) VALUES (${id}::uuid, ${url.href}, 0, true)`);
+        await auditCrawl('start', id, { site: config.name, seed: url.href, scope: stored, limits, awaitingApproval: requireApproval }, database);
+        if (!requireApproval)
+            await enqueue(id);
+        return { id };
+    });
+    const approveStep = __flowStep({ ref: 'pipework/crawl:approve', capture: [null] }, async (id) => {
+        await requireCrawl(id);
+        const flow = getContext();
+        const actor = flow === undefined ? null : resolveActorId(flow);
+        const rows = await db().execute(sql `
+      UPDATE pipework_crawl SET state = 'running', approved_by = ${actor}, run_started_at = now()
+       WHERE id = ${id}::uuid AND site = ${config.name} AND state = 'awaiting_approval' RETURNING seed`);
+        if (rows.length === 0) {
+            const found = await db().execute(sql `SELECT state FROM pipework_crawl WHERE id = ${id}::uuid AND site = ${config.name}`);
+            if (found.length === 0)
+                throw new NotFoundError(`Crawl ${id}`);
+            throw new ConflictError(`Crawl ${id} is not awaiting approval (it is ${String(found[0]['state'])}).`);
+        }
+        await auditCrawl('approve', id, { site: config.name, seed: rows[0]['seed'] }, database);
+        await enqueue(id);
+    });
+    async function loadCrawl(id) {
+        const rows = await db().execute(sql `
+      SELECT state, limits, scope, seed, tenant_id, stopped_by, error, bytes::float8 AS bytes,
+             (extract(epoch from run_started_at) * 1000)::float8 AS run_started_ms
+        FROM pipework_crawl WHERE id = ${id}::uuid AND site = ${config.name}`);
+        const r = rows[0];
+        if (r === undefined)
+            return null;
+        return {
+            state: r['state'], limits: r['limits'], scope: r['scope'], seed: r['seed'],
+            runStartedAt: num(r['run_started_ms']), tenantId: str(r['tenant_id']),
+            stoppedBy: str(r['stopped_by']), error: str(r['error']), bytes: num(r['bytes']),
+        };
+    }
+    /** The crawl, or NotFoundError: also when it belongs to another tenant than the Flow's. */
+    const requireCrawl = async (id) => {
+        const crawl = await loadCrawl(id);
+        const tenant = getContext()?.tenant ?? null;
+        if (crawl === null || (crawl.tenantId !== null && crawl.tenantId !== tenant))
+            throw new NotFoundError(`Crawl ${id}`);
+        return crawl;
+    };
+    const status = async (id) => {
+        const crawl = await requireCrawl(id);
+        const counted = await db().execute(sql `
+      SELECT count(*) FILTER (WHERE state = 'queued')::int AS queued, count(*) FILTER (WHERE state = 'fetching')::int AS fetching,
+             count(*) FILTER (WHERE state = 'done')::int AS done, count(*) FILTER (WHERE state = 'refused')::int AS refused
+        FROM pipework_crawl_url WHERE crawl_id = ${id}::uuid`);
+        const listed = await db().execute(sql `
+      SELECT count(*)::int AS listed, count(*) FILTER (WHERE unchanged)::int AS unchanged FROM pipework_crawl_resource WHERE crawl_id = ${id}::uuid`);
+        const c = counted[0];
+        return {
+            state: crawl.state,
+            scope: crawl.scope.kind === 'host' ? { host: crawl.scope.host } : { domain: crawl.scope.domain },
+            ...(crawl.stoppedBy !== null ? { stoppedBy: crawl.stoppedBy } : {}),
+            ...(crawl.error !== null ? { error: crawl.error } : {}),
+            counts: {
+                queued: num(c['queued']), fetching: num(c['fetching']), done: num(c['done']), refused: num(c['refused']),
+                listed: num(listed[0]['listed']), unchanged: num(listed[0]['unchanged']), bytes: crawl.bytes,
+            },
+        };
+    };
+    const resources = async (id) => {
+        await requireCrawl(id);
+        const rows = await db().execute(sql `
+      SELECT r.url, r.from_url, r.status, r.content_type, r.bytes::float8 AS bytes, r.etag, r.last_modified, r.sha256, r.file_id, r.unchanged
+        FROM pipework_crawl_resource r JOIN pipework_crawl c ON c.id = r.crawl_id
+       WHERE r.crawl_id = ${id}::uuid AND c.site = ${config.name} ORDER BY r.created_at, r.id`);
+        return rows.map(r => ({
+            url: r['url'], from: str(r['from_url']), status: num(r['status']), contentType: str(r['content_type']), bytes: num(r['bytes']),
+            etag: str(r['etag']), lastModified: str(r['last_modified']), sha256: str(r['sha256']), fileId: str(r['file_id']), unchanged: r['unchanged'] === true,
+        }));
+    };
+    const refusals = async (id) => {
+        await requireCrawl(id);
+        const rows = await db().execute(sql `
+      SELECT u.url, u.from_url, u.reason, u.message
+        FROM pipework_crawl_url u JOIN pipework_crawl c ON c.id = u.crawl_id
+       WHERE u.crawl_id = ${id}::uuid AND c.site = ${config.name} AND (u.state = 'refused' OR u.reason = 'page_hook') ORDER BY u.seq`);
+        return rows.map(r => ({ url: r['url'], from: str(r['from_url']), reason: str(r['reason']) ?? '', message: str(r['message']) ?? '' }));
+    };
+    // ---- the job ----
+    async function run(crawlId, signal) {
+        const crawl = await loadCrawl(crawlId);
+        if (crawl === null || crawl.state !== 'running')
+            return;
+        const { limits } = crawl;
+        let scope = crawl.scope;
+        const state = { stop: null, inFlight: 0, last: new Map(), hostInit: new Map(), robots: new Map() };
+        // Only one job runs a crawl, so a row still 'fetching' is the one that was in flight when the last attempt died.
+        await db().execute(sql `UPDATE pipework_crawl_url SET state = 'queued' WHERE crawl_id = ${crawlId}::uuid AND state = 'fetching'`);
+        const hostRow = async (host) => {
+            await db().execute(sql `INSERT INTO pipework_crawl_host (crawl_id, host) VALUES (${crawlId}::uuid, ${host}) ON CONFLICT (crawl_id, host) DO NOTHING`);
+            const r = (await db().execute(sql `
+        SELECT robots_state, robots_text, robots_attempts, last_request_at_ms::float8 AS last_ms FROM pipework_crawl_host WHERE crawl_id = ${crawlId}::uuid AND host = ${host}`))[0];
+            return { robotsState: str(r['robots_state']), robotsText: str(r['robots_text']), attempts: num(r['robots_attempts']), lastMs: num(r['last_ms']) };
+        };
+        const gapFor = (rules) => Math.max(limits.minGapMs, (rules?.crawlDelaySeconds ?? 0) * 1000);
+        /** Waits for this host's turn, then returns a function that records the request's end. A request starts at least `gap` after the last one ended (started, with several in flight). */
+        const takeTurn = async (host, gap) => {
+            let init = state.hostInit.get(host);
+            if (init === undefined) {
+                init = hostRow(host).then(row => { state.last.set(host, row.lastMs); });
+                state.hostInit.set(host, init);
+            }
+            await init;
+            const start = Math.max(Date.now(), state.last.get(host) + gap);
+            if (limits.perHostConcurrency > 1)
+                state.last.set(host, start);
+            await sleep(start - Date.now(), signal);
+            return async () => {
+                const end = Date.now();
+                state.last.set(host, Math.max(state.last.get(host), end));
+                await db().execute(sql `UPDATE pipework_crawl_host SET last_request_at_ms = ${end} WHERE crawl_id = ${crawlId}::uuid AND host = ${host}`);
+            };
+        };
+        const ua = { 'user-agent': config.userAgent };
+        const network = {
+            ...(config.network?.resolve !== undefined ? { resolve: config.network.resolve } : {}),
+            ...(config.network?.connect !== undefined ? { connect: config.network.connect } : {}),
+        };
+        /** The host's robots rules, fetched once before any other request to it; null when it stayed unreachable (the crawl stops). */
+        const robotsFor = (host, protocol) => {
+            let pending = state.robots.get(host);
+            if (pending === undefined) {
+                pending = loadRobots(host, protocol);
+                state.robots.set(host, pending);
+            }
+            return pending;
+        };
+        const loadRobots = async (host, protocol) => {
+            const row = await hostRow(host);
+            if (row.robotsState === 'ok')
+                return parseRobots(row.robotsText ?? '', config.userAgent);
+            if (row.robotsState === 'allow_all')
+                return ALLOW_ALL;
+            if (row.robotsState === 'disallow_all') {
+                const kept = JSON.parse(row.robotsText ?? 'null');
+                return kept === null ? DISALLOW_ALL : barred(kept.reason, kept.message);
+            }
+            const settle = async (robotsState, text) => {
+                await db().execute(sql `UPDATE pipework_crawl_host SET robots_state = ${robotsState}, robots_text = ${text} WHERE crawl_id = ${crawlId}::uuid AND host = ${host}`);
+            };
+            for (let attempt = row.attempts; attempt < ROBOTS_TRIES; attempt++) {
+                if (signal.aborted)
+                    return null;
+                if (attempt > 0)
+                    await sleep(robotsRetryMs * 2 ** (attempt - 1), signal);
+                const done = await takeTurn(host, limits.minGapMs);
+                const got = await untrustedGet(`${protocol}//${host}/robots.txt`, {
+                    maxBytes: ROBOTS_MAX_BYTES, timeoutMs: limits.requestTimeoutMs, maxRedirects: limits.maxRedirects, headers: ua, urlIsSafeToTrace,
+                    followRedirect: (to) => new URL(to).host === host ? true : 'robots.txt redirects to another host',
+                    ...network,
+                });
+                await done();
+                if (got.ok) {
+                    const text = new TextDecoder().decode(got.body);
+                    await settle('ok', text);
+                    return parseRobots(text, config.userAgent);
+                }
+                if (got.reason === 'status' && got.status !== undefined && got.status >= 400 && got.status < 500) {
+                    await settle('allow_all', null);
+                    return ALLOW_ALL;
+                }
+                if (got.reason === 'status' || got.reason === 'network' || got.reason === 'timeout') {
+                    await db().execute(sql `UPDATE pipework_crawl_host SET robots_attempts = ${attempt + 1} WHERE crawl_id = ${crawlId}::uuid AND host = ${host}`);
+                    continue;
+                }
+                const refusal = { reason: got.reason, message: got.message };
+                await settle('disallow_all', JSON.stringify(refusal));
+                return barred(refusal.reason, refusal.message);
+            }
+            return signal.aborted ? null : (state.stop = 'robots_unreachable', null);
+        };
+        const insertLinks = async (tx, rows) => {
+            if (rows.length === 0)
+                return;
+            await tx.execute(sql `
+        INSERT INTO pipework_crawl_url (crawl_id, url, depth, from_url, redirects, state, reason, message, seed_chain)
+        SELECT ${crawlId}::uuid, x.url, x.depth, x.from_url, x.redirects, x.state, x.reason, x.message, x.seed_chain
+          FROM jsonb_to_recordset(${JSON.stringify(rows.map(r => ({ url: r.url, depth: r.depth, from_url: r.from, redirects: r.redirects, state: r.state, reason: r.reason, message: r.message, seed_chain: r.seedChain === true })))}::jsonb)
+               AS x(url text, depth int, from_url text, redirects int, state text, reason text, message text, seed_chain boolean)
+        ON CONFLICT (crawl_id, url) DO NOTHING`);
+        };
+        const finishUrl = async (row, outcome, extra) => {
+            await db().transaction(async (tx) => {
+                await extra?.(tx);
+                await tx.execute(sql `
+          UPDATE pipework_crawl_url SET state = ${outcome.state}, reason = ${outcome.reason ?? null}, message = ${outcome.message ?? null}, status = ${outcome.status ?? null}, requested = requested OR ${outcome.requested === true}
+           WHERE id = ${row.id}::uuid`);
+            });
+        };
+        const runHook = async (url, html) => {
+            const hook = config.onPage;
+            if (hook === undefined)
+                return null;
+            const controller = new AbortController();
+            let timer;
+            const overrun = new Promise((_, reject) => {
+                timer = setTimeout(() => {
+                    const err = new Error(`onPage took longer than ${String(limits.pageHookMs)} ms.`);
+                    controller.abort(err);
+                    reject(err);
+                }, limits.pageHookMs);
+            });
+            try {
+                const work = Promise.resolve().then(() => hook({ url, html, signal: controller.signal }));
+                work.catch(() => undefined);
+                await Promise.race([work, overrun]);
+                return null;
+            }
+            catch (err) {
+                return describeError(err);
+            }
+            finally {
+                clearTimeout(timer);
+            }
+        };
+        const previous = async (url) => {
+            const rows = await db().execute(sql `
+        SELECT r.etag, r.last_modified, r.status, r.content_type, r.bytes::float8 AS bytes, r.sha256, r.file_id
+          FROM pipework_crawl_resource r JOIN pipework_crawl c ON c.id = r.crawl_id
+         WHERE c.site = ${config.name} AND c.seed = ${crawl.seed} AND c.id <> ${crawlId}::uuid AND c.created_at < (SELECT created_at FROM pipework_crawl WHERE id = ${crawlId}::uuid)
+           AND r.url = ${url} AND (r.etag IS NOT NULL OR r.last_modified IS NOT NULL)
+         ORDER BY c.created_at DESC LIMIT 1`);
+            const r = rows[0];
+            if (r === undefined)
+                return null;
+            return { etag: str(r['etag']), lastModified: str(r['last_modified']), status: num(r['status']), contentType: str(r['content_type']), bytes: num(r['bytes']), sha256: str(r['sha256']), fileId: str(r['file_id']) };
+        };
+        const movesScope = (row, to) => scope.kind === 'host' && row.seedChain && row.depth === 0 && !inScope(scope, to.hostname);
+        const processUrl = async (row) => {
+            const target = new URL(row.url);
+            const rules = await robotsFor(target.host, target.protocol);
+            if (rules === null) {
+                if (!signal.aborted)
+                    await db().execute(sql `UPDATE pipework_crawl_url SET state = 'queued' WHERE id = ${row.id}::uuid`);
+                return;
+            }
+            if (!rules.allows(target.pathname + target.search)) {
+                const why = rules.refusal;
+                await finishUrl(row, { state: 'refused', reason: why?.reason ?? 'robots', message: why?.message ?? 'robots.txt does not allow this path.' });
+                return;
+            }
+            const done = await takeTurn(target.host, gapFor(rules));
+            if (signal.aborted) {
+                await db().execute(sql `UPDATE pipework_crawl_url SET state = 'queued' WHERE id = ${row.id}::uuid`);
+                return;
+            }
+            const prior = await previous(row.url);
+            const headers = { ...ua };
+            if (prior?.etag != null)
+                headers['if-none-match'] = prior.etag;
+            if (prior?.lastModified != null)
+                headers['if-modified-since'] = prior.lastModified;
+            const hop = { to: null };
+            const got = await untrustedGet(row.url, {
+                maxBytes: limits.maxBytesPerResponse, timeoutMs: limits.requestTimeoutMs, maxRedirects: limits.maxRedirects, headers, urlIsSafeToTrace,
+                // A redirect is not followed inside the request: its target is a URL of the crawl like any link, so it meets scope, robots and the host's gap.
+                followRedirect: (to) => { hop.to = to; return 'The crawler handles redirects itself.'; },
+                accept: (head) => {
+                    const type = baseType(head.contentType);
+                    if (HTML_TYPES.has(type))
+                        return true;
+                    return config.accept({ url: row.url, contentType: type }) || 'This is neither a page nor a file the app lists.';
+                },
+                ...network,
+            });
+            await done();
+            if (!got.ok) {
+                if (got.reason === 'rejected' && hop.to !== null) {
+                    const hops = row.redirects + 1;
+                    let to = null;
+                    try {
+                        to = new URL(hop.to);
+                    }
+                    catch { /* refused below */ }
+                    if (hops > limits.maxRedirects)
+                        await finishUrl(row, { requested: true, state: 'refused', reason: 'redirects', message: 'The link redirects too many times.' });
+                    else if (to === null || (!inScope(scope, to.hostname) && !movesScope(row, to)))
+                        await finishUrl(row, { requested: true, state: 'refused', reason: 'scope', message: 'The link redirects to a page outside the crawl\'s scope.' });
+                    else {
+                        to.hash = '';
+                        // The seed's own redirect chain may move a host scope (example.gov.uk to www.example.gov.uk).
+                        const moved = inScope(scope, to.hostname) ? null : { kind: 'host', host: to.hostname };
+                        await finishUrl(row, { requested: true, state: 'done', reason: 'redirected', message: 'Redirected; the target is queued.' }, async (tx) => {
+                            if (moved !== null)
+                                await tx.execute(sql `UPDATE pipework_crawl SET scope = ${JSON.stringify(moved)}::jsonb WHERE id = ${crawlId}::uuid`);
+                            await insertLinks(tx, [{ url: to.href, depth: row.depth, from: row.fromUrl ?? row.url, state: 'queued', reason: null, message: null, redirects: hops, seedChain: row.seedChain }]);
+                        });
+                        if (moved !== null)
+                            scope = moved;
+                    }
+                    return;
+                }
+                if (got.reason === 'status' && got.status === 304 && prior !== null) {
+                    await finishUrl(row, { requested: true, state: 'done', status: 304 }, tx => tx.execute(sql `
+            INSERT INTO pipework_crawl_resource (crawl_id, url, from_url, status, content_type, bytes, etag, last_modified, sha256, file_id, unchanged)
+            VALUES (${crawlId}::uuid, ${row.url}, ${row.fromUrl}, ${prior.status}, ${prior.contentType}, ${prior.bytes}, ${prior.etag}, ${prior.lastModified}, ${prior.sha256}, ${prior.fileId}, true)
+            ON CONFLICT (crawl_id, url) DO NOTHING`).then(() => undefined));
+                    return;
+                }
+                await finishUrl(row, { requested: true, state: 'refused', reason: got.reason, message: got.message, status: got.status ?? null });
+                return;
+            }
+            const type = baseType(got.contentType);
+            const isHtml = HTML_TYPES.has(type);
+            const listed = config.accept({ url: got.url, contentType: type });
+            let hookFault = null;
+            let links = [];
+            if (isHtml) {
+                const html = new TextDecoder().decode(got.body);
+                links = extractLinks(html, got.url);
+                hookFault = await runHook(row.url, html);
+            }
+            let sha256 = null;
+            let fileId = null;
+            if (listed && config.store !== undefined) {
+                const file = await config.store.put({ body: Readable.from(Buffer.from(got.body)), mediaType: type === '' ? 'application/octet-stream' : type, metadata: { url: got.url, crawl: crawlId } });
+                sha256 = file.sha256;
+                fileId = file.id;
+            }
+            const children = links.map(link => {
+                const u = new URL(link);
+                const depth = row.depth + 1;
+                if (!inScope(scope, u.hostname))
+                    return { url: link, depth, from: row.url, state: 'refused', reason: 'scope', message: 'The link points outside the crawl\'s scope.', redirects: 0 };
+                if (depth > limits.maxDepth)
+                    return { url: link, depth, from: row.url, state: 'refused', reason: 'depth', message: `The link is more than ${String(limits.maxDepth)} links from the seed.`, redirects: 0 };
+                return { url: link, depth, from: row.url, state: 'queued', reason: null, message: null, redirects: 0 };
+            });
+            await finishUrl(row, hookFault === null ? { requested: true, state: 'done', status: got.status } : { requested: true, state: 'done', status: got.status, reason: 'page_hook', message: hookFault }, async (tx) => {
+                await tx.execute(sql `UPDATE pipework_crawl SET bytes = bytes + ${got.bytes} WHERE id = ${crawlId}::uuid`);
+                if (listed) {
+                    await tx.execute(sql `
+              INSERT INTO pipework_crawl_resource (crawl_id, url, from_url, status, content_type, bytes, etag, last_modified, sha256, file_id)
+              VALUES (${crawlId}::uuid, ${row.url}, ${row.fromUrl}, ${got.status}, ${type === '' ? null : type}, ${got.bytes}, ${got.etag}, ${got.lastModified}, ${sha256}, ${fileId})
+              ON CONFLICT (crawl_id, url) DO NOTHING`);
+                }
+                await insertLinks(tx, children);
+            });
+        };
+        /** The next queued URL, claimed so no other lane takes it; breadth first. */
+        const claim = async () => {
+            const rows = await db().execute(sql `
+        UPDATE pipework_crawl_url SET state = 'fetching'
+         WHERE id = (SELECT id FROM pipework_crawl_url WHERE crawl_id = ${crawlId}::uuid AND state = 'queued'
+                      ORDER BY depth, seq LIMIT 1 FOR UPDATE SKIP LOCKED)
+        RETURNING id::text AS id, url, depth, from_url, redirects, seed_chain`);
+            const r = rows[0];
+            return r === undefined ? null : { id: r['id'], url: r['url'], depth: num(r['depth']), fromUrl: str(r['from_url']), redirects: num(r['redirects']), seedChain: r['seed_chain'] === true };
+        };
+        const limitHit = async () => {
+            if (Date.now() - crawl.runStartedAt > limits.maxWallMs)
+                return 'wall';
+            const r = (await db().execute(sql `
+        SELECT (SELECT count(*) FROM pipework_crawl_url WHERE crawl_id = ${crawlId}::uuid AND requested)::int AS pages,
+               (SELECT bytes::float8 FROM pipework_crawl WHERE id = ${crawlId}::uuid) AS bytes`))[0];
+            if (num(r['pages']) >= limits.maxPages)
+                return 'pages';
+            if (num(r['bytes']) >= limits.maxBytes)
+                return 'bytes';
+            return null;
+        };
+        const lane = async () => {
+            while (!signal.aborted && state.stop === null) {
+                const hit = await limitHit();
+                if (hit !== null) {
+                    const queued = hit === 'wall' || hit === 'bytes' || num((await db().execute(sql `SELECT count(*)::int AS n FROM pipework_crawl_url WHERE crawl_id = ${crawlId}::uuid AND state = 'queued'`))[0]['n']) > 0;
+                    if (queued)
+                        state.stop = hit;
+                    else if (state.inFlight === 0)
+                        return;
+                }
+                if (state.stop !== null)
+                    return;
+                const row = await claim();
+                if (row === null) {
+                    if (state.inFlight === 0)
+                        return;
+                    await sleep(25, signal);
+                    continue;
+                }
+                state.inFlight++;
+                try {
+                    await processUrl(row);
+                }
+                finally {
+                    state.inFlight--;
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: limits.perHostConcurrency }, () => lane()));
+        if (signal.aborted)
+            return;
+        if (state.stop !== null) {
+            await db().execute(sql `UPDATE pipework_crawl SET state = 'stopped', stopped_by = ${state.stop}, finished_at = now() WHERE id = ${crawlId}::uuid`);
+            return;
+        }
+        const depthCut = await db().execute(sql `SELECT 1 FROM pipework_crawl_url WHERE crawl_id = ${crawlId}::uuid AND reason = 'depth' LIMIT 1`);
+        if (depthCut.length > 0) {
+            await db().execute(sql `UPDATE pipework_crawl SET state = 'stopped', stopped_by = 'depth', finished_at = now() WHERE id = ${crawlId}::uuid`);
+            return;
+        }
+        await db().execute(sql `UPDATE pipework_crawl SET state = 'done', finished_at = now() WHERE id = ${crawlId}::uuid`);
+    }
+    const builder = database !== undefined ? fitting.use(database) : fitting.use();
+    const job = builder.job(jobType)
+        .transaction('manual')
+        .input(schema.check.object({ crawlId: schema.check.string() }))
+        .fit((async ({ input, signal }) => {
+        try {
+            await run(input.crawlId, signal);
+        }
+        catch (err) {
+            const flow = getContext();
+            const last = flow?.jobAttempt != null && flow.jobMaxAttempts != null && flow.jobAttempt >= flow.jobMaxAttempts;
+            if (last) {
+                getBaseLogger().error({ err, crawlId: input.crawlId }, 'Crawl job failed on its last attempt');
+                await db().execute(sql `UPDATE pipework_crawl SET state = 'failed', error = ${describeError(err)}, finished_at = now() WHERE id = ${input.crawlId}::uuid`);
+            }
+            throw err;
+        }
+    }));
+    return {
+        job,
+        jobType,
+        start: (options) => startStep(options),
+        approve: (id) => approveStep(id),
+        status, resources, refusals,
+    };
+}
+//# sourceMappingURL=handle.js.map

package/dist/infra/crawl/handle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"handle.js","sourceRoot":"","sources":["../../../src/infra/crawl/handle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAA;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAA;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAA;AAChE,OAAO,EAAE,MAAM,EAAE,MAAM,gCAAgC,CAAA;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAA;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAGzE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAIjD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AACzC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAoB,MAAM,aAAa,CAAA;AA6IpF,MAAM,CAAC,MAAM,cAAc,GAAwB;IACjD,kBAAkB,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,EAAE,mBAAmB,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;IAC/H,SAAS,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI;CAClF,CAAA;AAED,MAAM,OAAO,GAAG,2BAA2B,CAAA;AAC3C,MAAM,gBAAgB,GAAG,GAAG,GAAG,IAAI,CAAA;AACnC,MAAM,YAAY,GAAG,CAAC,CAAA;AAGtB,MAAM,MAAM,GAAG,CAAC,MAAc,EAAE,OAAe,EAAe,EAAE,CAAC,CAAC,EAAE,GAAG,YAAY,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAA;AACpH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,uBAAuB,CAAC,CAAC,CAAA;AAelE,MAAM,GAAG,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AAClD,MAAM,GAAG,GAAG,CAAC,CAAU,EAAiB,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;AAC7F,MAAM,QAAQ,GAAG,CAAC,WAA0B,EAAU,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;AAChH,MAAM,aAAa,GAAG,CAAC,GAAY,EAAU,EAAE,CAAC,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;AAEhH,SAAS,KAAK,CAAC,EAAU,EAAE,MAAmB;IAC5C,OAAO,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE;QACjC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAAC,OAAO,EAAE,CAAC;YAAC,OAAM;QAAC,CAAC;QACpD,MAAM,IAAI,GAAG,GAAS,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAA,CAAC,CAAC,CAAA;QACtG,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QAClC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;IACxD,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,KAAkB,EAAE,QAAgB;IACnD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,QAAQ,KAAK,KAAK,CAAC,IAAI,CAAA;IACzD,OAAO,QAAQ,KAAK,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAA;AAC3E,CAAC;AAED,SAAS,aAAa,CAAC,KAA8B;IACnD,MAAM,GAAG,GAA2B,EAAE,GAAG,cAAc,EAAE,CAAA;IACzD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC;QACvD,IAAI,CAAC,CAAC,GAAG,IAAI,cAAc,CAAC;YAAE,MAAM,IAAI,cAAc,CAAC,kBAAkB,GAAG,cAAc,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACrI,IAAI,KAAK,KAAK,SAAS;YAAE,SAAQ;QACjC,MAAM,GAAG,GAAG,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,cAAc,IAAI,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QACtF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,GAAG;YAAE,MAAM,IAAI,cAAc,CAAC,SAAS,GAAG,iCAAiC,MAAM,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QACpL,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;IAClB,CAAC;IACD,OAAO,GAAqC,CAAA;AAC9C,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,KAA6B;IAC5D,IAAI,GAAQ,CAAA;IACZ,IAAI,CAAC;QAAC,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAA;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,MAAM,IAAI,cAAc,CAAC,aAAa,IAAI,iBAAiB,CAAC,CAAA;IAAC,CAAC;IAClG,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,MAAM,IAAI,cAAc,CAAC,8CAA8C,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;IACjJ,GAAG,CAAC,IAAI,GAAG,EAAE,CAAA;IACb,IAAI,MAAmB,CAAA;IACvB,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;QAC5C,MAAM,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAA;IAC/C,CAAC;SAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACvG,MAAM,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,CAAA;IAClG,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,cAAc,CAAC,uDAAuD,CAAC,CAAA;IACnF,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,cAAc,CAAC,mBAAmB,GAAG,CAAC,QAAQ,6BAA6B,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAA;IAChJ,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAA;AACxB,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,WAAW,CAAC,MAAmB;IAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAA;IAClD,IAAI,SAAS,KAAK,SAAS;QAAE,MAAM,IAAI,oBAAoB,EAAE,CAAA;IAC7D,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAA;IAChJ,CAAC;IACD,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,+FAA+F,CAAC,CAAA;IAC5L,MAAM,eAAe,GAAG,SAAS,CAAC,eAAe,KAAK,IAAI,CAAA;IAC1D,MAAM,OAAO,GAAG,kBAAkB,MAAM,CAAC,IAAI,EAAE,CAAA;IAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAA;IAChC,MAAM,EAAE,GAAG,GAAO,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;IACzC,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,IAAI,CAAA;IAClD,MAAM,gBAAgB,GAAG,MAAM,CAAC,SAAS,KAAK,KAAK,CAAA;IAEnD,MAAM,OAAO,GAAG,KAAK,EAAE,EAAU,EAAiB,EAAE;QAClD,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC,CAAA;IACtJ,CAAC,CAAA;IAED,MAAM,SAAS,GAAG,UAAU,CAAC,EAAE,GAAG,EAAE,sBAAsB,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,OAA0B,EAA2B,EAAE;QAC3I,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;QAC9D,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAC5C,MAAM,IAAI,GAAG,UAAU,EAAE,CAAA;QACzB,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;QAC9D,MAAM,KAAK,GAAe,eAAe,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS,CAAA;QAC3E,MAAM,IAAI,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;gBAEvB,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,KAAK;gBACtG,IAAI,EAAE,MAAM,IAAI,IAAI,KAAK,KAAK,KAAK,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAA,OAAO;+BACvD,CAAC,CAAA;QAC5B,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC,IAAI,CAAW,CAAA;QACnC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,6EAA6E,EAAE,WAAW,GAAG,CAAC,IAAI,YAAY,CAAC,CAAA;QACrI,MAAM,UAAU,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,eAAe,EAAE,EAAE,QAAQ,CAAC,CAAA;QACxI,IAAI,CAAC,eAAe;YAAE,MAAM,OAAO,CAAC,EAAE,CAAC,CAAA;QACvC,OAAO,EAAE,EAAE,EAAE,CAAA;IACf,CAAC,CAAC,CAAA;IAEF,MAAM,WAAW,GAAG,UAAU,CAAC,EAAE,GAAG,EAAE,wBAAwB,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,EAAU,EAAiB,EAAE;QACrH,MAAM,YAAY,CAAC,EAAE,CAAC,CAAA;QACtB,MAAM,IAAI,GAAG,UAAU,EAAE,CAAA;QACzB,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;QAC9D,MAAM,IAAI,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;mEAC4B,KAAK;oBACpD,EAAE,qBAAqB,MAAM,CAAC,IAAI,iDAAiD,CAAC,CAAA;QACpG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,+CAA+C,EAAE,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;YACxH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;YAC9D,MAAM,IAAI,aAAa,CAAC,SAAS,EAAE,oCAAoC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAA;QACxG,CAAC;QACD,MAAM,UAAU,CAAC,SAAS,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;QACxF,MAAM,OAAO,CAAC,EAAE,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;IAEF,KAAK,UAAU,SAAS,CAAC,EAAU;QACjC,MAAM,IAAI,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;;yCAGE,EAAE,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;QAC1E,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,IAAI,CAAA;QAChC,OAAO;YACL,KAAK,EAAE,CAAC,CAAC,OAAO,CAAe,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAwB,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,CAAgB,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAW;YACxI,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YACrE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAA0B,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;SACzG,CAAA;IACH,CAAC;IAED,2FAA2F;IAC3F,MAAM,YAAY,GAAG,KAAK,EAAE,EAAU,EAA+D,EAAE;QACrG,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,UAAU,EAAE,EAAE,MAAM,IAAI,IAAI,CAAA;QAC3C,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,QAAQ,KAAK,MAAM,CAAC;YAAE,MAAM,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;QACpH,OAAO,KAAK,CAAA;IACd,CAAC,CAAA;IAED,MAAM,MAAM,GAAG,KAAK,EAAE,EAAU,EAAwB,EAAE;QACxD,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,EAAE,CAAC,CAAA;QACpC,MAAM,OAAO,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;;mDAGS,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;0IACiG,EAAE,QAAQ,CAAC,CAAA;QACjJ,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACrB,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE;YAChG,GAAG,CAAC,KAAK,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,MAAM,EAAE;gBACN,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACxG,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK;aAC/F;SACF,CAAA;IACH,CAAC,CAAA;IAED,MAAM,SAAS,GAAG,KAAK,EAAE,EAAU,EAA4B,EAAE;QAC/D,MAAM,YAAY,CAAC,EAAE,CAAC,CAAA;QACtB,MAAM,IAAI,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;;4BAGX,EAAE,uBAAuB,MAAM,CAAC,IAAI,8BAA8B,CAAC,CAAA;QAC3F,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpB,GAAG,EAAE,CAAC,CAAC,KAAK,CAAW,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxI,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,WAAW,CAAC,KAAK,IAAI;SACrJ,CAAC,CAAC,CAAA;IACL,CAAC,CAAA;IAED,MAAM,QAAQ,GAAG,KAAK,EAAE,EAAU,EAA2B,EAAE;QAC7D,MAAM,YAAY,CAAC,EAAE,CAAC,CAAA;QACtB,MAAM,IAAI,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;;4BAGX,EAAE,uBAAuB,MAAM,CAAC,IAAI,qEAAqE,CAAC,CAAA;QAClI,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAW,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAA;IACjJ,CAAC,CAAA;IAED,oBAAoB;IAEpB,KAAK,UAAU,GAAG,CAAC,OAAe,EAAE,MAAmB;QACrD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,CAAA;QACtC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;YAAE,OAAM;QACvD,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAA;QACxB,IAAI,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QACvB,MAAM,KAAK,GAAa,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,CAAA;QAC5G,iHAAiH;QACjH,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,mEAAmE,OAAO,+BAA+B,CAAC,CAAA;QAEhI,MAAM,OAAO,GAAG,KAAK,EAAE,IAAY,EAAwG,EAAE;YAC3I,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,4DAA4D,OAAO,WAAW,IAAI,2CAA2C,CAAC,CAAA;YACpJ,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;6IACsG,OAAO,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAE,CAAA;YAC/K,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAA;QACnJ,CAAC,CAAA;QAED,MAAM,MAAM,GAAG,CAAC,KAAyB,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,iBAAiB,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;QAEvH,sLAAsL;QACtL,MAAM,QAAQ,GAAG,KAAK,EAAE,IAAY,EAAE,GAAW,EAAgC,EAAE;YACjF,IAAI,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACnC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA,CAAC,CAAC,CAAC,CAAA;gBACtE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAChC,CAAC;YACD,MAAM,IAAI,CAAA;YACV,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,GAAG,GAAG,CAAC,CAAA;YAC/D,IAAI,MAAM,CAAC,kBAAkB,GAAG,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;YAC9D,MAAM,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAA;YACvC,OAAO,KAAK,IAAI,EAAE;gBAChB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;gBACtB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,EAAE,GAAG,CAAC,CAAC,CAAA;gBAC1D,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,uDAAuD,GAAG,qBAAqB,OAAO,qBAAqB,IAAI,EAAE,CAAC,CAAA;YAC1I,CAAC,CAAA;QACH,CAAC,CAAA;QAED,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,CAAA;QAC7C,MAAM,OAAO,GAAG;YACd,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrF,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtF,CAAA;QAED,+HAA+H;QAC/H,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,QAAgB,EAA+B,EAAE;YAChF,IAAI,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACpC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,OAAO,GAAG,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;gBACpC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YACjC,CAAC;YACD,OAAO,OAAO,CAAA;QAChB,CAAC,CAAA;QAED,MAAM,UAAU,GAAG,KAAK,EAAE,IAAY,EAAE,QAAgB,EAA+B,EAAE;YACvF,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;YAC/B,IAAI,GAAG,CAAC,WAAW,KAAK,IAAI;gBAAE,OAAO,WAAW,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;YACxF,IAAI,GAAG,CAAC,WAAW,KAAK,WAAW;gBAAE,OAAO,SAAS,CAAA;YACrD,IAAI,GAAG,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;gBACvC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,IAAI,MAAM,CAA+C,CAAA;gBAC/F,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YACzE,CAAC;YACD,MAAM,MAAM,GAAG,KAAK,EAAE,WAAmB,EAAE,IAAmB,EAAiB,EAAE;gBAC/E,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,iDAAiD,WAAW,mBAAmB,IAAI,qBAAqB,OAAO,qBAAqB,IAAI,EAAE,CAAC,CAAA;YACnK,CAAC,CAAA;YACD,KAAK,IAAI,OAAO,GAAG,GAAG,CAAC,QAAQ,EAAE,OAAO,GAAG,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;gBACnE,IAAI,MAAM,CAAC,OAAO;oBAAE,OAAO,IAAI,CAAA;gBAC/B,IAAI,OAAO,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;gBACxE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;gBAClD,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,GAAG,QAAQ,KAAK,IAAI,aAAa,EAAE;oBAChE,QAAQ,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,OAAO,EAAE,EAAE,EAAE,gBAAgB;oBAChI,cAAc,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,sCAAsC;oBACjG,GAAG,OAAO;iBACX,CAAC,CAAA;gBACF,MAAM,IAAI,EAAE,CAAA;gBACZ,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;oBACX,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAC/C,MAAM,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;oBACxB,OAAO,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;gBAC5C,CAAC;gBACD,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBACjG,MAAM,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;oBAC/B,OAAO,SAAS,CAAA;gBAClB,CAAC;gBACD,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;oBACpF,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,oDAAoD,OAAO,GAAG,CAAC,qBAAqB,OAAO,qBAAqB,IAAI,EAAE,CAAC,CAAA;oBAC7I,SAAQ;gBACV,CAAC;gBACD,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAA;gBAC5D,MAAM,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAA;gBACrD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;YAChD,CAAC;YACD,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,oBAAoB,EAAE,IAAI,CAAC,CAAA;QAC1E,CAAC,CAAA;QAED,MAAM,WAAW,GAAG,KAAK,EAAE,EAAM,EAAE,IAA+J,EAAiB,EAAE;YACnN,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAM;YAC7B,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;iBAET,OAAO;oCACY,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;;+CAEtL,CAAC,CAAA;QAC5C,CAAC,CAAA;QAED,MAAM,SAAS,GAAG,KAAK,EAAE,GAAW,EAAE,OAAsH,EAAE,KAAiC,EAAiB,EAAE;YAChN,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAClC,MAAM,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;gBACjB,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;kDACsB,OAAO,CAAC,KAAK,cAAc,OAAO,CAAC,MAAM,IAAI,IAAI,eAAe,OAAO,CAAC,OAAO,IAAI,IAAI,cAAc,OAAO,CAAC,MAAM,IAAI,IAAI,8BAA8B,OAAO,CAAC,SAAS,KAAK,IAAI;wBAC7M,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAA;YACjC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,KAAK,EAAE,GAAW,EAAE,IAAY,EAA0B,EAAE;YAC1E,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAA;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAA;YACnC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;YACxC,IAAI,KAAgD,CAAA;YACpD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/C,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;oBACjF,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;oBACrB,MAAM,CAAC,GAAG,CAAC,CAAA;gBACb,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;YACvB,CAAC,CAAC,CAAA;YACF,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;gBACzF,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;gBAC3B,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;gBACnC,OAAO,IAAI,CAAA;YACb,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,aAAa,CAAC,GAAG,CAAC,CAAA;YAC3B,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,KAAK,CAAC,CAAA;YACrB,CAAC;QACH,CAAC,CAAA;QAED,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAiL,EAAE;YACpN,MAAM,IAAI,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;;0BAGf,MAAM,CAAC,IAAI,iBAAiB,KAAK,CAAC,IAAI,gBAAgB,OAAO,+EAA+E,OAAO;yBACpJ,GAAG;4CACgB,CAAC,CAAA;YACvC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,IAAI,CAAC,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAA;YAChC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAA;QACpN,CAAC,CAAA;QAED,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,EAAO,EAAW,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAA;QAE/I,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAiB,EAAE;YACtD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC/B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;YAC3D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,IAAI,CAAC,MAAM,CAAC,OAAO;oBAAE,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,6DAA6D,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAA;gBACvH,OAAM;YACR,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnD,MAAM,GAAG,GAAI,KAA8B,CAAC,OAAO,CAAA;gBACnD,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,IAAI,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,IAAI,sCAAsC,EAAE,CAAC,CAAA;gBAC5I,OAAM;YACR,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;YACvD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,6DAA6D,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAA;gBAClG,OAAM;YACR,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACrC,MAAM,OAAO,GAA2B,EAAE,GAAG,EAAE,EAAE,CAAA;YACjD,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI;gBAAE,OAAO,CAAC,eAAe,CAAC,GAAG,KAAK,CAAC,IAAI,CAAA;YAC9D,IAAI,KAAK,EAAE,YAAY,IAAI,IAAI;gBAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC,YAAY,CAAA;YAClF,MAAM,GAAG,GAA0B,EAAE,EAAE,EAAE,IAAI,EAAE,CAAA;YAC/C,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;gBACtC,QAAQ,EAAE,MAAM,CAAC,mBAAmB,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,OAAO,EAAE,gBAAgB;gBACtI,+IAA+I;gBAC/I,cAAc,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,uCAAuC,CAAA,CAAC,CAAC;gBACvF,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;oBACf,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;oBACvC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,OAAO,IAAI,CAAA;oBACrC,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,IAAI,kDAAkD,CAAA;gBACjH,CAAC;gBACD,GAAG,OAAO;aACX,CAAC,CAAA;YACF,MAAM,IAAI,EAAE,CAAA;YAEZ,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU,IAAI,GAAG,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;oBACjD,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,GAAG,CAAC,CAAA;oBAC9B,IAAI,EAAE,GAAe,IAAI,CAAA;oBACzB,IAAI,CAAC;wBAAC,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,CAAC;oBAC1D,IAAI,IAAI,GAAG,MAAM,CAAC,YAAY;wBAAE,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,oCAAoC,EAAE,CAAC,CAAA;yBAC1J,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBAAE,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,0DAA0D,EAAE,CAAC,CAAA;yBAC5N,CAAC;wBACJ,EAAE,CAAC,IAAI,GAAG,EAAE,CAAA;wBACZ,8FAA8F;wBAC9F,MAAM,KAAK,GAAuB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,QAAQ,EAAE,CAAA;wBAC1G,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,mCAAmC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;4BACxI,IAAI,KAAK,KAAK,IAAI;gCAAE,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,qCAAqC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,sBAAsB,OAAO,QAAQ,CAAC,CAAA;4BACxI,MAAM,WAAW,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;wBACpL,CAAC,CAAC,CAAA;wBACF,IAAI,KAAK,KAAK,IAAI;4BAAE,KAAK,GAAG,KAAK,CAAA;oBACnC,CAAC;oBACD,OAAM;gBACR,CAAC;gBACD,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;oBACpE,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;sBAE9E,OAAO,WAAW,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,YAAY,KAAK,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;mDAC3I,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAA;oBACjE,OAAM;gBACR,CAAC;gBACD,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,CAAA;gBACjI,OAAM;YACR,CAAC;YAED,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;YACtC,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACnC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;YACjE,IAAI,SAAS,GAAkB,IAAI,CAAA;YACnC,IAAI,KAAK,GAAa,EAAE,CAAA;YACxB,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC/C,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,CAAA;gBACnC,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YAC1C,CAAC;YACD,IAAI,MAAM,GAAkB,IAAI,CAAA;YAChC,IAAI,MAAM,GAAkB,IAAI,CAAA;YAChC,IAAI,MAAM,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBACzC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAA;gBAC3L,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;gBACpB,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;YAClB,CAAC;YACD,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;gBAChC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAA;gBACvB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,CAAA;gBAC3B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC;oBAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,6CAA6C,EAAE,SAAS,EAAE,CAAC,EAAE,CAAA;gBACpL,IAAI,KAAK,GAAG,MAAM,CAAC,QAAQ;oBAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,yBAAyB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,EAAE,SAAS,EAAE,CAAC,EAAE,CAAA;gBAC1M,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAA;YACxG,CAAC,CAAC,CAAA;YACF,MAAM,SAAS,CACb,GAAG,EACH,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,EAC7K,KAAK,EAAE,EAAE,EAAE,EAAE;gBACX,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,6CAA6C,GAAG,CAAC,KAAK,eAAe,OAAO,QAAQ,CAAC,CAAA;gBACzG,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;wBAER,OAAO,WAAW,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,YAAY,KAAK,MAAM,KAAK,MAAM;qDAC/H,CAAC,CAAA;gBAC5C,CAAC;gBACD,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;YACjC,CAAC,CACF,CAAA;QACH,CAAC,CAAA;QAED,6EAA6E;QAC7E,MAAM,KAAK,GAAG,KAAK,IAA4B,EAAE;YAC/C,MAAM,IAAI,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;0EAEiC,OAAO;;8EAEH,CAAC,CAAA;YACzE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAW,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAW,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAA;QAC9M,CAAC,CAAA;QAED,MAAM,QAAQ,GAAG,KAAK,IAAoC,EAAE;YAC1D,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,SAAS;gBAAE,OAAO,MAAM,CAAA;YACrE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;2EACoC,OAAO;sEACZ,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAE,CAAA;YAC/F,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ;gBAAE,OAAO,OAAO,CAAA;YACtD,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ;gBAAE,OAAO,OAAO,CAAA;YACtD,OAAO,IAAI,CAAA;QACb,CAAC,CAAA;QAED,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;YACrC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gBAC9C,MAAM,GAAG,GAAG,MAAM,QAAQ,EAAE,CAAA;gBAC5B,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;oBACjB,MAAM,MAAM,GAAG,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,sEAAsE,OAAO,6BAA6B,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;oBACzM,IAAI,MAAM;wBAAE,KAAK,CAAC,IAAI,GAAG,GAAG,CAAA;yBACvB,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC;wBAAE,OAAM;gBACvC,CAAC;gBACD,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;oBAAE,OAAM;gBAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,CAAA;gBACzB,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;oBACjB,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC;wBAAE,OAAM;oBAChC,MAAM,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;oBACvB,SAAQ;gBACV,CAAC;gBACD,KAAK,CAAC,QAAQ,EAAE,CAAA;gBAChB,IAAI,CAAC;oBAAC,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;gBAAC,CAAC;wBAAS,CAAC;oBAAC,KAAK,CAAC,QAAQ,EAAE,CAAA;gBAAC,CAAC;YAC5D,CAAC;QACH,CAAC,CAAA;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,kBAAkB,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;QAClF,IAAI,MAAM,CAAC,OAAO;YAAE,OAAM;QAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,6DAA6D,KAAK,CAAC,IAAI,oCAAoC,OAAO,QAAQ,CAAC,CAAA;YACjJ,OAAM;QACR,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,qDAAqD,OAAO,qCAAqC,CAAC,CAAA;QACzI,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,qGAAqG,OAAO,QAAQ,CAAC,CAAA;YAC3I,OAAM;QACR,CAAC;QACD,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,4EAA4E,OAAO,QAAQ,CAAC,CAAA;IACpH,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAA;IAC9E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;SAC7B,WAAW,CAAC,QAAQ,CAAC;SACrB,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;SAC9D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAuD,EAAE,EAAE;QACrF,IAAI,CAAC;YACH,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAClC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,UAAU,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,IAAI,EAAE,UAAU,IAAI,IAAI,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,cAAc,CAAA;YAC9G,IAAI,IAAI,EAAE,CAAC;gBACT,aAAa,EAAE,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,sCAAsC,CAAC,CAAA;gBAC9F,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA,uDAAuD,aAAa,CAAC,GAAG,CAAC,oCAAoC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAA;YAC3J,CAAC;YACD,MAAM,GAAG,CAAA;QACX,CAAC;IACH,CAAC,CAAU,CAAyD,CAAA;IAEtE,OAAO;QACL,GAAG;QACH,OAAO;QACP,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC;QACtC,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC;QAChC,MAAM,EAAE,SAAS,EAAE,QAAQ;KAC5B,CAAA;AACH,CAAC"}

package/dist/infra/crawl/index.d.ts

@@ -0,0 +1,7 @@
+export { createCrawl, DEFAULT_LIMITS } from './handle.js';
+export type { CrawlConfig, CrawlSite, CrawlStartOptions, CrawlLimits, ResolvedCrawlLimits, CrawlScope, CrawlState, CrawlStoppedBy, CrawlStatus, CrawlCounts, CrawlResource, CrawlRefusal, CrawlPage, } from './handle.js';
+export { CrawlNotEnabledError, CrawlSeedError } from './errors.js';
+export { parseRobots, productToken, ALLOW_ALL, DISALLOW_ALL } from './robots.js';
+export type { RobotsRules } from './robots.js';
+export { extractLinks } from './links.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/infra/crawl/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/infra/crawl/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AACzD,YAAY,EACV,WAAW,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,mBAAmB,EAAE,UAAU,EAAE,UAAU,EAAE,cAAc,EACnH,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,SAAS,GACjE,MAAM,aAAa,CAAA;AACpB,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAClE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAChF,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA"}

package/dist/infra/crawl/index.js

@@ -0,0 +1,5 @@
+export { createCrawl, DEFAULT_LIMITS } from './handle.js';
+export { CrawlNotEnabledError, CrawlSeedError } from './errors.js';
+export { parseRobots, productToken, ALLOW_ALL, DISALLOW_ALL } from './robots.js';
+export { extractLinks } from './links.js';
+//# sourceMappingURL=index.js.map

package/dist/infra/crawl/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/infra/crawl/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAKzD,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAClE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAEhF,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA"}