polite-crawler 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +533 -8
- package/build/cli-daemon--cTmq0Z5.mjs +38 -0
- package/build/cli-daemon--cTmq0Z5.mjs.map +1 -0
- package/build/cli.mjs +288 -20
- package/build/cli.mjs.map +1 -1
- package/build/{open-Ca1lcWR3.d.mts → daemon-DP4EqjFL.d.mts} +913 -16
- package/build/daemon-fZTE24P5.mjs +515 -0
- package/build/daemon-fZTE24P5.mjs.map +1 -0
- package/build/{diff-Dk2eSvi-.mjs → diff-NQcQqKe9.mjs} +2 -2
- package/build/{diff-Dk2eSvi-.mjs.map → diff-NQcQqKe9.mjs.map} +1 -1
- package/build/{extract-DcwNrTK7.mjs → extract-ByFfc8Zk.mjs} +2 -2
- package/build/{extract-DcwNrTK7.mjs.map → extract-ByFfc8Zk.mjs.map} +1 -1
- package/build/hours.d.mts +26 -0
- package/build/hours.mjs +48 -1
- package/build/hours.mjs.map +1 -1
- package/build/index.d.mts +491 -11
- package/build/index.mjs +898 -9
- package/build/index.mjs.map +1 -0
- package/build/loopback-C_QYowSf.mjs +22 -0
- package/build/loopback-C_QYowSf.mjs.map +1 -0
- package/build/mcp.d.mts +158 -2
- package/build/mcp.mjs +567 -18
- package/build/mcp.mjs.map +1 -1
- package/build/ollama-BJxkjny4.mjs +160 -0
- package/build/ollama-BJxkjny4.mjs.map +1 -0
- package/build/{open-DnrWgkC0.mjs → open-C4fhXJgD.mjs} +1404 -21
- package/build/open-C4fhXJgD.mjs.map +1 -0
- package/build/{runtime-k8mo1_7U.mjs → runtime-JToRO63o.mjs} +3 -3
- package/build/{runtime-k8mo1_7U.mjs.map → runtime-JToRO63o.mjs.map} +1 -1
- package/build/transformers-BQMlaGj4.mjs +133 -0
- package/build/transformers-BQMlaGj4.mjs.map +1 -0
- package/build/{duration-iIuMMofi.mjs → version-ntbY-6wh.mjs} +10 -10
- package/build/version-ntbY-6wh.mjs.map +1 -0
- package/build/{wacz-sfOLT8e9.mjs → wacz-BFNG42Kt.mjs} +3 -3
- package/build/{wacz-sfOLT8e9.mjs.map → wacz-BFNG42Kt.mjs.map} +1 -1
- package/build/{warc-VIPscWxY.mjs → warc-CVS0t1fW.mjs} +2 -2
- package/build/{warc-VIPscWxY.mjs.map → warc-CVS0t1fW.mjs.map} +1 -1
- package/package.json +10 -2
- package/build/duration-iIuMMofi.mjs.map +0 -1
- package/build/open-DnrWgkC0.mjs.map +0 -1
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { n as RenderRefused } from "./render-C1CUraWF.mjs";
|
|
2
|
+
import { n as DEFAULT_TOKEN } from "./version-ntbY-6wh.mjs";
|
|
2
3
|
import { classify, isClientError, parseHttpDate } from "./classify.mjs";
|
|
3
4
|
import { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, checkPolicy, equalJitterMs, policyFor, realClock, restAfter, validatorsToSend } from "./politeness.mjs";
|
|
4
5
|
import { decide, isProductToken, robotsFromResponse, robotsFromStatus, signalsFor, unreachableRobots } from "./robots.mjs";
|
|
5
6
|
import { NO_SIGNALS, metaSignals, readSignals } from "./signals.mjs";
|
|
6
|
-
import { r as DEFAULT_TOKEN } from "./duration-iIuMMofi.mjs";
|
|
7
7
|
import { n as canonicalUrl, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
|
|
8
8
|
import { a as signerOf } from "./webbotauth-CjJdTAfE.mjs";
|
|
9
9
|
import { o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
|
|
@@ -4319,6 +4319,6 @@ const catchUpIndex = async (runtime) => {
|
|
|
4319
4319
|
};
|
|
4320
4320
|
};
|
|
4321
4321
|
//#endregion
|
|
4322
|
-
export {
|
|
4322
|
+
export { snippetOf as A, egressLookup as B, fallbackReason as C, foldText as D, fold as E, renderedKey as F, makeRecord as I, sha256 as L, Crawler as M, DEADLINE_NOTE as N, locate as O, DEFAULT_SECRET_PARAMS as P, stableStringify as R, archivedOf as S, readPdf as T, versionAtTime as _, fetchPage as a, archivedDocId as b, isHeldBack as c, pageExtractor as d, readDocument as f, verifyQuote as g, renderAndIndex as h, fetchAndIndex as i, CAPTURE_SCHEMA as j, parseQuery as k, isoTime as l, reindex as m, HELD_BACK as n, findDocuments as o, recordOverride as p, catchUpIndex as r, heldBackMeta as s, Extractors as t, pageDocId as u, archiveExtractor as v, isPdf as w, archivedNote as x, archivedCopy as y, EgressRefused as z };
|
|
4323
4323
|
|
|
4324
|
-
//# sourceMappingURL=runtime-
|
|
4324
|
+
//# sourceMappingURL=runtime-JToRO63o.mjs.map
|