polite-crawler 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/README.md +20 -2
  2. package/build/{cli-daemon-DkqyVxAI.mjs → cli-daemon-DAq4B_u-.mjs} +2 -2
  3. package/build/{cli-daemon-DkqyVxAI.mjs.map → cli-daemon-DAq4B_u-.mjs.map} +1 -1
  4. package/build/{cli-recipes-D96ppgSX.mjs → cli-recipes-BXTuQHc3.mjs} +2 -2
  5. package/build/{cli-recipes-D96ppgSX.mjs.map → cli-recipes-BXTuQHc3.mjs.map} +1 -1
  6. package/build/cli.mjs +7 -7
  7. package/build/{daemon-Bu1wpAwl.mjs → daemon-CSge_vif.mjs} +3 -3
  8. package/build/{daemon-Bu1wpAwl.mjs.map → daemon-CSge_vif.mjs.map} +1 -1
  9. package/build/{diff-Dyzr2-Oi.mjs → diff-DxhmN3bo.mjs} +2 -2
  10. package/build/{diff-Dyzr2-Oi.mjs.map → diff-DxhmN3bo.mjs.map} +1 -1
  11. package/build/{extract-DCClsy1y.mjs → extract-DmNeGMHX.mjs} +2 -2
  12. package/build/{extract-DCClsy1y.mjs.map → extract-DmNeGMHX.mjs.map} +1 -1
  13. package/build/index.d.mts +42 -4
  14. package/build/index.mjs +10 -10
  15. package/build/{open-DXTLJWhU.mjs → open-DWzixXcq.mjs} +35 -10
  16. package/build/{open-DXTLJWhU.mjs.map → open-DWzixXcq.mjs.map} +1 -1
  17. package/build/{recipe-WTgRhbwY.mjs → recipe-BQX7GI1z.mjs} +3 -3
  18. package/build/{recipe-WTgRhbwY.mjs.map → recipe-BQX7GI1z.mjs.map} +1 -1
  19. package/build/{recipe-product-CuUlNx4_.mjs → recipe-product-DVj9oHdJ.mjs} +4 -4
  20. package/build/{recipe-product-CuUlNx4_.mjs.map → recipe-product-DVj9oHdJ.mjs.map} +1 -1
  21. package/build/{runtime-BwOZJ71n.mjs → runtime-CmAOsSPn.mjs} +2 -2
  22. package/build/{runtime-BwOZJ71n.mjs.map → runtime-CmAOsSPn.mjs.map} +1 -1
  23. package/build/{version-CHqkjAh1.mjs → version-BBVugNVm.mjs} +2 -2
  24. package/build/{version-CHqkjAh1.mjs.map → version-BBVugNVm.mjs.map} +1 -1
  25. package/build/{wacz-CBMk0bzI.mjs → wacz-t-JDRIMf.mjs} +3 -3
  26. package/build/{wacz-CBMk0bzI.mjs.map → wacz-t-JDRIMf.mjs.map} +1 -1
  27. package/build/{warc-D632nC5y.mjs → warc-Ci4ygka2.mjs} +2 -2
  28. package/build/{warc-D632nC5y.mjs.map → warc-Ci4ygka2.mjs.map} +1 -1
  29. package/package.json +1 -1
package/build/index.d.mts CHANGED
@@ -3302,6 +3302,26 @@ interface WatchOptions {
3302
3302
  readonly onBehalfOf?: readonly string[] | undefined;
3303
3303
  /** This value is the number of checks that must see a change before it is confirmed. The default is 2. */
3304
3304
  readonly confirm?: 1 | 2 | undefined;
3305
+ /**
3306
+ * The value `true` keeps the time of the next check of a page that the list has. The default
3307
+ * `false` sets that time one `every` after the last check. A new entry is due at once in the two cases.
3308
+ *
3309
+ * @beta
3310
+ */
3311
+ readonly keepSchedule?: boolean | undefined;
3312
+ }
3313
+ /**
3314
+ * These options change the time of the next check of an entry (`reschedule`).
3315
+ *
3316
+ * @beta
3317
+ */
3318
+ interface WatchRescheduleOptions {
3319
+ /**
3320
+ * This time is the next check of the entry: an ISO 8601 text or a `Date`. A run checks the due
3321
+ * entries in the sequence of these times, so an earlier time also gives an earlier check. The
3322
+ * time is not before one floor (`watchFloor`) after the last check, so a watch cannot hammer a site.
3323
+ */
3324
+ readonly at: string | Date;
3305
3325
  }
3306
3326
  /** A change that one check saw, and that the next check can confirm. */
3307
3327
  interface WatchPending {
@@ -3370,6 +3390,8 @@ interface WatchRunOptions {
3370
3390
  interface WatchRunnerOptions extends WatchRunOptions {
3371
3391
  /** These agents get their robots.txt rules applied to each check, before the agents of the entry. */
3372
3392
  readonly onBehalfOf?: readonly string[] | undefined;
3393
+ /** This name connects the captures of each check to the job that asked for them, as `PageOptions.job` does. */
3394
+ readonly job?: string | undefined;
3373
3395
  /** This function operates before the first check. `openCrawler` waits there for the open of the store. */
3374
3396
  readonly ready?: (() => Promise<void>) | undefined;
3375
3397
  }
@@ -3439,6 +3461,13 @@ interface WatchList {
3439
3461
  list(): Promise<WatchEntry[]>;
3440
3462
  /** This method checks the entries that are due, to a budget. A `for await` loop gets each event. */
3441
3463
  run(options?: WatchRunOptions): WatchRun;
3464
+ /**
3465
+ * This optional method changes the time of the next check of a page (`WatchStore.reschedule`). It
3466
+ * gives the changed entry, or undefined when the list does not have the page. `openCrawler` gives it.
3467
+ *
3468
+ * @beta
3469
+ */
3470
+ reschedule?(url: string, options: WatchRescheduleOptions): Promise<WatchEntry | undefined>;
3442
3471
  }
3443
3472
  /**
3444
3473
  * These options open a watch store.
@@ -3540,10 +3569,17 @@ export declare class WatchStore {
3540
3569
  private row;
3541
3570
  /**
3542
3571
  * This method adds a page, or changes the options of its watch and keeps its state. A new entry
3543
- * is due at once. A changed entry is due one `every` after its last check. A bad option throws
3544
- * a `RangeError`.
3572
+ * is due at once. A changed entry is due one `every` after its last check. With `keepSchedule`,
3573
+ * it keeps the time of its next check. A bad option throws a `RangeError`.
3545
3574
  */
3546
3575
  add(url: string, options: WatchOptions, now: number): WatchEntry;
3576
+ /**
3577
+ * This method changes the time of the next check of a page, and nothing else. It gives the
3578
+ * changed entry, or undefined when the list does not have the page. A bad time, or a time before
3579
+ * one floor after the last check, throws a `RangeError`. A check in progress sets the time again
3580
+ * when it ends.
3581
+ */
3582
+ reschedule(url: string, options: WatchRescheduleOptions, now: number): WatchEntry | undefined;
3547
3583
  /** This method removes a page. It gives false when the list does not have the page. */
3548
3584
  remove(url: string): boolean;
3549
3585
  /** This method gives the entry of a page, or undefined. */
@@ -3579,7 +3615,7 @@ export declare const runWatch: (runtime: CrawlRuntime, source: WatchStore | (()
3579
3615
  //#endregion
3580
3616
  //#region src/version.d.ts
3581
3617
  /** This is the version of the package. A test makes sure that it agrees with `package.json`. */
3582
- export declare const VERSION = "0.2.2";
3618
+ export declare const VERSION = "0.2.3";
3583
3619
  /** This is the product token that robots.txt rules match when an application gives no token. */
3584
3620
  export declare const DEFAULT_TOKEN = "PoliteCrawler";
3585
3621
  /** This page tells a site owner what the crawler is and how to control it. */
@@ -3858,6 +3894,8 @@ interface OpenedCrawler extends PoliteCrawler {
3858
3894
  interface HybridCrawler extends OpenedCrawler {
3859
3895
  /** This method searches the stored documents. It also takes `rank: "hybrid"`, when the crawler has an embedder. */
3860
3896
  find(q: string, options?: SearchOptions): Promise<Found[]>;
3897
+ /** This list is the watch list of `OpenedCrawler`, with each optional method, for example `reschedule`. */
3898
+ readonly watch: Required<WatchList>;
3861
3899
  /** This search ranks with the vectors of the embedder, or it is undefined when the crawler has no embedder. */
3862
3900
  readonly hybrid: HybridSearch | undefined;
3863
3901
  /** This method gives vectors to the stored documents that have none, within a budget. It rejects when the crawler has no embedder. */
@@ -4920,5 +4958,5 @@ interface WaczResult extends ArchiveCounts {
4920
4958
  */
4921
4959
  export declare const exportWacz: (source: ArchiveSource, options: ArchiveOptions) => Promise<WaczResult>;
4922
4960
  //#endregion
4923
- export { ALLOW_ALL, type AiInput, type ArchiveCounts, type ArchiveFilter, type ArchiveOptions, type ArchiveSource, type Archived, type BlobStore, type CacheEntry, type CacheIndex, type CaptureFilter, type CaptureLogOptions, type CapturePayload, type CaptureReader, type CaptureRecord, type CaptureWriter, type CatchUpResult, type ChangeCommandOptions, type Changefreq, type ChunkOptions, type Clock, type CrawlEvent, type CrawlOptions, type CrawlPage, type CrawlRun, type CrawlRunOptions, type CrawlRuntime, type CrawlStop, type CrawlSummary, type CrawlerOptions, DAYS, DEFAULT_POLICY, type Daemon, type DaemonDue, type DaemonMcpHooks, type DaemonMcpHost, type DaemonMcpOptions, type DaemonMcpService, type DaemonOptions, type DaemonState, type DaemonStatus, type DaemonTask, type DaemonTick, type DamagedLine, type Day, type DiffHunk, type DiffInput, type DiffLimits, type DiffLine, type DiffOptions, type DiffResult, type DiffSide, type DiffSummary, type DigestOptions, type DirectoryKey, type DiscoverOptions, type DiscoverResult, type DiscoverRunOptions, type DiscoverSource, type DiscoverStop, type DiscoveredUrl, type Doc, type DocDigest, type DocIndexOptions, type DocInput, type DocVersion, type Duration, type EgressOptions, type EgressResolve, type EmbedCallOptions, type EmbedOptions, type EmbedResult, type EmbedStop, type Embedder, type EmbeddingModel, type ExtractOptions, type ExtractResult, type ExtractSource, type ExtractSummary, type ExtractedDoc, type Extractor, type FeatureExtractor, type FetchOptions, type FetchOutcome, type FetchRequest, type Fetched, type FieldCandidate, type FieldCheck, type FieldClaim, type FieldKind, type FieldResult, type FieldSpan, type FieldStatus, type FieldsChecked, type FileBlobStoreOptions, type FindIndexOptions, type FindOptions, type Found, type FusedRank, type HostCounts, type HostResult, type HostStateStore, type HostStop, type Hours, type HoursComparison, type HybridCrawler, type HybridFindOptions, type HybridOptions, type HybridSearch, type HybridSearchOptions, type IndexBuild, type IndexCounts, type Indexed, type Interval, type JsonRecord, type JwkInput, type KeyDirectory, type KeyInput, type Latest, MemoryHostStateStore, type MetaChange, type Money, type OllamaEmbedderOptions, type OpenOptions, type OpenedCrawler, type Outcome, type PageExtract, type PageLink, type PageOptions, type PageResult, type Passage, type PdfOptions, type PdfText, type PlaywrightRenderer, type PolicyOptions, type PoliteCrawler, type PolitenessPolicy, type Pooling, type ProductOptions, type ProductRead, type ProductResult, type ProductVariant, type RankMode, type RankOptions, type Ranked, type ReadOptions, type ReadResult, type ReadingIndex, type Recipe, type RecipeCapture, type RecipeDefinition, type RecipeDoc, type RecipeExtract, type RecipeRequest, type RecipeRequestOptions, type RecipeResult, type RecipeRunOptions, type RecipeTarget, type RecordInput, type RefusedReason, type ReindexResult, type RenderOptions, type RenderRequest, type Rendered, type Renderer, type RequestSigner, type Robots, type RobotsDecision, type RobotsGroup, type RobotsRule, type RobotsSignal, type RuntimeOptions, type ScanOptions, type ScheduleOptions, type ScheduleResult, type ScopeOptions, type SearchOptions, type SearchRank, type Signals, type SignatureAgentType, type SignatureCheck, type SignedRequest, type SitemapEntry, type SitemapKind, type SitemapLimits, type SitemapRead, type SqliteHostStateOptions, type StoreRuntime, type StoredChunk, type TextChunk, type TextDiff, type TransformersEmbedderOptions, type TransformersModule, type UrlPattern, VALIDATOR_MEMO, type ValidatorAnswer, type ValidatorLimits, type ValidatorMemo, type ValidatorReport, type ValidatorVerdict, type Variant, type VectorCounts, type VectorHit, type VectorStoreOptions, type VectorWrite, type VenueLink, type VenueLocation, type VenueOptions, type VenueOsm, type VenuePage, type VenueRead, type VenueReading, type VenueResult, type VenueSource, type Verified, type VerifyOptions, type VerifyResult, type VersionAt, type VersionChoice, type VersionText, type WaczResult, type WarcResult, type WatchEntry, type WatchEvent, type WatchKind, type WatchList, type WatchOptions, type WatchPending, type WatchRow, type WatchRun, type WatchRunOptions, type WatchRunnerOptions, type WatchSchedule, type WatchState, type WatchStop, type WatchStoreOptions, type WatchSummary, type WatchUpdate, type WebArchive, type WebBotAuthOptions, type WebBotAuthSigner, type Week, type WordPart, bodyEncoding, compareHours, decide, decodeBody, describeWeek, disagreements, extractPage, fromCompact, fromSchemaOrg, fromText, nextValidatorMemo, parseRobots, realClock, toHoursWeek, typesOf };
4961
+ export { ALLOW_ALL, type AiInput, type ArchiveCounts, type ArchiveFilter, type ArchiveOptions, type ArchiveSource, type Archived, type BlobStore, type CacheEntry, type CacheIndex, type CaptureFilter, type CaptureLogOptions, type CapturePayload, type CaptureReader, type CaptureRecord, type CaptureWriter, type CatchUpResult, type ChangeCommandOptions, type Changefreq, type ChunkOptions, type Clock, type CrawlEvent, type CrawlOptions, type CrawlPage, type CrawlRun, type CrawlRunOptions, type CrawlRuntime, type CrawlStop, type CrawlSummary, type CrawlerOptions, DAYS, DEFAULT_POLICY, type Daemon, type DaemonDue, type DaemonMcpHooks, type DaemonMcpHost, type DaemonMcpOptions, type DaemonMcpService, type DaemonOptions, type DaemonState, type DaemonStatus, type DaemonTask, type DaemonTick, type DamagedLine, type Day, type DiffHunk, type DiffInput, type DiffLimits, type DiffLine, type DiffOptions, type DiffResult, type DiffSide, type DiffSummary, type DigestOptions, type DirectoryKey, type DiscoverOptions, type DiscoverResult, type DiscoverRunOptions, type DiscoverSource, type DiscoverStop, type DiscoveredUrl, type Doc, type DocDigest, type DocIndexOptions, type DocInput, type DocVersion, type Duration, type EgressOptions, type EgressResolve, type EmbedCallOptions, type EmbedOptions, type EmbedResult, type EmbedStop, type Embedder, type EmbeddingModel, type ExtractOptions, type ExtractResult, type ExtractSource, type ExtractSummary, type ExtractedDoc, type Extractor, type FeatureExtractor, type FetchOptions, type FetchOutcome, type FetchRequest, type Fetched, type FieldCandidate, type FieldCheck, type FieldClaim, type FieldKind, type FieldResult, type FieldSpan, type FieldStatus, type FieldsChecked, type FileBlobStoreOptions, type FindIndexOptions, type FindOptions, type Found, type FusedRank, type HostCounts, type HostResult, type HostStateStore, type HostStop, type Hours, type HoursComparison, type HybridCrawler, type HybridFindOptions, type HybridOptions, type HybridSearch, type HybridSearchOptions, type IndexBuild, type IndexCounts, type Indexed, type Interval, type JsonRecord, type JwkInput, type KeyDirectory, type KeyInput, type Latest, MemoryHostStateStore, type MetaChange, type Money, type OllamaEmbedderOptions, type OpenOptions, type OpenedCrawler, type Outcome, type PageExtract, type PageLink, type PageOptions, type PageResult, type Passage, type PdfOptions, type PdfText, type PlaywrightRenderer, type PolicyOptions, type PoliteCrawler, type PolitenessPolicy, type Pooling, type ProductOptions, type ProductRead, type ProductResult, type ProductVariant, type RankMode, type RankOptions, type Ranked, type ReadOptions, type ReadResult, type ReadingIndex, type Recipe, type RecipeCapture, type RecipeDefinition, type RecipeDoc, type RecipeExtract, type RecipeRequest, type RecipeRequestOptions, type RecipeResult, type RecipeRunOptions, type RecipeTarget, type RecordInput, type RefusedReason, type ReindexResult, type RenderOptions, type RenderRequest, type Rendered, type Renderer, type RequestSigner, type Robots, type RobotsDecision, type RobotsGroup, type RobotsRule, type RobotsSignal, type RuntimeOptions, type ScanOptions, type ScheduleOptions, type ScheduleResult, type ScopeOptions, type SearchOptions, type SearchRank, type Signals, type SignatureAgentType, type SignatureCheck, type SignedRequest, type SitemapEntry, type SitemapKind, type SitemapLimits, type SitemapRead, type SqliteHostStateOptions, type StoreRuntime, type StoredChunk, type TextChunk, type TextDiff, type TransformersEmbedderOptions, type TransformersModule, type UrlPattern, VALIDATOR_MEMO, type ValidatorAnswer, type ValidatorLimits, type ValidatorMemo, type ValidatorReport, type ValidatorVerdict, type Variant, type VectorCounts, type VectorHit, type VectorStoreOptions, type VectorWrite, type VenueLink, type VenueLocation, type VenueOptions, type VenueOsm, type VenuePage, type VenueRead, type VenueReading, type VenueResult, type VenueSource, type Verified, type VerifyOptions, type VerifyResult, type VersionAt, type VersionChoice, type VersionText, type WaczResult, type WarcResult, type WatchEntry, type WatchEvent, type WatchKind, type WatchList, type WatchOptions, type WatchPending, type WatchRescheduleOptions, type WatchRow, type WatchRun, type WatchRunOptions, type WatchRunnerOptions, type WatchSchedule, type WatchState, type WatchStop, type WatchStoreOptions, type WatchSummary, type WatchUpdate, type WebArchive, type WebBotAuthOptions, type WebBotAuthSigner, type Week, type WordPart, bodyEncoding, compareHours, decide, decodeBody, describeWeek, disagreements, extractPage, fromCompact, fromSchemaOrg, fromText, nextValidatorMemo, parseRobots, realClock, toHoursWeek, typesOf };
4924
4962
  //# sourceMappingURL=index.d.mts.map
package/build/index.mjs CHANGED
@@ -1,21 +1,21 @@
1
1
  import { n as FileBlobStore, t as CaptureLog } from "./log-CmBWcyQm.mjs";
2
2
  import { n as RenderRefused, r as playwrightRenderer, t as NO_PLAYWRIGHT } from "./render-CKE2LILH.mjs";
3
3
  import { t as browserMissing } from "./browser-56dBBLec.mjs";
4
- import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-CHqkjAh1.mjs";
5
- import { A as DEADLINE_NOTE, F as EgressRefused, I as egressLookup, M as makeRecord, O as CAPTURE_SCHEMA, S as readPdf, T as locate, _ as archivedDocId, a as fetchPage, c as isHeldBack, d as recordOverride, f as reindex, h as archiveExtractor, i as fetchAndIndex, j as DEFAULT_SECRET_PARAMS, k as Crawler, l as isoTime, m as versionAtTime, n as HELD_BACK, o as findDocuments, p as verifyQuote, r as catchUpIndex, s as heldBackMeta, t as Extractors, u as readDocument, x as isPdf } from "./runtime-BwOZJ71n.mjs";
4
+ import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-BBVugNVm.mjs";
5
+ import { A as DEADLINE_NOTE, F as EgressRefused, I as egressLookup, M as makeRecord, O as CAPTURE_SCHEMA, S as readPdf, T as locate, _ as archivedDocId, a as fetchPage, c as isHeldBack, d as recordOverride, f as reindex, h as archiveExtractor, i as fetchAndIndex, j as DEFAULT_SECRET_PARAMS, k as Crawler, l as isoTime, m as versionAtTime, n as HELD_BACK, o as findDocuments, p as verifyQuote, r as catchUpIndex, s as heldBackMeta, t as Extractors, u as readDocument, x as isPdf } from "./runtime-CmAOsSPn.mjs";
6
6
  import { DEFAULT_POLICY, MemoryHostStateStore, VALIDATOR_MEMO, nextValidatorMemo, realClock } from "./politeness.mjs";
7
7
  import { ALLOW_ALL, decide, parseRobots } from "./robots.mjs";
8
8
  import { i as keyDirectory, n as KEY_DIRECTORY_TYPE, o as verifySignature, r as createWebBotAuthSigner, t as KEY_DIRECTORY_PATH } from "./webbotauth-Dq_dA_u3.mjs";
9
- import { A as NO_EMBEDDER, C as DISCOVER_LIMITS, D as readSitemap, E as feedDate, O as DocIndex, S as crawlSite, T as SITEMAP_LIMITS, _ as estimateTokens, b as SqliteHostStateStore, c as WatchStore, d as VectorStore, f as createHybridSearch, g as chunkTokensOf, h as chunkText, i as openCrawler, k as FAST_RANK_SHARE, l as runWatch, m as HYBRID_DEFAULTS, n as defaultUserAgent, o as WATCH_LIMITS, p as CHUNK_LIMITS, r as isMailbox, s as WATCH_SCHEMA, t as createRuntime, u as VECTORS_SCHEMA, v as fuseRanks, w as discoverSite, x as CRAWL_LIMITS, y as unitVector } from "./open-DXTLJWhU.mjs";
9
+ import { A as NO_EMBEDDER, C as DISCOVER_LIMITS, D as readSitemap, E as feedDate, O as DocIndex, S as crawlSite, T as SITEMAP_LIMITS, _ as estimateTokens, b as SqliteHostStateStore, c as WatchStore, d as VectorStore, f as createHybridSearch, g as chunkTokensOf, h as chunkText, i as openCrawler, k as FAST_RANK_SHARE, l as runWatch, m as HYBRID_DEFAULTS, n as defaultUserAgent, o as WATCH_LIMITS, p as CHUNK_LIMITS, r as isMailbox, s as WATCH_SCHEMA, t as createRuntime, u as VECTORS_SCHEMA, v as fuseRanks, w as discoverSite, x as CRAWL_LIMITS, y as unitVector } from "./open-DWzixXcq.mjs";
10
10
  import { a as bodyEncoding, o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
11
- import { a as formatDiff, n as diffDocument, r as diffTexts, t as DIFF_LIMITS } from "./diff-Dyzr2-Oi.mjs";
12
- import { n as checkFields, r as extractDocument, t as FIELD_KINDS } from "./extract-DCClsy1y.mjs";
13
- import { a as readJson, i as defineRecipe, n as RESERVED_RECIPES, o as registerRecipe, s as runRecipe, t as RECIPE_LIMITS } from "./recipe-WTgRhbwY.mjs";
14
- import { c as startDaemon, i as daemonStateFile, n as DAEMON_STATE_SCHEMA, o as readDaemonState, r as changeCommand, s as scheduleWatches, t as DAEMON_DEFAULTS } from "./daemon-Bu1wpAwl.mjs";
11
+ import { a as formatDiff, n as diffDocument, r as diffTexts, t as DIFF_LIMITS } from "./diff-DxhmN3bo.mjs";
12
+ import { n as checkFields, r as extractDocument, t as FIELD_KINDS } from "./extract-DmNeGMHX.mjs";
13
+ import { a as readJson, i as defineRecipe, n as RESERVED_RECIPES, o as registerRecipe, s as runRecipe, t as RECIPE_LIMITS } from "./recipe-BQX7GI1z.mjs";
14
+ import { c as startDaemon, i as daemonStateFile, n as DAEMON_STATE_SCHEMA, o as readDaemonState, r as changeCommand, s as scheduleWatches, t as DAEMON_DEFAULTS } from "./daemon-CSge_vif.mjs";
15
15
  import { i as ollamaEmbedder, n as OLLAMA_MODELS, r as isLoopback, t as NO_OLLAMA } from "./ollama-D1xRh3hY.mjs";
16
16
  import { n as TRANSFORMERS_MODELS, r as transformersEmbedder, t as NO_TRANSFORMERS } from "./transformers-BNiuiLxT.mjs";
17
17
  import { DAYS, compareHours, describeWeek, disagreements, fromCompact, fromSchemaOrg, fromText, toHoursWeek } from "./hours.mjs";
18
- import { a as readAmount, c as locationMatches, d as readVenue, f as venueReadingsOf, i as productOf, l as namesVenue, n as pickProduct, o as readProduct, r as pickVariant, s as VENUE_LIMITS, t as PRODUCT_LIMITS, u as rankHoursLinks } from "./recipe-product-CuUlNx4_.mjs";
19
- import { t as exportWacz } from "./wacz-CBMk0bzI.mjs";
20
- import { n as exportWarc } from "./warc-D632nC5y.mjs";
18
+ import { a as readAmount, c as locationMatches, d as readVenue, f as venueReadingsOf, i as productOf, l as namesVenue, n as pickProduct, o as readProduct, r as pickVariant, s as VENUE_LIMITS, t as PRODUCT_LIMITS, u as rankHoursLinks } from "./recipe-product-DVj9oHdJ.mjs";
19
+ import { t as exportWacz } from "./wacz-t-JDRIMf.mjs";
20
+ import { n as exportWarc } from "./warc-Ci4ygka2.mjs";
21
21
  export { ALLOW_ALL, BOT_PAGE, CAPTURE_SCHEMA, CHUNK_LIMITS, CRAWL_LIMITS, CaptureLog, Crawler, DAEMON_DEFAULTS, DAEMON_STATE_SCHEMA, DAYS, DEADLINE_NOTE, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, DEFAULT_TOKEN, DIFF_LIMITS, DISCOVER_LIMITS, DocIndex, EgressRefused, Extractors, FAST_RANK_SHARE, FIELD_KINDS, FileBlobStore, HELD_BACK, HYBRID_DEFAULTS, KEY_DIRECTORY_PATH, KEY_DIRECTORY_TYPE, MemoryHostStateStore, NO_EMBEDDER, NO_OLLAMA, NO_PLAYWRIGHT, NO_TRANSFORMERS, OLLAMA_MODELS, PRODUCT_LIMITS, RECIPE_LIMITS, RESERVED_RECIPES, RenderRefused, SITEMAP_LIMITS, SqliteHostStateStore, TRANSFORMERS_MODELS, VALIDATOR_MEMO, VECTORS_SCHEMA, VENUE_LIMITS, VERSION, VectorStore, WATCH_LIMITS, WATCH_SCHEMA, WatchStore, archiveExtractor, archivedDocId, bodyEncoding, browserMissing, catchUpIndex, changeCommand, checkFields, chunkText, chunkTokensOf, compareHours, crawlSite, createHybridSearch, createRuntime, createWebBotAuthSigner, daemonStateFile, decide, decodeBody, defaultUserAgent, defineRecipe, describeWeek, diffDocument, diffTexts, disagreements, discoverSite, durationMs, egressLookup, estimateTokens, exportWacz, exportWarc, extractDocument, extractPage, feedDate, fetchAndIndex, fetchPage, findDocuments, formatDiff, fromCompact, fromSchemaOrg, fromText, fuseRanks, heldBackMeta, isHeldBack, isLoopback, isMailbox, isPdf, isoTime, keyDirectory, locate, locationMatches, makeRecord, namesVenue, nextValidatorMemo, ollamaEmbedder, openCrawler, parseRobots, pickProduct, pickVariant, playwrightRenderer, productOf, rankHoursLinks, readAmount, readDaemonState, readDocument, readJson, readPdf, readProduct, readSitemap, readVenue, realClock, recordOverride, registerRecipe, reindex, runRecipe, runWatch, scheduleWatches, startDaemon, toHoursWeek, transformersEmbedder, typesOf, unitVector, venueReadingsOf, verifyQuote, verifySignature, versionAtTime };
@@ -2,15 +2,15 @@ import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
2
2
  import { n as FileBlobStore, t as CaptureLog } from "./log-CmBWcyQm.mjs";
3
3
  import { r as playwrightRenderer } from "./render-CKE2LILH.mjs";
4
4
  import { r as checkedRenderer } from "./browser-56dBBLec.mjs";
5
- import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-CHqkjAh1.mjs";
6
- import { A as DEADLINE_NOTE, C as fold, D as snippetOf, E as parseQuery, N as sha256$1, P as stableStringify, a as fetchPage, c as isHeldBack, f as reindex, i as fetchAndIndex, k as Crawler, o as findDocuments, p as verifyQuote, r as catchUpIndex, t as Extractors, u as readDocument, y as archivedOf } from "./runtime-BwOZJ71n.mjs";
5
+ import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-BBVugNVm.mjs";
6
+ import { A as DEADLINE_NOTE, C as fold, D as snippetOf, E as parseQuery, N as sha256$1, P as stableStringify, a as fetchPage, c as isHeldBack, f as reindex, i as fetchAndIndex, k as Crawler, l as isoTime, o as findDocuments, p as verifyQuote, r as catchUpIndex, t as Extractors, u as readDocument, y as archivedOf } from "./runtime-CmAOsSPn.mjs";
7
7
  import { MemoryHostStateStore, nextValidatorMemo, realClock } from "./politeness.mjs";
8
8
  import { matchesPattern } from "./robots.mjs";
9
9
  import { n as canonicalUrl, r as scopeTest, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
10
10
  import { o as decodeBody, t as extractPage } from "./html-DkQUAE_d.mjs";
11
- import { n as diffDocument, r as diffTexts } from "./diff-Dyzr2-Oi.mjs";
12
- import { r as extractDocument } from "./extract-DCClsy1y.mjs";
13
- import { o as registerRecipe } from "./recipe-WTgRhbwY.mjs";
11
+ import { n as diffDocument, r as diffTexts } from "./diff-DxhmN3bo.mjs";
12
+ import { r as extractDocument } from "./extract-DmNeGMHX.mjs";
13
+ import { o as registerRecipe } from "./recipe-BQX7GI1z.mjs";
14
14
  import { createHash, randomUUID } from "node:crypto";
15
15
  import { dirname, join } from "node:path";
16
16
  import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
@@ -3636,8 +3636,8 @@ var WatchStore = class {
3636
3636
  }
3637
3637
  /**
3638
3638
  * This method adds a page, or changes the options of its watch and keeps its state. A new entry
3639
- * is due at once. A changed entry is due one `every` after its last check. A bad option throws
3640
- * a `RangeError`.
3639
+ * is due at once. A changed entry is due one `every` after its last check. With `keepSchedule`,
3640
+ * it keeps the time of its next check. A bad option throws a `RangeError`.
3641
3641
  */
3642
3642
  add(url, options, now) {
3643
3643
  const { key, target, host } = targetOf(url);
@@ -3655,11 +3655,32 @@ var WatchStore = class {
3655
3655
  ON CONFLICT(url) DO UPDATE SET target = excluded.target, host = excluded.host,
3656
3656
  every_ms = excluded.every_ms, render = excluded.render, confirm = excluded.confirm,
3657
3657
  scope = excluded.scope, labels = excluded.labels, on_behalf_of = excluded.on_behalf_of,
3658
- next_at = CASE WHEN watches.last_at IS NULL THEN watches.next_at
3659
- ELSE watches.last_at + excluded.every_ms END`).run(key, target, host, every, render, confirm, options.scope ?? null, labels, agents, now, now);
3658
+ next_at = CASE WHEN ? = 1 OR watches.last_at IS NULL THEN watches.next_at
3659
+ ELSE watches.last_at + excluded.every_ms END`).run(key, target, host, every, render, confirm, options.scope ?? null, labels, agents, now, now, options.keepSchedule === true ? 1 : 0);
3660
3660
  return entryOf(this.row(key), now);
3661
3661
  });
3662
3662
  }
3663
+ /**
3664
+ * This method changes the time of the next check of a page, and nothing else. It gives the
3665
+ * changed entry, or undefined when the list does not have the page. A bad time, or a time before
3666
+ * one floor after the last check, throws a `RangeError`. A check in progress sets the time again
3667
+ * when it ends.
3668
+ */
3669
+ reschedule(url, options, now) {
3670
+ const { key } = targetOf(url);
3671
+ const at = Date.parse(isoTime("at", options.at));
3672
+ return this.step(() => {
3673
+ const row = this.row(key);
3674
+ if (row === void 0) return void 0;
3675
+ const earliest = row.last_at === null ? -Infinity : row.last_at + this.floorMs;
3676
+ if (at < earliest) throw new RangeError(`the next check must be at ${iso(earliest)} or later, ${String(this.floorMs / 6e4)} minutes after the last check, so a watch cannot hammer a site, not ${iso(at)}`);
3677
+ this.stmt("UPDATE watches SET next_at = ? WHERE url = ?").run(at, key);
3678
+ return entryOf({
3679
+ ...row,
3680
+ next_at: at
3681
+ }, now);
3682
+ });
3683
+ }
3663
3684
  /** This method removes a page. It gives false when the list does not have the page. */
3664
3685
  remove(url) {
3665
3686
  const { key } = targetOf(url);
@@ -3799,6 +3820,7 @@ const runWatch = (runtime, source, options = {}) => {
3799
3820
  render: row.render,
3800
3821
  maxAgeMs: 0,
3801
3822
  onBehalfOf: agents,
3823
+ job: options.job,
3802
3824
  signal: abort.signal,
3803
3825
  scope: row.scope ?? void 0,
3804
3826
  labels,
@@ -4304,6 +4326,9 @@ const openCrawler = (options) => {
4304
4326
  async list() {
4305
4327
  return (await watchStore()).list(clockNow());
4306
4328
  },
4329
+ async reschedule(url, rescheduleOptions) {
4330
+ return (await watchStore()).reschedule(url, rescheduleOptions, clockNow());
4331
+ },
4307
4332
  run(runOptions = {}) {
4308
4333
  return runWatch(runtime, watchStore, {
4309
4334
  ...runOptions,
@@ -4331,4 +4356,4 @@ const openCrawler = (options) => {
4331
4356
  //#endregion
4332
4357
  export { NO_EMBEDDER as A, DISCOVER_LIMITS as C, readSitemap as D, feedDate as E, DocIndex as O, crawlSite as S, SITEMAP_LIMITS as T, estimateTokens as _, open_exports as a, SqliteHostStateStore as b, WatchStore as c, VectorStore as d, createHybridSearch as f, chunkTokensOf as g, chunkText as h, openCrawler as i, FAST_RANK_SHARE as k, runWatch as l, HYBRID_DEFAULTS as m, defaultUserAgent as n, WATCH_LIMITS as o, CHUNK_LIMITS as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, VECTORS_SCHEMA as u, fuseRanks as v, discoverSite as w, CRAWL_LIMITS as x, unitVector as y };
4333
4358
 
4334
- //# sourceMappingURL=open-DXTLJWhU.mjs.map
4359
+ //# sourceMappingURL=open-DWzixXcq.mjs.map