website-to-markdown 1.0.1 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -1
- package/dist/cli.d.ts +1 -5
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +30 -9
- package/dist/cli.js.map +1 -1
- package/dist/client.d.ts +2 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +7 -8
- package/dist/client.js.map +1 -1
- package/dist/crawl-snapshot.d.ts +6 -24
- package/dist/crawl-snapshot.d.ts.map +1 -1
- package/dist/crawl-snapshot.js +2 -2
- package/dist/crawl-snapshot.js.map +1 -1
- package/dist/fetch-page.d.ts +16 -0
- package/dist/fetch-page.d.ts.map +1 -1
- package/dist/fetch-page.js +17 -2
- package/dist/fetch-page.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/limit.d.ts +3 -2
- package/dist/limit.d.ts.map +1 -1
- package/dist/limit.js.map +1 -1
- package/dist/report-api-error.d.ts +9 -0
- package/dist/report-api-error.d.ts.map +1 -0
- package/dist/report-api-error.js +13 -0
- package/dist/report-api-error.js.map +1 -0
- package/dist/schema/index.d.ts +12 -0
- package/dist/schema/index.d.ts.map +1 -0
- package/dist/schema/index.js +7 -0
- package/dist/schema/index.js.map +1 -0
- package/package.json +8 -3
- package/src/cli.ts +41 -8
- package/src/client.ts +9 -11
- package/src/crawl-snapshot.ts +14 -16
- package/src/fetch-page.ts +36 -2
- package/src/index.ts +1 -0
- package/src/limit.ts +3 -2
- package/src/report-api-error.ts +16 -0
- package/src/schema/index.ts +14 -0
package/README.md
CHANGED
|
@@ -40,9 +40,18 @@ per-host limits.
|
|
|
40
40
|
| `cloudflare` | required | `{ apiToken, accountId }` |
|
|
41
41
|
| `createLimit` | `createMemoryConcurrencyLimit` | Builds every pool. Pass a shared (Redis) limiter when several processes share the Cloudflare quota. |
|
|
42
42
|
| `onApiError` | none | Workers AI client errors (hanzio `ApiError`) other than 429s. |
|
|
43
|
+
| `reportTimeouts` | `true` | `false` keeps Workers AI timeouts from `onApiError`; the page falls through to the next tier anyway. |
|
|
43
44
|
| `logger` | silent | `{ info, warn }` |
|
|
44
45
|
| `parsePdf` | none | OCR tier for a PDF Workers AI returned nothing for. |
|
|
45
46
|
| `isHostAllowed` | public addresses only | SSRF guard run before every origin request. |
|
|
47
|
+
| `crawlSnapshot` | none | `{ root, logger? }` for the returned `withCrawlSnapshot`. |
|
|
48
|
+
|
|
49
|
+
Besides the converters, the factory returns:
|
|
50
|
+
|
|
51
|
+
- `isHostAllowed(hostname)`: the SSRF guard the converter runs, bound to its
|
|
52
|
+
`logger`. Use it for any other fetch of a customer-supplied URL.
|
|
53
|
+
- `withCrawlSnapshot(input, run)`: [crawl snapshots](#crawl-snapshots) under
|
|
54
|
+
`crawlSnapshot.root`. A `null` input just runs `run`.
|
|
46
55
|
|
|
47
56
|
A limiter is a function `run => Promise` with `tripCooldown(ms)` and
|
|
48
57
|
`dispose()`:
|
|
@@ -51,6 +60,25 @@ A limiter is a function `run => Promise` with `tripCooldown(ms)` and
|
|
|
51
60
|
import type { CreateConcurrencyLimit } from 'website-to-markdown'
|
|
52
61
|
```
|
|
53
62
|
|
|
63
|
+
`hanzio/redis` has one that shares the pools across processes; bind its
|
|
64
|
+
`send`:
|
|
65
|
+
|
|
66
|
+
```ts
|
|
67
|
+
import { createRedisConcurrencyLimit } from 'hanzio/redis'
|
|
68
|
+
|
|
69
|
+
const converter = createWebsiteToMarkdown({
|
|
70
|
+
cloudflare,
|
|
71
|
+
createLimit: (options) =>
|
|
72
|
+
createRedisConcurrencyLimit({
|
|
73
|
+
...options,
|
|
74
|
+
send: (command, args) => redis.send(command, args)
|
|
75
|
+
})
|
|
76
|
+
})
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`options.metricLabel` names the pool a per-host limiter belongs to; report
|
|
80
|
+
metrics under it rather than under `key`, which is one per customer host.
|
|
81
|
+
|
|
54
82
|
### Cancellation
|
|
55
83
|
|
|
56
84
|
Pass `signal`, or run inside `withOperationSignal(signal, run)` so every read
|
|
@@ -60,7 +88,20 @@ in a job inherits it.
|
|
|
60
88
|
|
|
61
89
|
`withCrawlSnapshot({ mode: 'record' | 'replay', name: 'label/host' }, run, { root })`
|
|
62
90
|
records every page and sitemap read under `root` and replays them, so two
|
|
63
|
-
benchmark runs read the same site. Never in production.
|
|
91
|
+
benchmark runs read the same site. Never in production. Pass
|
|
92
|
+
`crawlSnapshot: { root }` to the factory to get it bound as
|
|
93
|
+
`converter.withCrawlSnapshot(input, run)`.
|
|
94
|
+
|
|
95
|
+
A shared schema can check a name without importing the runtime:
|
|
96
|
+
|
|
97
|
+
```ts
|
|
98
|
+
import { CRAWL_SNAPSHOT_NAME_PATTERN } from 'website-to-markdown/schema'
|
|
99
|
+
|
|
100
|
+
const CrawlSnapshotInput = z.object({
|
|
101
|
+
mode: z.enum(['record', 'replay']),
|
|
102
|
+
name: z.string().regex(CRAWL_SNAPSHOT_NAME_PATTERN)
|
|
103
|
+
})
|
|
104
|
+
```
|
|
64
105
|
|
|
65
106
|
## Sitemaps
|
|
66
107
|
|
|
@@ -77,3 +118,14 @@ const readable = pages.filter((page) => robotsAllows(robots.rules, page.url))
|
|
|
77
118
|
```sh
|
|
78
119
|
CLOUDFLARE_API_TOKEN=… CLOUDFLARE_ACCOUNT_ID=… bunx website-to-markdown https://example.com [--no-render]
|
|
79
120
|
```
|
|
121
|
+
|
|
122
|
+
`--env-file <path>` reads the two variables from a dotenv file instead; a
|
|
123
|
+
variable already in the environment wins. With a secrets manager, export to a
|
|
124
|
+
file or run the CLI under it:
|
|
125
|
+
|
|
126
|
+
```sh
|
|
127
|
+
infisical export --format=dotenv > .env.cloudflare
|
|
128
|
+
bunx website-to-markdown https://example.com --env-file .env.cloudflare
|
|
129
|
+
|
|
130
|
+
infisical run -- bunx website-to-markdown https://example.com
|
|
131
|
+
```
|
package/dist/cli.d.ts
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
|
-
/**
|
|
3
|
-
* Fetch a public URL as markdown and print the result. Page metadata goes to
|
|
4
|
-
* stderr; markdown goes to stdout.
|
|
5
|
-
*/
|
|
6
2
|
import { type CliIO } from 'hanzio/cli';
|
|
7
3
|
export declare const cli: import("hanzio/cli").Cli;
|
|
8
|
-
/** The bin
|
|
4
|
+
/** The bin. `--help` without a URL exits 1, as in 1.0. */
|
|
9
5
|
export declare function main(argv: readonly string[], io?: CliIO): Promise<number>;
|
|
10
6
|
//# sourceMappingURL=cli.d.ts.map
|
package/dist/cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAOA,OAAO,EACN,KAAK,KAAK,EAKV,MAAM,YAAY,CAAA;AAInB,eAAO,MAAM,GAAG,0BAqFd,CAAA;AAQF,0DAA0D;AAC1D,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,mBAG7D"}
|
package/dist/cli.js
CHANGED
|
@@ -3,12 +3,24 @@
|
|
|
3
3
|
* Fetch a public URL as markdown and print the result. Page metadata goes to
|
|
4
4
|
* stderr; markdown goes to stdout.
|
|
5
5
|
*/
|
|
6
|
-
import {
|
|
6
|
+
import { readFile } from 'node:fs/promises';
|
|
7
|
+
import { parseEnv } from 'node:util';
|
|
8
|
+
import { CliError, CliUsageError, defineCli, defineCommand } from 'hanzio/cli';
|
|
7
9
|
import { z } from 'zod';
|
|
8
10
|
import { createWebsiteToMarkdown } from './fetch-page.js';
|
|
9
11
|
export const cli = defineCli({
|
|
10
12
|
name: 'website-to-markdown',
|
|
13
|
+
// No URL prints the help and exits 1, as 1.0 did.
|
|
14
|
+
onError: (error, io) => {
|
|
15
|
+
if (error instanceof CliUsageError &&
|
|
16
|
+
error.message.split('\n').some((line) => line.startsWith('<url>: '))) {
|
|
17
|
+
io.stdout.write(error.help);
|
|
18
|
+
return 1;
|
|
19
|
+
}
|
|
20
|
+
},
|
|
11
21
|
commands: defineCommand({
|
|
22
|
+
// Unknown flags and extra positionals are ignored, as in 1.0.
|
|
23
|
+
strict: false,
|
|
12
24
|
description: 'Fetch a public URL as markdown. Needs CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.',
|
|
13
25
|
input: z.object({
|
|
14
26
|
url: z
|
|
@@ -18,10 +30,16 @@ export const cli = defineCli({
|
|
|
18
30
|
render: z
|
|
19
31
|
.boolean()
|
|
20
32
|
.default(true)
|
|
21
|
-
.describe('Use the Browser Rendering tier; --no-render keeps to HTML tiers.')
|
|
33
|
+
.describe('Use the Browser Rendering tier; --no-render keeps to HTML tiers.'),
|
|
34
|
+
envFile: z
|
|
35
|
+
.string()
|
|
36
|
+
.optional()
|
|
37
|
+
.describe('Read CLOUDFLARE_* from a dotenv file; the environment wins.')
|
|
22
38
|
}),
|
|
23
|
-
run: async ({ url, render }, ctx) => {
|
|
24
|
-
const
|
|
39
|
+
run: async ({ url, render, envFile }, ctx) => {
|
|
40
|
+
const file = envFile ? parseEnv(await readEnvFile(envFile)) : {};
|
|
41
|
+
const CLOUDFLARE_API_TOKEN = ctx.env.CLOUDFLARE_API_TOKEN || file.CLOUDFLARE_API_TOKEN;
|
|
42
|
+
const CLOUDFLARE_ACCOUNT_ID = ctx.env.CLOUDFLARE_ACCOUNT_ID || file.CLOUDFLARE_ACCOUNT_ID;
|
|
25
43
|
if (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID)
|
|
26
44
|
throw new CliError('Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.');
|
|
27
45
|
const { fetchPageAsMarkdown } = createWebsiteToMarkdown({
|
|
@@ -57,12 +75,15 @@ export const cli = defineCli({
|
|
|
57
75
|
}
|
|
58
76
|
})
|
|
59
77
|
});
|
|
60
|
-
|
|
78
|
+
function readEnvFile(path) {
|
|
79
|
+
return readFile(path, 'utf8').catch(() => {
|
|
80
|
+
throw new CliError(`Cannot read --env-file ${path}.`);
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
/** The bin. `--help` without a URL exits 1, as in 1.0. */
|
|
61
84
|
export async function main(argv, io) {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
await cli.run(['--help'], io);
|
|
65
|
-
return 1;
|
|
85
|
+
const code = await cli.run(argv, io);
|
|
86
|
+
return code === 0 && argv.every((arg) => arg.startsWith('-')) ? 1 : code;
|
|
66
87
|
}
|
|
67
88
|
if (import.meta.main)
|
|
68
89
|
process.exitCode = await main(process.argv.slice(2));
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,EAEN,QAAQ,EACR,aAAa,EACb,SAAS,EACT,aAAa,EACb,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAEzD,MAAM,CAAC,MAAM,GAAG,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,qBAAqB;IAC3B,kDAAkD;IAClD,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACtB,IACC,KAAK,YAAY,aAAa;YAC9B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EACnE,CAAC;YACF,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YAC3B,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IACD,QAAQ,EAAE,aAAa,CAAC;QACvB,8DAA8D;QAC9D,MAAM,EAAE,KAAK;QACb,WAAW,EACV,uFAAuF;QACxF,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,GAAG,EAAE,CAAC;iBACJ,MAAM,EAAE;iBACR,QAAQ,CAAC,gCAAgC,CAAC;iBAC1C,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC;iBACP,OAAO,EAAE;iBACT,OAAO,CAAC,IAAI,CAAC;iBACb,QAAQ,CACR,kEAAkE,CAClE;YACF,OAAO,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,EAAE;iBACV,QAAQ,CAAC,6DAA6D,CAAC;SACzE,CAAC;QACF,GAAG,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE;YAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;YAChE,MAAM,oBAAoB,GACzB,GAAG,CAAC,GAAG,CAAC,oBAAoB,IAAI,IAAI,CAAC,oBAAoB,CAAA;YAC1D,MAAM,qBAAqB,GAC1B,GAAG,CAAC,GAAG,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB,CAAA;YAC5D,IAAI,CAAC,oBAAoB,IAAI,CAAC,qBAAqB;gBAClD,MAAM,IAAI,QAAQ,CACjB,qDAAqD,CACrD,CAAA;YAEF,MAAM,EAAE,mBAAmB,EAAE,GAAG,uBAAuB,CAAC;gBACvD,UAAU,EAAE;oBACX,QAAQ,EAAE,oBAAoB;oBAC9B,SAAS,EAAE,qBAAqB;iBAChC;aACD,CAAC,CAAA;YAEF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,CAAA;YAEtC,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAA;YAEvD,GAAG,CAAC,MAAM,CAAC,KAAK,CACf,GAAG,IAAI,CAAC,SAAS,CAChB;gBACC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAChC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBACnC,UAAU,EAAE,IAAI,CAAC,UAAU;aAC3B,EACD,IAAI,EACJ,CAAC,CACD,IAAI,CACL,CAAA;YAED,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;gBAAE,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAA;YAC3C,OAAO,IAAI,CAAC,QAAQ,CAAA;QACrB,CAAC;KACD,CAAC;CACF,CAAC,CAAA;AAEF,SAAS,WAAW,CAAC,IAAY;IAChC,OAAO,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;QACxC,MAAM,IAAI,QAAQ,CAAC,0BAA0B,IAAI,GAAG,CAAC,CAAA;IACtD,CAAC,CAAC,CAAA;AACH,CAAC;AAED,0DAA0D;AAC1D,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAuB,EAAE,EAAU;IAC7D,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IACpC,OAAO,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;AACzE,CAAC;AAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI;IAAE,OAAO,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA","sourcesContent":["#!/usr/bin/env bun\n/**\n * Fetch a public URL as markdown and print the result. Page metadata goes to\n * stderr; markdown goes to stdout.\n */\nimport { readFile } from 'node:fs/promises'\nimport { parseEnv } from 'node:util'\nimport {\n\ttype CliIO,\n\tCliError,\n\tCliUsageError,\n\tdefineCli,\n\tdefineCommand\n} from 'hanzio/cli'\nimport { z } from 'zod'\nimport { createWebsiteToMarkdown } from './fetch-page.js'\n\nexport const cli = defineCli({\n\tname: 'website-to-markdown',\n\t// No URL prints the help and exits 1, as 1.0 did.\n\tonError: (error, io) => {\n\t\tif (\n\t\t\terror instanceof CliUsageError &&\n\t\t\terror.message.split('\\n').some((line) => line.startsWith('<url>: '))\n\t\t) {\n\t\t\tio.stdout.write(error.help)\n\t\t\treturn 1\n\t\t}\n\t},\n\tcommands: defineCommand({\n\t\t// Unknown flags and extra positionals are ignored, as in 1.0.\n\t\tstrict: false,\n\t\tdescription:\n\t\t\t'Fetch a public URL as markdown. Needs CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.',\n\t\tinput: z.object({\n\t\t\turl: z\n\t\t\t\t.string()\n\t\t\t\t.describe('e.g. https://example.com/about')\n\t\t\t\t.meta({ positional: true }),\n\t\t\trender: z\n\t\t\t\t.boolean()\n\t\t\t\t.default(true)\n\t\t\t\t.describe(\n\t\t\t\t\t'Use the Browser Rendering tier; --no-render keeps to HTML tiers.'\n\t\t\t\t),\n\t\t\tenvFile: z\n\t\t\t\t.string()\n\t\t\t\t.optional()\n\t\t\t\t.describe('Read CLOUDFLARE_* from a dotenv file; the environment wins.')\n\t\t}),\n\t\trun: async ({ url, render, envFile }, ctx) => {\n\t\t\tconst file = envFile ? parseEnv(await readEnvFile(envFile)) : {}\n\t\t\tconst CLOUDFLARE_API_TOKEN =\n\t\t\t\tctx.env.CLOUDFLARE_API_TOKEN || file.CLOUDFLARE_API_TOKEN\n\t\t\tconst CLOUDFLARE_ACCOUNT_ID =\n\t\t\t\tctx.env.CLOUDFLARE_ACCOUNT_ID || file.CLOUDFLARE_ACCOUNT_ID\n\t\t\tif (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID)\n\t\t\t\tthrow new CliError(\n\t\t\t\t\t'Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.'\n\t\t\t\t)\n\n\t\t\tconst { fetchPageAsMarkdown } = createWebsiteToMarkdown({\n\t\t\t\tcloudflare: {\n\t\t\t\t\tapiToken: CLOUDFLARE_API_TOKEN,\n\t\t\t\t\taccountId: CLOUDFLARE_ACCOUNT_ID\n\t\t\t\t}\n\t\t\t})\n\n\t\t\tctx.stderr.write(`Fetching ${url}…\\n`)\n\n\t\t\tconst page = await fetchPageAsMarkdown({ url, render })\n\n\t\t\tctx.stderr.write(\n\t\t\t\t`${JSON.stringify(\n\t\t\t\t\t{\n\t\t\t\t\t\toutcome: page.outcome,\n\t\t\t\t\t\tstatus: page.status,\n\t\t\t\t\t\tsource: page.source,\n\t\t\t\t\t\turl: page.url,\n\t\t\t\t\t\tfinalUrl: page.finalUrl,\n\t\t\t\t\t\tcontentType: page.contentType,\n\t\t\t\t\t\tetag: page.etag,\n\t\t\t\t\t\tlastModified: page.lastModified,\n\t\t\t\t\t\ttitle: page.title,\n\t\t\t\t\t\tlang: page.lang,\n\t\t\t\t\t\tcanonical: page.canonical,\n\t\t\t\t\t\tmetaDescription: page.metaDescription,\n\t\t\t\t\t\tjsonLdBlocks: page.jsonLd.length,\n\t\t\t\t\t\tlinks: page.links.length,\n\t\t\t\t\t\timages: page.images.length,\n\t\t\t\t\t\tmarkdownChars: page.markdown.length,\n\t\t\t\t\t\tdurationMs: page.durationMs\n\t\t\t\t\t},\n\t\t\t\t\tnull,\n\t\t\t\t\t2\n\t\t\t\t)}\\n`\n\t\t\t)\n\n\t\t\tif (page.outcome !== 'ok') ctx.exitCode = 1\n\t\t\treturn page.markdown\n\t\t}\n\t})\n})\n\nfunction readEnvFile(path: string) {\n\treturn readFile(path, 'utf8').catch(() => {\n\t\tthrow new CliError(`Cannot read --env-file ${path}.`)\n\t})\n}\n\n/** The bin. `--help` without a URL exits 1, as in 1.0. */\nexport async function main(argv: readonly string[], io?: CliIO) {\n\tconst code = await cli.run(argv, io)\n\treturn code === 0 && argv.every((arg) => arg.startsWith('-')) ? 1 : code\n}\n\nif (import.meta.main) process.exitCode = await main(process.argv.slice(2))\n"]}
|
package/dist/client.d.ts
CHANGED
|
@@ -27,6 +27,8 @@ export declare function createCloudflareClient(options: {
|
|
|
27
27
|
createLimit: CreateConcurrencyLimit;
|
|
28
28
|
/** Every Workers AI client error except a 429, which is retried instead. */
|
|
29
29
|
onApiError?: ((error: ApiError) => void) | undefined;
|
|
30
|
+
/** Pass timeouts to `onApiError`; default true. */
|
|
31
|
+
reportTimeouts?: boolean | undefined;
|
|
30
32
|
logger: Logger;
|
|
31
33
|
}): {
|
|
32
34
|
convertFileWithWorkersAi: (input: ConvertFileInput) => Promise<string | null>;
|
package/dist/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AACA,OAAO,
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,QAAQ,EAAmB,MAAM,oBAAoB,CAAA;AACnE,OAAO,EAEN,KAAK,sBAAsB,EAE3B,MAAM,YAAY,CAAA;AACnB,OAAO,EAAsB,KAAK,MAAM,EAAE,MAAM,aAAa,CAAA;AAC7D,OAAO,EACN,0BAA0B,EAE1B,MAAM,uBAAuB,CAAA;AAM9B,OAAO,EAAE,0BAA0B,EAAE,CAAA;AAErC,8DAA8D;AAC9D,MAAM,MAAM,qBAAqB,GAAG;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;CACjB,CAAA;AASD,MAAM,MAAM,gBAAgB,GAAG;IAC9B,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;IAChC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAA;IAC1B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC7B,CAAA;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE;IAC/C,WAAW,EAAE,qBAAqB,CAAA;IAClC,WAAW,EAAE,sBAAsB,CAAA;IACnC,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC,GAAG,SAAS,CAAA;IACpD,mDAAmD;IACnD,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IACpC,MAAM,EAAE,MAAM,CAAA;CACd;sCAiF+C,gBAAgB;sCAmCtB;QACxC,IAAI,EAAE,MAAM,CAAA;QACZ,QAAQ,EAAE,MAAM,CAAA;QAChB,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;KAChC;+BAUuC,MAAM,WAAW,WAAW;EA+BpE"}
|
package/dist/client.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import Cloudflare from 'cloudflare';
|
|
2
|
-
import { createApiClient
|
|
2
|
+
import { createApiClient } from 'hanzio/api-wrapper';
|
|
3
3
|
import { createCloudflareLimits } from './limit.js';
|
|
4
4
|
import { errorLogAttributes } from './logger.js';
|
|
5
5
|
import { CloudflareRateLimitedError, withCloudflareRateLimitRetry } from './rate-limit-retry.js';
|
|
6
|
+
import { isReportableApiError } from './report-api-error.js';
|
|
6
7
|
import { rateLimitFromError } from './retry-after.js';
|
|
7
8
|
import { toMarkdown } from './to-markdown.js';
|
|
8
9
|
import { BROWSER_USER_AGENT } from './user-agent.js';
|
|
@@ -21,12 +22,10 @@ const BROWSER_RENDERING_SETTLE_MS = 1_500;
|
|
|
21
22
|
export function createCloudflareClient(options) {
|
|
22
23
|
const { credentials, logger } = options;
|
|
23
24
|
const limits = createCloudflareLimits(options.createLimit);
|
|
24
|
-
function
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
return;
|
|
29
|
-
options.onApiError?.(error);
|
|
25
|
+
function report(error) {
|
|
26
|
+
if (isReportableApiError(error, options.reportTimeouts)) {
|
|
27
|
+
options.onApiError?.(error);
|
|
28
|
+
}
|
|
30
29
|
}
|
|
31
30
|
let cloudflareSdk;
|
|
32
31
|
let toMarkdownApi;
|
|
@@ -41,7 +40,7 @@ export function createCloudflareClient(options) {
|
|
|
41
40
|
},
|
|
42
41
|
timeoutMs: 30_000,
|
|
43
42
|
retries: 0,
|
|
44
|
-
onError:
|
|
43
|
+
onError: report,
|
|
45
44
|
endpoints: {
|
|
46
45
|
toMarkdown
|
|
47
46
|
}
|
package/dist/client.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,YAAY,CAAA;AACnC,OAAO,EAEN,eAAe,EACf,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAGN,sBAAsB,EACtB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAA;AAC7D,OAAO,EACN,0BAA0B,EAC1B,4BAA4B,EAC5B,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAEpD,OAAO,EAAE,0BAA0B,EAAE,CAAA;AAQrC,MAAM,4BAA4B,GAAG,MAAM,CAAA;AAC3C;;;GAGG;AACH,MAAM,2BAA2B,GAAG,KAAK,CAAA;AAYzC;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAMtC;IACA,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACvC,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAE1D,SAAS,uBAAuB,CAAC,KAAe;QAC/C,uEAAuE;QACvE,sEAAsE;QACtE,IAAI,KAAK,YAAY,iBAAiB,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG;YAAE,OAAM;QACtE,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAA;IAC5B,CAAC;IAED,IAAI,aAAqC,CAAA;IACzC,IAAI,aAAiE,CAAA;IAErE,SAAS,mBAAmB;QAC3B,OAAO,eAAe,CAAC;YACtB,IAAI,EAAE,yBAAyB;YAC/B,WAAW,EAAE;gBACZ,OAAO,EAAE,sCAAsC;aAC/C;YACD,cAAc,EAAE;gBACf,aAAa,EAAE,UAAU,WAAW,CAAC,QAAQ,EAAE;aAC/C;YACD,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,uBAAuB;YAChC,SAAS,EAAE;gBACV,UAAU;aACV;SACD,CAAC,CAAA;IACH,CAAC;IAED,SAAS,MAAM;QACd,aAAa,KAAK,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAA;QACpE,OAAO,aAAa,CAAA;IACrB,CAAC;IAED,KAAK,UAAU,SAAS,CAAC,IAAsB;QAC9C,aAAa,KAAK,mBAAmB,EAAE,CAAA;QACvC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,UAAU,CAAC;YAC1C,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3C,SAAS,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE;YAC/C,OAAO,EAAE;gBACR,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC3C,IAAI,EAAE,IAAI,CAAC,QAAQ;iBACnB,CAAC;gBACF,iEAAiE;gBACjE,sDAAsD;gBACtD,iBAAiB,EAAE,IAAI,CAAC,QAAQ;oBAC/B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;oBACvD,CAAC,CAAC,SAAS;aACZ;SACD,CAAC,CAAA;QACF,OAAO,GAAG,CAAC,IAAI,CAAA;IAChB,CAAC;IAED,KAAK,UAAU,0BAA0B,CAAI,KAK5C;QACA,OAAO,4BAA4B,CAAC;YACnC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,kBAAkB;YAClB,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC;YAC1D,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE;gBACpC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,eAAe,EAAE;oBAC1C,OAAO;oBACP,OAAO,EAAE,MAAM;iBACf,CAAC,CAAA;YACH,CAAC;YACD,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC;SACjC,CAAC,CAAA;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,wBAAwB,CAAC,KAAuB;QAC9D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,MAAM,CAAC,SAAS;gBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,GAAG,EAAE;oBACT,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;oBAC9B,OAAO,SAAS,CAAC,KAAK,CAAC,CAAA;gBACxB,CAAC;aACD,CAAC,CAAA;YACF,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;YAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAChC,IACC,CAAC,QAAQ,CAAC,OAAO;gBACjB,CAAC,KAAK;gBACN,KAAK,CAAC,MAAM,KAAK,OAAO;gBACxB,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,EAClB,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;oBAC9C,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,EAAE;oBAC3B,SAAS,EAAE,KAAK,CAAC,QAAQ;oBACzB,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;iBAChC,CAAC,CAAA;gBACF,OAAO,IAAI,CAAA;YACZ,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YACvE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,SAAS,wBAAwB,CAAC,KAIjC;QACA,OAAO,wBAAwB,CAAC;YAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,UAAU,mBAAmB,CAAC,GAAW,EAAE,MAAoB;QACnE,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,mBAAmB;gBAC1B,KAAK,EAAE,MAAM,CAAC,gBAAgB;gBAC9B,MAAM;gBACN,GAAG,EAAE,GAAG,EAAE,CACT,MAAM,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CACxC;oBACC,UAAU,EAAE,WAAW,CAAC,SAAS;oBACjC,GAAG;oBACH,SAAS,EAAE,kBAAkB;oBAC7B,WAAW,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE;oBAClC,cAAc,EAAE,2BAA2B;iBAC3C,EACD,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,CACjD;aACF,CAAC,CAAA;YACF,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,IAAI,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YAClE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO;QACN,wBAAwB;QACxB,wBAAwB;QACxB,mBAAmB;KACnB,CAAA;AACF,CAAC","sourcesContent":["import Cloudflare from 'cloudflare'\nimport {\n\ttype ApiError,\n\tcreateApiClient,\n\tHttpResponseError\n} from 'hanzio/api-wrapper'\nimport {\n\ttype ConcurrencyLimit,\n\ttype CreateConcurrencyLimit,\n\tcreateCloudflareLimits\n} from './limit.js'\nimport { errorLogAttributes, type Logger } from './logger.js'\nimport {\n\tCloudflareRateLimitedError,\n\twithCloudflareRateLimitRetry\n} from './rate-limit-retry.js'\nimport { rateLimitFromError } from './retry-after.js'\nimport { toMarkdown } from './to-markdown.js'\nimport { BROWSER_USER_AGENT } from './user-agent.js'\n\nexport { CloudflareRateLimitedError }\n\n/** Read on first use, so getters over a secret store work. */\nexport type CloudflareCredentials = {\n\tapiToken: string\n\taccountId: string\n}\n\nconst BROWSER_RENDERING_TIMEOUT_MS = 60_000\n/**\n * `networkidle0` never settles on Squarespace commerce pages and burned 30s\n * timeouts in the probe; `load` plus a short settle gets the same markdown.\n */\nconst BROWSER_RENDERING_SETTLE_MS = 1_500\n\nexport type ConvertFileInput = {\n\tsignal?: AbortSignal | undefined\n\tbody: string | ArrayBuffer\n\t/** Cloudflare picks its converter from the extension. */\n\tfilename: string\n\tmimeType: string\n\t/** HTML only: base host for relative links. */\n\thostname?: string | undefined\n}\n\n/**\n * The two Cloudflare tiers behind their pools. Rate-limit retries live in\n * `withCloudflareRateLimitRetry` so a 429 releases the slot and trips the\n * shared cooldown before the next try.\n */\nexport function createCloudflareClient(options: {\n\tcredentials: CloudflareCredentials\n\tcreateLimit: CreateConcurrencyLimit\n\t/** Every Workers AI client error except a 429, which is retried instead. */\n\tonApiError?: ((error: ApiError) => void) | undefined\n\tlogger: Logger\n}) {\n\tconst { credentials, logger } = options\n\tconst limits = createCloudflareLimits(options.createLimit)\n\n\tfunction reportUnlessRateLimited(error: ApiError) {\n\t\t// Expected transient throttles: we retry and trip the shared cooldown.\n\t\t// Reporting every 429 floods the error tracker without adding signal.\n\t\tif (error instanceof HttpResponseError && error.status === 429) return\n\t\toptions.onApiError?.(error)\n\t}\n\n\tlet cloudflareSdk: Cloudflare | undefined\n\tlet toMarkdownApi: ReturnType<typeof createToMarkdownApi> | undefined\n\n\tfunction createToMarkdownApi() {\n\t\treturn createApiClient({\n\t\t\tname: 'cloudflareToMarkdownApi',\n\t\t\tbaseApiUrls: {\n\t\t\t\tdefault: 'https://api.cloudflare.com/client/v4'\n\t\t\t},\n\t\t\tdefaultHeaders: {\n\t\t\t\tAuthorization: `Bearer ${credentials.apiToken}`\n\t\t\t},\n\t\t\ttimeoutMs: 30_000,\n\t\t\tretries: 0,\n\t\t\tonError: reportUnlessRateLimited,\n\t\t\tendpoints: {\n\t\t\t\ttoMarkdown\n\t\t\t}\n\t\t})\n\t}\n\n\tfunction getSdk() {\n\t\tcloudflareSdk ??= new Cloudflare({ apiToken: credentials.apiToken })\n\t\treturn cloudflareSdk\n\t}\n\n\tasync function transform(file: ConvertFileInput) {\n\t\ttoMarkdownApi ??= createToMarkdownApi()\n\t\tconst res = await toMarkdownApi.toMarkdown({\n\t\t\t...(file.signal && { signal: file.signal }),\n\t\t\treqParams: { accountId: credentials.accountId },\n\t\t\treqBody: {\n\t\t\t\tfiles: new File([file.body], file.filename, {\n\t\t\t\t\ttype: file.mimeType\n\t\t\t\t}),\n\t\t\t\t// Only the HTML converter takes options; it uses the hostname to\n\t\t\t\t// resolve relative links while it walks the document.\n\t\t\t\tconversionOptions: file.hostname\n\t\t\t\t\t? JSON.stringify({ html: { hostname: file.hostname } })\n\t\t\t\t\t: undefined\n\t\t\t}\n\t\t})\n\t\treturn res.data\n\t}\n\n\tasync function runWithCloudflareRateLimit<T>(input: {\n\t\tevent: string\n\t\tlimit: ConcurrencyLimit\n\t\tsignal?: AbortSignal | undefined\n\t\trun: () => Promise<T>\n\t}) {\n\t\treturn withCloudflareRateLimitRetry({\n\t\t\tevent: input.event,\n\t\t\tsignal: input.signal,\n\t\t\trateLimitFromError,\n\t\t\ttripCooldown: (waitMs) => input.limit.tripCooldown(waitMs),\n\t\t\tonRateLimit: ({ attempt, waitMs }) => {\n\t\t\t\tlogger.warn(`${input.event}.rate_limited`, {\n\t\t\t\t\tattempt,\n\t\t\t\t\twait_ms: waitMs\n\t\t\t\t})\n\t\t\t},\n\t\t\trun: () => input.limit(input.run)\n\t\t})\n\t}\n\n\t/**\n\t * Workers AI `toMarkdown` over any file it accepts - HTML, PDF, docx, xlsx,\n\t * pptx. Returns null for a soft tier miss. Throws `CloudflareRateLimitedError`\n\t * when 429 retries are exhausted so callers can surface a clear capture issue.\n\t */\n\tasync function convertFileWithWorkersAi(input: ConvertFileInput) {\n\t\ttry {\n\t\t\tconst envelope = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'workers-ai.to-markdown',\n\t\t\t\tlimit: limits.workersAi,\n\t\t\t\tsignal: input.signal,\n\t\t\t\trun: () => {\n\t\t\t\t\tinput.signal?.throwIfAborted()\n\t\t\t\t\treturn transform(input)\n\t\t\t\t}\n\t\t\t})\n\t\t\tinput.signal?.throwIfAborted()\n\t\t\tconst first = envelope.result[0]\n\t\t\tif (\n\t\t\t\t!envelope.success ||\n\t\t\t\t!first ||\n\t\t\t\tfirst.format === 'error' ||\n\t\t\t\t!first.data?.trim()\n\t\t\t) {\n\t\t\t\tlogger.warn('workers-ai.to-markdown.unusable', {\n\t\t\t\t\tsuccess: envelope.success,\n\t\t\t\t\tformat: first?.format ?? '',\n\t\t\t\t\tmime_type: input.mimeType,\n\t\t\t\t\thas_error: Boolean(first?.error)\n\t\t\t\t})\n\t\t\t\treturn null\n\t\t\t}\n\t\t\treturn first.data\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('workers-ai.to-markdown.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\tfunction convertHtmlWithWorkersAi(input: {\n\t\thtml: string\n\t\thostname: string\n\t\tsignal?: AbortSignal | undefined\n\t}) {\n\t\treturn convertFileWithWorkersAi({\n\t\t\tbody: input.html,\n\t\t\tfilename: 'page.html',\n\t\t\tmimeType: 'text/html',\n\t\t\thostname: input.hostname,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\tasync function renderUrlToMarkdown(url: string, signal?: AbortSignal) {\n\t\ttry {\n\t\t\tconst markdown = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'browser-rendering',\n\t\t\t\tlimit: limits.browserRendering,\n\t\t\t\tsignal,\n\t\t\t\trun: () =>\n\t\t\t\t\tgetSdk().browserRendering.markdown.create(\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\taccount_id: credentials.accountId,\n\t\t\t\t\t\t\turl,\n\t\t\t\t\t\t\tuserAgent: BROWSER_USER_AGENT,\n\t\t\t\t\t\t\tgotoOptions: { waitUntil: 'load' },\n\t\t\t\t\t\t\twaitForTimeout: BROWSER_RENDERING_SETTLE_MS\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{ timeout: BROWSER_RENDERING_TIMEOUT_MS, signal }\n\t\t\t\t\t)\n\t\t\t})\n\t\t\treturn markdown.trim() || null\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('browser-rendering.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\treturn {\n\t\tconvertFileWithWorkersAi,\n\t\tconvertHtmlWithWorkersAi,\n\t\trenderUrlToMarkdown\n\t}\n}\n"]}
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,YAAY,CAAA;AACnC,OAAO,EAAiB,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACnE,OAAO,EAGN,sBAAsB,EACtB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAA;AAC7D,OAAO,EACN,0BAA0B,EAC1B,4BAA4B,EAC5B,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAEpD,OAAO,EAAE,0BAA0B,EAAE,CAAA;AAQrC,MAAM,4BAA4B,GAAG,MAAM,CAAA;AAC3C;;;GAGG;AACH,MAAM,2BAA2B,GAAG,KAAK,CAAA;AAYzC;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAQtC;IACA,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACvC,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAE1D,SAAS,MAAM,CAAC,KAAe;QAC9B,IAAI,oBAAoB,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YACzD,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAA;QAC5B,CAAC;IACF,CAAC;IAED,IAAI,aAAqC,CAAA;IACzC,IAAI,aAAiE,CAAA;IAErE,SAAS,mBAAmB;QAC3B,OAAO,eAAe,CAAC;YACtB,IAAI,EAAE,yBAAyB;YAC/B,WAAW,EAAE;gBACZ,OAAO,EAAE,sCAAsC;aAC/C;YACD,cAAc,EAAE;gBACf,aAAa,EAAE,UAAU,WAAW,CAAC,QAAQ,EAAE;aAC/C;YACD,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,MAAM;YACf,SAAS,EAAE;gBACV,UAAU;aACV;SACD,CAAC,CAAA;IACH,CAAC;IAED,SAAS,MAAM;QACd,aAAa,KAAK,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAA;QACpE,OAAO,aAAa,CAAA;IACrB,CAAC;IAED,KAAK,UAAU,SAAS,CAAC,IAAsB;QAC9C,aAAa,KAAK,mBAAmB,EAAE,CAAA;QACvC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,UAAU,CAAC;YAC1C,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3C,SAAS,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE;YAC/C,OAAO,EAAE;gBACR,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC3C,IAAI,EAAE,IAAI,CAAC,QAAQ;iBACnB,CAAC;gBACF,iEAAiE;gBACjE,sDAAsD;gBACtD,iBAAiB,EAAE,IAAI,CAAC,QAAQ;oBAC/B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;oBACvD,CAAC,CAAC,SAAS;aACZ;SACD,CAAC,CAAA;QACF,OAAO,GAAG,CAAC,IAAI,CAAA;IAChB,CAAC;IAED,KAAK,UAAU,0BAA0B,CAAI,KAK5C;QACA,OAAO,4BAA4B,CAAC;YACnC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,kBAAkB;YAClB,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC;YAC1D,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE;gBACpC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,eAAe,EAAE;oBAC1C,OAAO;oBACP,OAAO,EAAE,MAAM;iBACf,CAAC,CAAA;YACH,CAAC;YACD,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC;SACjC,CAAC,CAAA;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,wBAAwB,CAAC,KAAuB;QAC9D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,MAAM,CAAC,SAAS;gBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,GAAG,EAAE;oBACT,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;oBAC9B,OAAO,SAAS,CAAC,KAAK,CAAC,CAAA;gBACxB,CAAC;aACD,CAAC,CAAA;YACF,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;YAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAChC,IACC,CAAC,QAAQ,CAAC,OAAO;gBACjB,CAAC,KAAK;gBACN,KAAK,CAAC,MAAM,KAAK,OAAO;gBACxB,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,EAClB,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;oBAC9C,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,EAAE;oBAC3B,SAAS,EAAE,KAAK,CAAC,QAAQ;oBACzB,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;iBAChC,CAAC,CAAA;gBACF,OAAO,IAAI,CAAA;YACZ,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YACvE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,SAAS,wBAAwB,CAAC,KAIjC;QACA,OAAO,wBAAwB,CAAC;YAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,UAAU,mBAAmB,CAAC,GAAW,EAAE,MAAoB;QACnE,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,mBAAmB;gBAC1B,KAAK,EAAE,MAAM,CAAC,gBAAgB;gBAC9B,MAAM;gBACN,GAAG,EAAE,GAAG,EAAE,CACT,MAAM,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CACxC;oBACC,UAAU,EAAE,WAAW,CAAC,SAAS;oBACjC,GAAG;oBACH,SAAS,EAAE,kBAAkB;oBAC7B,WAAW,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE;oBAClC,cAAc,EAAE,2BAA2B;iBAC3C,EACD,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,CACjD;aACF,CAAC,CAAA;YACF,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,IAAI,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YAClE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO;QACN,wBAAwB;QACxB,wBAAwB;QACxB,mBAAmB;KACnB,CAAA;AACF,CAAC","sourcesContent":["import Cloudflare from 'cloudflare'\nimport { type ApiError, createApiClient } from 'hanzio/api-wrapper'\nimport {\n\ttype ConcurrencyLimit,\n\ttype CreateConcurrencyLimit,\n\tcreateCloudflareLimits\n} from './limit.js'\nimport { errorLogAttributes, type Logger } from './logger.js'\nimport {\n\tCloudflareRateLimitedError,\n\twithCloudflareRateLimitRetry\n} from './rate-limit-retry.js'\nimport { isReportableApiError } from './report-api-error.js'\nimport { rateLimitFromError } from './retry-after.js'\nimport { toMarkdown } from './to-markdown.js'\nimport { BROWSER_USER_AGENT } from './user-agent.js'\n\nexport { CloudflareRateLimitedError }\n\n/** Read on first use, so getters over a secret store work. */\nexport type CloudflareCredentials = {\n\tapiToken: string\n\taccountId: string\n}\n\nconst BROWSER_RENDERING_TIMEOUT_MS = 60_000\n/**\n * `networkidle0` never settles on Squarespace commerce pages and burned 30s\n * timeouts in the probe; `load` plus a short settle gets the same markdown.\n */\nconst BROWSER_RENDERING_SETTLE_MS = 1_500\n\nexport type ConvertFileInput = {\n\tsignal?: AbortSignal | undefined\n\tbody: string | ArrayBuffer\n\t/** Cloudflare picks its converter from the extension. */\n\tfilename: string\n\tmimeType: string\n\t/** HTML only: base host for relative links. */\n\thostname?: string | undefined\n}\n\n/**\n * The two Cloudflare tiers behind their pools. Rate-limit retries live in\n * `withCloudflareRateLimitRetry` so a 429 releases the slot and trips the\n * shared cooldown before the next try.\n */\nexport function createCloudflareClient(options: {\n\tcredentials: CloudflareCredentials\n\tcreateLimit: CreateConcurrencyLimit\n\t/** Every Workers AI client error except a 429, which is retried instead. */\n\tonApiError?: ((error: ApiError) => void) | undefined\n\t/** Pass timeouts to `onApiError`; default true. */\n\treportTimeouts?: boolean | undefined\n\tlogger: Logger\n}) {\n\tconst { credentials, logger } = options\n\tconst limits = createCloudflareLimits(options.createLimit)\n\n\tfunction report(error: ApiError) {\n\t\tif (isReportableApiError(error, options.reportTimeouts)) {\n\t\t\toptions.onApiError?.(error)\n\t\t}\n\t}\n\n\tlet cloudflareSdk: Cloudflare | undefined\n\tlet toMarkdownApi: ReturnType<typeof createToMarkdownApi> | undefined\n\n\tfunction createToMarkdownApi() {\n\t\treturn createApiClient({\n\t\t\tname: 'cloudflareToMarkdownApi',\n\t\t\tbaseApiUrls: {\n\t\t\t\tdefault: 'https://api.cloudflare.com/client/v4'\n\t\t\t},\n\t\t\tdefaultHeaders: {\n\t\t\t\tAuthorization: `Bearer ${credentials.apiToken}`\n\t\t\t},\n\t\t\ttimeoutMs: 30_000,\n\t\t\tretries: 0,\n\t\t\tonError: report,\n\t\t\tendpoints: {\n\t\t\t\ttoMarkdown\n\t\t\t}\n\t\t})\n\t}\n\n\tfunction getSdk() {\n\t\tcloudflareSdk ??= new Cloudflare({ apiToken: credentials.apiToken })\n\t\treturn cloudflareSdk\n\t}\n\n\tasync function transform(file: ConvertFileInput) {\n\t\ttoMarkdownApi ??= createToMarkdownApi()\n\t\tconst res = await toMarkdownApi.toMarkdown({\n\t\t\t...(file.signal && { signal: file.signal }),\n\t\t\treqParams: { accountId: credentials.accountId },\n\t\t\treqBody: {\n\t\t\t\tfiles: new File([file.body], file.filename, {\n\t\t\t\t\ttype: file.mimeType\n\t\t\t\t}),\n\t\t\t\t// Only the HTML converter takes options; it uses the hostname to\n\t\t\t\t// resolve relative links while it walks the document.\n\t\t\t\tconversionOptions: file.hostname\n\t\t\t\t\t? JSON.stringify({ html: { hostname: file.hostname } })\n\t\t\t\t\t: undefined\n\t\t\t}\n\t\t})\n\t\treturn res.data\n\t}\n\n\tasync function runWithCloudflareRateLimit<T>(input: {\n\t\tevent: string\n\t\tlimit: ConcurrencyLimit\n\t\tsignal?: AbortSignal | undefined\n\t\trun: () => Promise<T>\n\t}) {\n\t\treturn withCloudflareRateLimitRetry({\n\t\t\tevent: input.event,\n\t\t\tsignal: input.signal,\n\t\t\trateLimitFromError,\n\t\t\ttripCooldown: (waitMs) => input.limit.tripCooldown(waitMs),\n\t\t\tonRateLimit: ({ attempt, waitMs }) => {\n\t\t\t\tlogger.warn(`${input.event}.rate_limited`, {\n\t\t\t\t\tattempt,\n\t\t\t\t\twait_ms: waitMs\n\t\t\t\t})\n\t\t\t},\n\t\t\trun: () => input.limit(input.run)\n\t\t})\n\t}\n\n\t/**\n\t * Workers AI `toMarkdown` over any file it accepts - HTML, PDF, docx, xlsx,\n\t * pptx. Returns null for a soft tier miss. Throws `CloudflareRateLimitedError`\n\t * when 429 retries are exhausted so callers can surface a clear capture issue.\n\t */\n\tasync function convertFileWithWorkersAi(input: ConvertFileInput) {\n\t\ttry {\n\t\t\tconst envelope = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'workers-ai.to-markdown',\n\t\t\t\tlimit: limits.workersAi,\n\t\t\t\tsignal: input.signal,\n\t\t\t\trun: () => {\n\t\t\t\t\tinput.signal?.throwIfAborted()\n\t\t\t\t\treturn transform(input)\n\t\t\t\t}\n\t\t\t})\n\t\t\tinput.signal?.throwIfAborted()\n\t\t\tconst first = envelope.result[0]\n\t\t\tif (\n\t\t\t\t!envelope.success ||\n\t\t\t\t!first ||\n\t\t\t\tfirst.format === 'error' ||\n\t\t\t\t!first.data?.trim()\n\t\t\t) {\n\t\t\t\tlogger.warn('workers-ai.to-markdown.unusable', {\n\t\t\t\t\tsuccess: envelope.success,\n\t\t\t\t\tformat: first?.format ?? '',\n\t\t\t\t\tmime_type: input.mimeType,\n\t\t\t\t\thas_error: Boolean(first?.error)\n\t\t\t\t})\n\t\t\t\treturn null\n\t\t\t}\n\t\t\treturn first.data\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('workers-ai.to-markdown.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\tfunction convertHtmlWithWorkersAi(input: {\n\t\thtml: string\n\t\thostname: string\n\t\tsignal?: AbortSignal | undefined\n\t}) {\n\t\treturn convertFileWithWorkersAi({\n\t\t\tbody: input.html,\n\t\t\tfilename: 'page.html',\n\t\t\tmimeType: 'text/html',\n\t\t\thostname: input.hostname,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\tasync function renderUrlToMarkdown(url: string, signal?: AbortSignal) {\n\t\ttry {\n\t\t\tconst markdown = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'browser-rendering',\n\t\t\t\tlimit: limits.browserRendering,\n\t\t\t\tsignal,\n\t\t\t\trun: () =>\n\t\t\t\t\tgetSdk().browserRendering.markdown.create(\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\taccount_id: credentials.accountId,\n\t\t\t\t\t\t\turl,\n\t\t\t\t\t\t\tuserAgent: BROWSER_USER_AGENT,\n\t\t\t\t\t\t\tgotoOptions: { waitUntil: 'load' },\n\t\t\t\t\t\t\twaitForTimeout: BROWSER_RENDERING_SETTLE_MS\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{ timeout: BROWSER_RENDERING_TIMEOUT_MS, signal }\n\t\t\t\t\t)\n\t\t\t})\n\t\t\treturn markdown.trim() || null\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('browser-rendering.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\treturn {\n\t\tconvertFileWithWorkersAi,\n\t\tconvertHtmlWithWorkersAi,\n\t\trenderUrlToMarkdown\n\t}\n}\n"]}
|
package/dist/crawl-snapshot.d.ts
CHANGED
|
@@ -1,31 +1,13 @@
|
|
|
1
1
|
import type { Logger } from './logger.js';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* site that published a page in between, a 429, or a host that timed out on
|
|
6
|
-
* one run and not the other.
|
|
7
|
-
*
|
|
8
|
-
* Recorded: the network reads - pages as `fetchPageAsMarkdown` returns them,
|
|
9
|
-
* robots.txt and sitemaps - plus anything the caller routes through
|
|
10
|
-
* `throughCrawlSnapshot`. Everything else runs live on every replay, which is
|
|
11
|
-
* the point.
|
|
12
|
-
*
|
|
13
|
-
* Set per job through `withCrawlSnapshot`, never globally: a worker drains
|
|
14
|
-
* other tenants' jobs on the same lane. Never in production.
|
|
15
|
-
*/
|
|
16
|
-
export type CrawlSnapshotInput = {
|
|
17
|
-
mode: 'record' | 'replay';
|
|
18
|
-
/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
|
|
19
|
-
name: string;
|
|
20
|
-
};
|
|
21
|
-
/** The name becomes a path, so it may not climb out of `root`. */
|
|
22
|
-
export declare const CRAWL_SNAPSHOT_NAME_PATTERN: RegExp;
|
|
23
|
-
export declare function withCrawlSnapshot<T>(input: CrawlSnapshotInput | null | undefined, run: () => Promise<T>, options: {
|
|
2
|
+
import { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput } from './schema/index.js';
|
|
3
|
+
export { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput };
|
|
4
|
+
export type CrawlSnapshotOptions = {
|
|
24
5
|
/** Directory the snapshots live under. */
|
|
25
6
|
root: string;
|
|
26
7
|
/** Told about replay misses. */
|
|
27
|
-
logger?: Logger;
|
|
28
|
-
}
|
|
8
|
+
logger?: Logger | undefined;
|
|
9
|
+
};
|
|
10
|
+
export declare function withCrawlSnapshot<T>(input: CrawlSnapshotInput | null | undefined, run: () => Promise<T>, options: CrawlSnapshotOptions): Promise<T>;
|
|
29
11
|
/**
|
|
30
12
|
* `read` through the active snapshot. Recording stores what the network
|
|
31
13
|
* returned; replaying returns what was stored. A read the recording never made
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawl-snapshot.d.ts","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"crawl-snapshot.d.ts","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EACN,2BAA2B,EAC3B,KAAK,kBAAkB,EACvB,MAAM,mBAAmB,CAAA;AAE1B,OAAO,EAAE,2BAA2B,EAAE,KAAK,kBAAkB,EAAE,CAAA;AAsB/D,MAAM,MAAM,oBAAoB,GAAG;IAClC,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAA;IACZ,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC3B,CAAA;AAID,wBAAgB,iBAAiB,CAAC,CAAC,EAClC,KAAK,EAAE,kBAAkB,GAAG,IAAI,GAAG,SAAS,EAC5C,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACrB,OAAO,EAAE,oBAAoB,cAmB7B;AAED;;;;;GAKG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC3C,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,CAAC,CAeZ;AAED,8EAA8E;AAC9E,wBAAgB,oBAAoB,IAAI,kBAAkB,GAAG,IAAI,CAGhE"}
|
package/dist/crawl-snapshot.js
CHANGED
|
@@ -2,8 +2,8 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
|
|
2
2
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
4
|
import { sha256Hex } from './hash.js';
|
|
5
|
-
|
|
6
|
-
export
|
|
5
|
+
import { CRAWL_SNAPSHOT_NAME_PATTERN } from './schema/index.js';
|
|
6
|
+
export { CRAWL_SNAPSHOT_NAME_PATTERN };
|
|
7
7
|
const snapshot = new AsyncLocalStorage();
|
|
8
8
|
export function withCrawlSnapshot(input, run, options) {
|
|
9
9
|
if (!input)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawl-snapshot.js","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"crawl-snapshot.js","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EACN,2BAA2B,EAE3B,MAAM,mBAAmB,CAAA;AAE1B,OAAO,EAAE,2BAA2B,EAA2B,CAAA;AA6B/D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAiB,CAAA;AAEvD,MAAM,UAAU,iBAAiB,CAChC,KAA4C,EAC5C,GAAqB,EACrB,OAA6B;IAE7B,IAAI,CAAC,KAAK;QAAE,OAAO,GAAG,EAAE,CAAA;IACxB,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAC1D,CAAC;IACD,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,gCAAgC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IAC9D,CAAC;IAED,OAAO,QAAQ,CAAC,GAAG,CAClB;QACC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;KACtB,EACD,GAAG,CACH,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,GAAW,EACX,IAAsB;IAEtB,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAClC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,EAAE,CAAA;IAE1B,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAA;IAC7E,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;QAC7D,IAAI,MAAM,KAAK,IAAI;YAAE,OAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,CAAkB,CAAC,KAAK,CAAA;QACtE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAA;IAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAClD,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IACrD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,oBAAoB;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAClC,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AAChE,CAAC","sourcesContent":["import { AsyncLocalStorage } from 'node:async_hooks'\nimport { mkdir, readFile, writeFile } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { sha256Hex } from './hash.js'\nimport type { Logger } from './logger.js'\nimport {\n\tCRAWL_SNAPSHOT_NAME_PATTERN,\n\ttype CrawlSnapshotInput\n} from './schema/index.js'\n\nexport { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput }\n\n/**\n * A crawl recorded once and replayed, for benchmarks. Two runs of a pipeline\n * then read the same pages, so a difference between them is the code: not a\n * site that published a page in between, a 429, or a host that timed out on\n * one run and not the other.\n *\n * Recorded: the network reads - pages as `fetchPageAsMarkdown` returns them,\n * robots.txt and sitemaps - plus anything the caller routes through\n * `throughCrawlSnapshot`. Everything else runs live on every replay, which is\n * the point.\n *\n * Set per job through `withCrawlSnapshot`, never globally: a worker drains\n * other tenants' jobs on the same lane. Never in production.\n */\n\ntype CrawlSnapshot = CrawlSnapshotInput & {\n\tdirectory: string\n\tlogger: Logger | undefined\n}\n\nexport type CrawlSnapshotOptions = {\n\t/** Directory the snapshots live under. */\n\troot: string\n\t/** Told about replay misses. */\n\tlogger?: Logger | undefined\n}\n\nconst snapshot = new AsyncLocalStorage<CrawlSnapshot>()\n\nexport function withCrawlSnapshot<T>(\n\tinput: CrawlSnapshotInput | null | undefined,\n\trun: () => Promise<T>,\n\toptions: CrawlSnapshotOptions\n) {\n\tif (!input) return run()\n\tif (process.env.NODE_ENV === 'production') {\n\t\tthrow new Error('A crawl snapshot is a local bench tool')\n\t}\n\tif (!CRAWL_SNAPSHOT_NAME_PATTERN.test(input.name)) {\n\t\tthrow new Error(`Invalid crawl snapshot name: ${input.name}`)\n\t}\n\n\treturn snapshot.run(\n\t\t{\n\t\t\tmode: input.mode,\n\t\t\tname: input.name,\n\t\t\tdirectory: resolve(options.root, input.name),\n\t\t\tlogger: options.logger\n\t\t},\n\t\trun\n\t)\n}\n\n/**\n * `read` through the active snapshot. Recording stores what the network\n * returned; replaying returns what was stored. A read the recording never made\n * (a code change that discovers a new URL) goes to the network and is added,\n * with a warning, so the snapshot stays complete without hiding the drift.\n */\nexport async function throughCrawlSnapshot<T>(\n\tkey: string,\n\tread: () => Promise<T>\n): Promise<T> {\n\tconst active = snapshot.getStore()\n\tif (!active) return read()\n\n\tconst file = resolve(active.directory, `${sha256Hex(key).slice(0, 32)}.json`)\n\tif (active.mode === 'replay') {\n\t\tconst stored = await readFile(file, 'utf8').catch(() => null)\n\t\tif (stored !== null) return (JSON.parse(stored) as { value: T }).value\n\t\tactive.logger?.warn('crawl-snapshot.miss', { key })\n\t}\n\n\tconst value = await read()\n\tawait mkdir(active.directory, { recursive: true })\n\tawait writeFile(file, JSON.stringify({ key, value }))\n\treturn value\n}\n\n/** The snapshot this job reads or writes, to hand on to the job it queues. */\nexport function currentCrawlSnapshot(): CrawlSnapshotInput | null {\n\tconst active = snapshot.getStore()\n\treturn active ? { mode: active.mode, name: active.name } : null\n}\n"]}
|
package/dist/fetch-page.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ApiError } from 'hanzio/api-wrapper';
|
|
2
2
|
import { type CloudflareCredentials } from './client.js';
|
|
3
|
+
import { type CrawlSnapshotInput, type CrawlSnapshotOptions } from './crawl-snapshot.js';
|
|
3
4
|
import { type CreateConcurrencyLimit } from './limit.js';
|
|
4
5
|
import { type Logger } from './logger.js';
|
|
5
6
|
/** Stored markdown ceiling; a page over this is a listing or an upload, not knowledge. */
|
|
@@ -99,7 +100,18 @@ export type WebsiteToMarkdownOptions = {
|
|
|
99
100
|
createLimit?: CreateConcurrencyLimit;
|
|
100
101
|
/** Workers AI client errors other than 429s, for an error tracker. */
|
|
101
102
|
onApiError?: (error: ApiError) => void;
|
|
103
|
+
/**
|
|
104
|
+
* Pass Workers AI timeouts to `onApiError`. A timed-out page falls through
|
|
105
|
+
* to the next tier, so `false` keeps import bursts out of error tracking.
|
|
106
|
+
* Default true.
|
|
107
|
+
*/
|
|
108
|
+
reportTimeouts?: boolean;
|
|
102
109
|
logger?: Logger;
|
|
110
|
+
/**
|
|
111
|
+
* Where the returned `withCrawlSnapshot` keeps snapshots. Its logger
|
|
112
|
+
* defaults to `logger`.
|
|
113
|
+
*/
|
|
114
|
+
crawlSnapshot?: CrawlSnapshotOptions;
|
|
103
115
|
/** The OCR tier for a PDF Workers AI returned nothing for. */
|
|
104
116
|
parsePdf?: PdfParser;
|
|
105
117
|
/**
|
|
@@ -128,6 +140,10 @@ export declare function createWebsiteToMarkdown(options: WebsiteToMarkdownOption
|
|
|
128
140
|
}>;
|
|
129
141
|
convertFileWithWorkersAi: (input: import("./client.js").ConvertFileInput) => Promise<string | null>;
|
|
130
142
|
renderUrlToMarkdown: (url: string, signal?: AbortSignal) => Promise<string | null>;
|
|
143
|
+
/** The SSRF guard the converter runs, bound to its `logger`. */
|
|
144
|
+
isHostAllowed: (hostname: string) => Promise<boolean>;
|
|
145
|
+
/** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */
|
|
146
|
+
withCrawlSnapshot<T>(input: CrawlSnapshotInput | null | undefined, run: () => Promise<T>): Promise<T>;
|
|
131
147
|
};
|
|
132
148
|
export type WebsiteToMarkdown = ReturnType<typeof createWebsiteToMarkdown>;
|
|
133
149
|
//# sourceMappingURL=fetch-page.d.ts.map
|
package/dist/fetch-page.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-page.d.ts","sourceRoot":"","sources":["../src/fetch-page.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAElD,OAAO,EACN,KAAK,qBAAqB,EAG1B,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"fetch-page.d.ts","sourceRoot":"","sources":["../src/fetch-page.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAElD,OAAO,EACN,KAAK,qBAAqB,EAG1B,MAAM,aAAa,CAAA;AAMpB,OAAO,EACN,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EAIzB,MAAM,qBAAqB,CAAA;AAQ5B,OAAO,EAAE,KAAK,sBAAsB,EAAoB,MAAM,YAAY,CAAA;AAC1E,OAAO,EAAsB,KAAK,MAAM,EAAgB,MAAM,aAAa,CAAA;AAsB3E,0FAA0F;AAC1F,eAAO,MAAM,kBAAkB,SAAU,CAAA;AAEzC,MAAM,MAAM,YAAY,GACrB,IAAI,GACJ,WAAW,GACX,MAAM,GACN,SAAS,GACT,WAAW,GACX,aAAa,CAAA;AAChB,MAAM,MAAM,WAAW,GACpB,qBAAqB,GACrB,YAAY,GACZ,mBAAmB,GACnB,cAAc,GACd,iBAAiB,CAAA;AAEpB,MAAM,MAAM,WAAW,GAAG;IACzB,wEAAwE;IACxE,GAAG,EAAE,MAAM,CAAA;IACX,mDAAmD;IACnD,QAAQ,EAAE,MAAM,CAAA;IAChB,iFAAiF;IACjF,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,YAAY,CAAA;IACrB,MAAM,EAAE,WAAW,GAAG,IAAI,CAAA;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAA;IAChB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,yDAAyD;IACzD,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,wCAAwC;IACxC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,iIAAiI;IACjI,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IACrB,oEAAoE;IACpE,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC3C,uDAAuD;IACvD,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,sHAAsH;IACtH,MAAM,EAAE,OAAO,EAAE,CAAA;IACjB;;;;OAIG;IACH,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,0DAA0D;IAC1D,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxC;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CACpC,CAAA;AAYD,MAAM,MAAM,QAAQ,GAAG;IACtB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxC,OAAO,EAAE,YAAY,CAAA;IACrB,MAAM,EAAE,WAAW,GAAG,IAAI,CAAA;IAC1B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACpC,gFAAgF;IAChF,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CAChC,CAAA;AAyTD,MAAM,MAAM,cAAc,GAAG;IAC5B,GAAG,EAAE,MAAM,CAAA;IACX,8EAA8E;IAC9E,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxC,sDAAsD;IACtD,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IAC5B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACnC,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACpC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAChC,CAAA;AAED,MAAM,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE;IAC/B,KAAK,EAAE,WAAW,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAChC,KAAK,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAA;AAE5D,MAAM,MAAM,wBAAwB,GAAG;IACtC,4EAA4E;IAC5E,UAAU,EAAE,qBAAqB,CAAA;IACjC;;;;OAIG;IACH,WAAW,CAAC,EAAE,sBAAsB,CAAA;IACpC,sEAAsE;IACtE,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAA;IACtC;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAA;IACpC,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,SAAS,CAAA;IACpB;;;OAGG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CACtD,CAAA;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,wBAAwB;iCA8UhE,cAAc,KACnB,OAAO,CAAC,WAAW,CAAC;kCA7QoB;QAC1C,KAAK,EAAE,WAAW,CAAA;QAClB,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;QACtC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;KAChC,KAAG,OAAO,CAAC,QAAQ,CAAC;+BA4amB,MAAM;;;;;;IAiB7C,gEAAgE;8BA7gBtC,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC;IA+gBrD,+DAA+D;sBAC7C,CAAC,SACX,kBAAkB,GAAG,IAAI,GAAG,SAAS,OACvC,MAAM,OAAO,CAAC,CAAC,CAAC;EAYvB;AAED,MAAM,MAAM,iBAAiB,GAAG,UAAU,CAAC,OAAO,uBAAuB,CAAC,CAAA"}
|
package/dist/fetch-page.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { parseRetryAfter } from 'hanzio';
|
|
2
2
|
import { CloudflareRateLimitedError, createCloudflareClient } from './client.js';
|
|
3
3
|
import { classifyContentType, convertibleDocumentType } from './content-type.js';
|
|
4
|
-
import { currentCrawlSnapshot, throughCrawlSnapshot } from './crawl-snapshot.js';
|
|
4
|
+
import { currentCrawlSnapshot, throughCrawlSnapshot, withCrawlSnapshot } from './crawl-snapshot.js';
|
|
5
5
|
import { isApplicationShell, isBlankMarkdown, looksLikeHtml } from './detect-markdown.js';
|
|
6
6
|
import { sha256Hex } from './hash.js';
|
|
7
7
|
import { extractHtmlMetadata } from './html-metadata.js';
|
|
@@ -305,6 +305,7 @@ export function createWebsiteToMarkdown(options) {
|
|
|
305
305
|
credentials: options.cloudflare,
|
|
306
306
|
createLimit,
|
|
307
307
|
onApiError: options.onApiError,
|
|
308
|
+
reportTimeouts: options.reportTimeouts,
|
|
308
309
|
logger
|
|
309
310
|
});
|
|
310
311
|
const hostLimits = createHostLimits(createLimit);
|
|
@@ -738,7 +739,21 @@ export function createWebsiteToMarkdown(options) {
|
|
|
738
739
|
convertDocumentBytes,
|
|
739
740
|
urlToMarkdown,
|
|
740
741
|
convertFileWithWorkersAi: client.convertFileWithWorkersAi,
|
|
741
|
-
renderUrlToMarkdown: client.renderUrlToMarkdown
|
|
742
|
+
renderUrlToMarkdown: client.renderUrlToMarkdown,
|
|
743
|
+
/** The SSRF guard the converter runs, bound to its `logger`. */
|
|
744
|
+
isHostAllowed: allowHost,
|
|
745
|
+
/** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */
|
|
746
|
+
withCrawlSnapshot(input, run) {
|
|
747
|
+
if (!input)
|
|
748
|
+
return run();
|
|
749
|
+
if (!options.crawlSnapshot) {
|
|
750
|
+
throw new Error('Set crawlSnapshot.root to use crawl snapshots');
|
|
751
|
+
}
|
|
752
|
+
return withCrawlSnapshot(input, run, {
|
|
753
|
+
root: options.crawlSnapshot.root,
|
|
754
|
+
logger: options.crawlSnapshot.logger ?? logger
|
|
755
|
+
});
|
|
756
|
+
}
|
|
742
757
|
};
|
|
743
758
|
}
|
|
744
759
|
//# sourceMappingURL=fetch-page.js.map
|