website-to-markdown 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +14 -6
- package/dist/cli.js.map +1 -1
- package/package.json +3 -3
- package/src/cli.ts +22 -5
package/dist/cli.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { type CliIO } from 'hanzio/cli';
|
|
3
3
|
export declare const cli: import("hanzio/cli").Cli;
|
|
4
|
-
/** The bin
|
|
4
|
+
/** The bin. `--help` without a URL exits 1, as in 1.0. */
|
|
5
5
|
export declare function main(argv: readonly string[], io?: CliIO): Promise<number>;
|
|
6
6
|
//# sourceMappingURL=cli.d.ts.map
|
package/dist/cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAOA,OAAO,
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAOA,OAAO,EACN,KAAK,KAAK,EAKV,MAAM,YAAY,CAAA;AAInB,eAAO,MAAM,GAAG,0BAqFd,CAAA;AAQF,0DAA0D;AAC1D,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,mBAG7D"}
|
package/dist/cli.js
CHANGED
|
@@ -5,12 +5,22 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { readFile } from 'node:fs/promises';
|
|
7
7
|
import { parseEnv } from 'node:util';
|
|
8
|
-
import { CliError, defineCli, defineCommand } from 'hanzio/cli';
|
|
8
|
+
import { CliError, CliUsageError, defineCli, defineCommand } from 'hanzio/cli';
|
|
9
9
|
import { z } from 'zod';
|
|
10
10
|
import { createWebsiteToMarkdown } from './fetch-page.js';
|
|
11
11
|
export const cli = defineCli({
|
|
12
12
|
name: 'website-to-markdown',
|
|
13
|
+
// No URL prints the help and exits 1, as 1.0 did.
|
|
14
|
+
onError: (error, io) => {
|
|
15
|
+
if (error instanceof CliUsageError &&
|
|
16
|
+
error.message.split('\n').some((line) => line.startsWith('<url>: '))) {
|
|
17
|
+
io.stdout.write(error.help);
|
|
18
|
+
return 1;
|
|
19
|
+
}
|
|
20
|
+
},
|
|
13
21
|
commands: defineCommand({
|
|
22
|
+
// Unknown flags and extra positionals are ignored, as in 1.0.
|
|
23
|
+
strict: false,
|
|
14
24
|
description: 'Fetch a public URL as markdown. Needs CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.',
|
|
15
25
|
input: z.object({
|
|
16
26
|
url: z
|
|
@@ -70,12 +80,10 @@ function readEnvFile(path) {
|
|
|
70
80
|
throw new CliError(`Cannot read --env-file ${path}.`);
|
|
71
81
|
});
|
|
72
82
|
}
|
|
73
|
-
/** The bin
|
|
83
|
+
/** The bin. `--help` without a URL exits 1, as in 1.0. */
|
|
74
84
|
export async function main(argv, io) {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
await cli.run(['--help'], io);
|
|
78
|
-
return 1;
|
|
85
|
+
const code = await cli.run(argv, io);
|
|
86
|
+
return code === 0 && argv.every((arg) => arg.startsWith('-')) ? 1 : code;
|
|
79
87
|
}
|
|
80
88
|
if (import.meta.main)
|
|
81
89
|
process.exitCode = await main(process.argv.slice(2));
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,EAEN,QAAQ,EACR,aAAa,EACb,SAAS,EACT,aAAa,EACb,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAEzD,MAAM,CAAC,MAAM,GAAG,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,qBAAqB;IAC3B,kDAAkD;IAClD,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACtB,IACC,KAAK,YAAY,aAAa;YAC9B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EACnE,CAAC;YACF,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YAC3B,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IACD,QAAQ,EAAE,aAAa,CAAC;QACvB,8DAA8D;QAC9D,MAAM,EAAE,KAAK;QACb,WAAW,EACV,uFAAuF;QACxF,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,GAAG,EAAE,CAAC;iBACJ,MAAM,EAAE;iBACR,QAAQ,CAAC,gCAAgC,CAAC;iBAC1C,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC;iBACP,OAAO,EAAE;iBACT,OAAO,CAAC,IAAI,CAAC;iBACb,QAAQ,CACR,kEAAkE,CAClE;YACF,OAAO,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,EAAE;iBACV,QAAQ,CAAC,6DAA6D,CAAC;SACzE,CAAC;QACF,GAAG,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE;YAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;YAChE,MAAM,oBAAoB,GACzB,GAAG,CAAC,GAAG,CAAC,oBAAoB,IAAI,IAAI,CAAC,oBAAoB,CAAA;YAC1D,MAAM,qBAAqB,GAC1B,GAAG,CAAC,GAAG,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB,CAAA;YAC5D,IAAI,CAAC,oBAAoB,IAAI,CAAC,qBAAqB;gBAClD,MAAM,IAAI,QAAQ,CACjB,qDAAqD,CACrD,CAAA;YAEF,MAAM,EAAE,mBAAmB,EAAE,GAAG,uBAAuB,CAAC;gBACvD,UAAU,EAAE;oBACX,QAAQ,EAAE,oBAAoB;oBAC9B,SAAS,EAAE,qBAAqB;iBAChC;aACD,CAAC,CAAA;YAEF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,CAAA;YAEtC,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAA;YAEvD,GAAG,CAAC,MAAM,CAAC,KAAK,CACf,GAAG,IAAI,CAAC,SAAS,CAChB;gBACC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAChC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBACnC,UAAU,EAAE,IAAI,CAAC,UAAU;aAC3B,EACD,IAAI,EACJ,CAAC,CACD,IAAI,CACL,CAAA;YAED,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;gBAAE,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAA;YAC3C,OAAO,IAAI,CAAC,QAAQ,CAAA;QACrB,CAAC;KACD,CAAC;CACF,CAAC,CAAA;AAEF,SAAS,WAAW,CAAC,IAAY;IAChC,OAAO,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;QACxC,MAAM,IAAI,QAAQ,CAAC,0BAA0B,IAAI,GAAG,CAAC,CAAA;IACtD,CAAC,CAAC,CAAA;AACH,CAAC;AAED,0DAA0D;AAC1D,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAuB,EAAE,EAAU;IAC7D,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IACpC,OAAO,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;AACzE,CAAC;AAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI;IAAE,OAAO,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA","sourcesContent":["#!/usr/bin/env bun\n/**\n * Fetch a public URL as markdown and print the result. Page metadata goes to\n * stderr; markdown goes to stdout.\n */\nimport { readFile } from 'node:fs/promises'\nimport { parseEnv } from 'node:util'\nimport {\n\ttype CliIO,\n\tCliError,\n\tCliUsageError,\n\tdefineCli,\n\tdefineCommand\n} from 'hanzio/cli'\nimport { z } from 'zod'\nimport { createWebsiteToMarkdown } from './fetch-page.js'\n\nexport const cli = defineCli({\n\tname: 'website-to-markdown',\n\t// No URL prints the help and exits 1, as 1.0 did.\n\tonError: (error, io) => {\n\t\tif (\n\t\t\terror instanceof CliUsageError &&\n\t\t\terror.message.split('\\n').some((line) => line.startsWith('<url>: '))\n\t\t) {\n\t\t\tio.stdout.write(error.help)\n\t\t\treturn 1\n\t\t}\n\t},\n\tcommands: defineCommand({\n\t\t// Unknown flags and extra positionals are ignored, as in 1.0.\n\t\tstrict: false,\n\t\tdescription:\n\t\t\t'Fetch a public URL as markdown. Needs CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.',\n\t\tinput: z.object({\n\t\t\turl: z\n\t\t\t\t.string()\n\t\t\t\t.describe('e.g. https://example.com/about')\n\t\t\t\t.meta({ positional: true }),\n\t\t\trender: z\n\t\t\t\t.boolean()\n\t\t\t\t.default(true)\n\t\t\t\t.describe(\n\t\t\t\t\t'Use the Browser Rendering tier; --no-render keeps to HTML tiers.'\n\t\t\t\t),\n\t\t\tenvFile: z\n\t\t\t\t.string()\n\t\t\t\t.optional()\n\t\t\t\t.describe('Read CLOUDFLARE_* from a dotenv file; the environment wins.')\n\t\t}),\n\t\trun: async ({ url, render, envFile }, ctx) => {\n\t\t\tconst file = envFile ? parseEnv(await readEnvFile(envFile)) : {}\n\t\t\tconst CLOUDFLARE_API_TOKEN =\n\t\t\t\tctx.env.CLOUDFLARE_API_TOKEN || file.CLOUDFLARE_API_TOKEN\n\t\t\tconst CLOUDFLARE_ACCOUNT_ID =\n\t\t\t\tctx.env.CLOUDFLARE_ACCOUNT_ID || file.CLOUDFLARE_ACCOUNT_ID\n\t\t\tif (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID)\n\t\t\t\tthrow new CliError(\n\t\t\t\t\t'Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.'\n\t\t\t\t)\n\n\t\t\tconst { fetchPageAsMarkdown } = createWebsiteToMarkdown({\n\t\t\t\tcloudflare: {\n\t\t\t\t\tapiToken: CLOUDFLARE_API_TOKEN,\n\t\t\t\t\taccountId: CLOUDFLARE_ACCOUNT_ID\n\t\t\t\t}\n\t\t\t})\n\n\t\t\tctx.stderr.write(`Fetching ${url}…\\n`)\n\n\t\t\tconst page = await fetchPageAsMarkdown({ url, render })\n\n\t\t\tctx.stderr.write(\n\t\t\t\t`${JSON.stringify(\n\t\t\t\t\t{\n\t\t\t\t\t\toutcome: page.outcome,\n\t\t\t\t\t\tstatus: page.status,\n\t\t\t\t\t\tsource: page.source,\n\t\t\t\t\t\turl: page.url,\n\t\t\t\t\t\tfinalUrl: page.finalUrl,\n\t\t\t\t\t\tcontentType: page.contentType,\n\t\t\t\t\t\tetag: page.etag,\n\t\t\t\t\t\tlastModified: page.lastModified,\n\t\t\t\t\t\ttitle: page.title,\n\t\t\t\t\t\tlang: page.lang,\n\t\t\t\t\t\tcanonical: page.canonical,\n\t\t\t\t\t\tmetaDescription: page.metaDescription,\n\t\t\t\t\t\tjsonLdBlocks: page.jsonLd.length,\n\t\t\t\t\t\tlinks: page.links.length,\n\t\t\t\t\t\timages: page.images.length,\n\t\t\t\t\t\tmarkdownChars: page.markdown.length,\n\t\t\t\t\t\tdurationMs: page.durationMs\n\t\t\t\t\t},\n\t\t\t\t\tnull,\n\t\t\t\t\t2\n\t\t\t\t)}\\n`\n\t\t\t)\n\n\t\t\tif (page.outcome !== 'ok') ctx.exitCode = 1\n\t\t\treturn page.markdown\n\t\t}\n\t})\n})\n\nfunction readEnvFile(path: string) {\n\treturn readFile(path, 'utf8').catch(() => {\n\t\tthrow new CliError(`Cannot read --env-file ${path}.`)\n\t})\n}\n\n/** The bin. `--help` without a URL exits 1, as in 1.0. */\nexport async function main(argv: readonly string[], io?: CliIO) {\n\tconst code = await cli.run(argv, io)\n\treturn code === 0 && argv.every((arg) => arg.startsWith('-')) ? 1 : code\n}\n\nif (import.meta.main) process.exitCode = await main(process.argv.slice(2))\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "website-to-markdown",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"description": "Turn URLs, HTML and files into markdown, with sitemap discovery and per-host limits.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -66,7 +66,7 @@
|
|
|
66
66
|
"fflate": "^0.8.2"
|
|
67
67
|
},
|
|
68
68
|
"peerDependencies": {
|
|
69
|
-
"hanzio": "^3.
|
|
69
|
+
"hanzio": "^3.3.0",
|
|
70
70
|
"zod": "^4.0.0"
|
|
71
71
|
},
|
|
72
72
|
"devDependencies": {
|
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
"@biomejs/biome": "^2.2.4",
|
|
75
75
|
"@types/bun": "^1.2.0",
|
|
76
76
|
"@types/node": "^24",
|
|
77
|
-
"hanzio": "^3.
|
|
77
|
+
"hanzio": "^3.3.0",
|
|
78
78
|
"publint": "^0.3.12",
|
|
79
79
|
"typescript": "^5.9.2",
|
|
80
80
|
"zod": "^4.4.3"
|
package/src/cli.ts
CHANGED
|
@@ -5,13 +5,31 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { readFile } from 'node:fs/promises'
|
|
7
7
|
import { parseEnv } from 'node:util'
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
type CliIO,
|
|
10
|
+
CliError,
|
|
11
|
+
CliUsageError,
|
|
12
|
+
defineCli,
|
|
13
|
+
defineCommand
|
|
14
|
+
} from 'hanzio/cli'
|
|
9
15
|
import { z } from 'zod'
|
|
10
16
|
import { createWebsiteToMarkdown } from './fetch-page.js'
|
|
11
17
|
|
|
12
18
|
export const cli = defineCli({
|
|
13
19
|
name: 'website-to-markdown',
|
|
20
|
+
// No URL prints the help and exits 1, as 1.0 did.
|
|
21
|
+
onError: (error, io) => {
|
|
22
|
+
if (
|
|
23
|
+
error instanceof CliUsageError &&
|
|
24
|
+
error.message.split('\n').some((line) => line.startsWith('<url>: '))
|
|
25
|
+
) {
|
|
26
|
+
io.stdout.write(error.help)
|
|
27
|
+
return 1
|
|
28
|
+
}
|
|
29
|
+
},
|
|
14
30
|
commands: defineCommand({
|
|
31
|
+
// Unknown flags and extra positionals are ignored, as in 1.0.
|
|
32
|
+
strict: false,
|
|
15
33
|
description:
|
|
16
34
|
'Fetch a public URL as markdown. Needs CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.',
|
|
17
35
|
input: z.object({
|
|
@@ -90,11 +108,10 @@ function readEnvFile(path: string) {
|
|
|
90
108
|
})
|
|
91
109
|
}
|
|
92
110
|
|
|
93
|
-
/** The bin
|
|
111
|
+
/** The bin. `--help` without a URL exits 1, as in 1.0. */
|
|
94
112
|
export async function main(argv: readonly string[], io?: CliIO) {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
return 1
|
|
113
|
+
const code = await cli.run(argv, io)
|
|
114
|
+
return code === 0 && argv.every((arg) => arg.startsWith('-')) ? 1 : code
|
|
98
115
|
}
|
|
99
116
|
|
|
100
117
|
if (import.meta.main) process.exitCode = await main(process.argv.slice(2))
|