webforai 2.1.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/README.md +86 -4
  2. package/dist/adapters/index.js +58 -0
  3. package/dist/adapters/registry.js +149 -0
  4. package/dist/adapters/sites/declarative.js +178 -0
  5. package/dist/adapters/sites/hackernews.js +94 -0
  6. package/dist/adapters/sites/youtube.js +165 -0
  7. package/dist/adapters/types.js +0 -0
  8. package/dist/bin.js +4142 -549
  9. package/dist/cjs/adapters/index.js +86 -0
  10. package/dist/cjs/adapters/registry.js +178 -0
  11. package/dist/cjs/adapters/sites/declarative.js +202 -0
  12. package/dist/cjs/adapters/sites/hackernews.js +117 -0
  13. package/dist/cjs/adapters/sites/youtube.js +191 -0
  14. package/dist/cjs/adapters/types.js +16 -0
  15. package/dist/cjs/constants.js +2 -2
  16. package/dist/cjs/detect-client-shell.js +78 -0
  17. package/dist/cjs/extract-meta-refresh.js +71 -0
  18. package/dist/cjs/extractors/index.js +5 -0
  19. package/dist/cjs/extractors/lib/constants.js +186 -0
  20. package/dist/cjs/extractors/lib/sanitize.js +429 -0
  21. package/dist/cjs/extractors/lib/score.js +264 -0
  22. package/dist/cjs/extractors/lib/terminators.js +131 -0
  23. package/dist/cjs/extractors/lib/ui-chrome.js +119 -0
  24. package/dist/cjs/extractors/pipeExtractors.js +12 -9
  25. package/dist/cjs/extractors/presets/auto.js +44 -0
  26. package/dist/cjs/extractors/presets/minimal-filter.js +27 -48
  27. package/dist/cjs/extractors/presets/takumi.js +84 -122
  28. package/dist/cjs/html-to-markdown.js +44 -7
  29. package/dist/cjs/html-to-mdast.js +10 -1
  30. package/dist/cjs/index.js +43 -2
  31. package/dist/cjs/link-replacer.js +19 -13
  32. package/dist/cjs/loaders/cf-puppeteer.js +14 -7
  33. package/dist/cjs/loaders/fetch.js +14 -2
  34. package/dist/cjs/loaders/playwright.js +32 -30
  35. package/dist/cjs/loaders/puppeteer.js +14 -7
  36. package/dist/cjs/md-splitter.js +24 -11
  37. package/dist/cjs/mdast-handlers/custom-a-handler.js +25 -9
  38. package/dist/cjs/mdast-handlers/custom-code-handler.js +18 -7
  39. package/dist/cjs/mdast-handlers/custom-div-handler.js +12 -33
  40. package/dist/cjs/mdast-handlers/custom-img-handler.js +23 -1
  41. package/dist/cjs/mdast-handlers/custom-table-handler.js +16 -0
  42. package/dist/cjs/mdast-handlers/definition-list-handler.js +110 -0
  43. package/dist/cjs/mdast-handlers/math-handler.js +12 -3
  44. package/dist/cjs/mdast-handlers/sup-sub-handler.js +39 -0
  45. package/dist/cjs/mdast-to-markdown.js +2 -5
  46. package/dist/cjs/metadata/index.js +223 -0
  47. package/dist/cjs/normalize/index.js +177 -0
  48. package/dist/cjs/platform/client.js +204 -0
  49. package/dist/cjs/platform/error.js +36 -0
  50. package/dist/cjs/platform/fetch.js +37 -0
  51. package/dist/cjs/platform/index.js +44 -0
  52. package/dist/cjs/platform/types.js +40 -0
  53. package/dist/cjs/utils/hast-fast.js +284 -0
  54. package/dist/constants.js +2 -2
  55. package/dist/detect-client-shell.js +53 -0
  56. package/dist/extract-meta-refresh.js +47 -0
  57. package/dist/extractors/index.js +3 -0
  58. package/dist/extractors/lib/constants.js +150 -0
  59. package/dist/extractors/lib/sanitize.js +419 -0
  60. package/dist/extractors/lib/score.js +242 -0
  61. package/dist/extractors/lib/terminators.js +107 -0
  62. package/dist/extractors/lib/ui-chrome.js +96 -0
  63. package/dist/extractors/pipeExtractors.js +12 -9
  64. package/dist/extractors/presets/auto.js +20 -0
  65. package/dist/extractors/presets/minimal-filter.js +35 -47
  66. package/dist/extractors/presets/takumi.js +91 -122
  67. package/dist/html-to-markdown.js +42 -6
  68. package/dist/html-to-mdast.js +10 -1
  69. package/dist/index.js +45 -2
  70. package/dist/link-replacer.js +19 -13
  71. package/dist/loaders/cf-puppeteer.js +15 -8
  72. package/dist/loaders/fetch.js +14 -2
  73. package/dist/loaders/playwright.js +32 -30
  74. package/dist/loaders/puppeteer.js +15 -8
  75. package/dist/md-splitter.js +24 -11
  76. package/dist/mdast-handlers/custom-a-handler.js +25 -9
  77. package/dist/mdast-handlers/custom-code-handler.js +17 -7
  78. package/dist/mdast-handlers/custom-div-handler.js +12 -33
  79. package/dist/mdast-handlers/custom-img-handler.js +23 -1
  80. package/dist/mdast-handlers/custom-table-handler.js +16 -0
  81. package/dist/mdast-handlers/definition-list-handler.js +87 -0
  82. package/dist/mdast-handlers/math-handler.js +12 -3
  83. package/dist/mdast-handlers/sup-sub-handler.js +16 -0
  84. package/dist/mdast-to-markdown.js +2 -5
  85. package/dist/metadata/index.js +198 -0
  86. package/dist/normalize/index.js +159 -0
  87. package/dist/platform/client.js +182 -0
  88. package/dist/platform/error.js +13 -0
  89. package/dist/platform/fetch.js +14 -0
  90. package/dist/platform/index.js +23 -0
  91. package/dist/platform/types.js +13 -0
  92. package/dist/types/adapters/index.d.ts +24 -0
  93. package/dist/types/adapters/index.js.map +1 -0
  94. package/dist/types/adapters/registry.d.ts +35 -0
  95. package/dist/types/adapters/registry.js.map +1 -0
  96. package/dist/types/adapters/sites/declarative.d.ts +13 -0
  97. package/dist/types/adapters/sites/declarative.js.map +1 -0
  98. package/dist/types/adapters/sites/hackernews.d.ts +13 -0
  99. package/dist/types/adapters/sites/hackernews.js.map +1 -0
  100. package/dist/types/adapters/sites/youtube.d.ts +31 -0
  101. package/dist/types/adapters/sites/youtube.js.map +1 -0
  102. package/dist/types/adapters/types.d.ts +99 -0
  103. package/dist/types/adapters/types.js.map +1 -0
  104. package/dist/types/cli/bin.js.map +1 -1
  105. package/dist/types/cli/commands/jobs/files.d.ts +28 -0
  106. package/dist/types/cli/commands/jobs/files.js.map +1 -0
  107. package/dist/types/cli/commands/jobs/llms-txt.d.ts +33 -0
  108. package/dist/types/cli/commands/jobs/llms-txt.js.map +1 -0
  109. package/dist/types/cli/commands/jobs/options.d.ts +58 -0
  110. package/dist/types/cli/commands/jobs/options.js.map +1 -0
  111. package/dist/types/cli/commands/jobs/run.d.ts +44 -0
  112. package/dist/types/cli/commands/jobs/run.js.map +1 -0
  113. package/dist/types/cli/commands/skill/content.d.ts +9 -0
  114. package/dist/types/cli/commands/skill/content.js.map +1 -0
  115. package/dist/types/cli/commands/skill/index.d.ts +19 -0
  116. package/dist/types/cli/commands/skill/index.js.map +1 -0
  117. package/dist/types/cli/commands/webforai/interactive.d.ts +7 -0
  118. package/dist/types/cli/commands/webforai/interactive.js.map +1 -0
  119. package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -1
  120. package/dist/types/cli/commands/webforai/options.d.ts +62 -0
  121. package/dist/types/cli/commands/webforai/options.js.map +1 -0
  122. package/dist/types/cli/commands/webforai/run.d.ts +10 -0
  123. package/dist/types/cli/commands/webforai/run.js.map +1 -0
  124. package/dist/types/cli/constants.d.ts +14 -4
  125. package/dist/types/cli/constants.js.map +1 -1
  126. package/dist/types/cli/helpers/selectExtractMode.js.map +1 -1
  127. package/dist/types/cli/helpers/selectLoader.js.map +1 -1
  128. package/dist/types/cli/utils.d.ts +8 -1
  129. package/dist/types/cli/utils.js.map +1 -1
  130. package/dist/types/constants.d.ts +7 -1
  131. package/dist/types/constants.js.map +1 -1
  132. package/dist/types/detect-client-shell.d.ts +24 -0
  133. package/dist/types/detect-client-shell.js.map +1 -0
  134. package/dist/types/extract-meta-refresh.d.ts +29 -0
  135. package/dist/types/extract-meta-refresh.js.map +1 -0
  136. package/dist/types/extractors/index.d.ts +1 -0
  137. package/dist/types/extractors/index.js.map +1 -1
  138. package/dist/types/extractors/lib/constants.d.ts +81 -0
  139. package/dist/types/extractors/lib/constants.js.map +1 -0
  140. package/dist/types/extractors/lib/sanitize.d.ts +66 -0
  141. package/dist/types/extractors/lib/sanitize.js.map +1 -0
  142. package/dist/types/extractors/lib/score.d.ts +85 -0
  143. package/dist/types/extractors/lib/score.js.map +1 -0
  144. package/dist/types/extractors/lib/terminators.d.ts +26 -0
  145. package/dist/types/extractors/lib/terminators.js.map +1 -0
  146. package/dist/types/extractors/lib/ui-chrome.d.ts +18 -0
  147. package/dist/types/extractors/lib/ui-chrome.js.map +1 -0
  148. package/dist/types/extractors/pipeExtractors.d.ts +5 -0
  149. package/dist/types/extractors/pipeExtractors.js.map +1 -1
  150. package/dist/types/extractors/presets/auto.d.ts +30 -0
  151. package/dist/types/extractors/presets/auto.js.map +1 -0
  152. package/dist/types/extractors/presets/minimal-filter.d.ts +10 -2
  153. package/dist/types/extractors/presets/minimal-filter.js.map +1 -1
  154. package/dist/types/extractors/presets/takumi.d.ts +19 -3
  155. package/dist/types/extractors/presets/takumi.js.map +1 -1
  156. package/dist/types/extractors/types.d.ts +8 -0
  157. package/dist/types/html-to-markdown.d.ts +38 -0
  158. package/dist/types/html-to-markdown.js.map +1 -1
  159. package/dist/types/html-to-mdast.d.ts +11 -0
  160. package/dist/types/html-to-mdast.js.map +1 -1
  161. package/dist/types/index.d.ts +8 -3
  162. package/dist/types/index.js.map +1 -1
  163. package/dist/types/link-replacer.d.ts +3 -1
  164. package/dist/types/link-replacer.js.map +1 -1
  165. package/dist/types/loaders/cf-puppeteer.js.map +1 -1
  166. package/dist/types/loaders/fetch.d.ts +4 -0
  167. package/dist/types/loaders/fetch.js.map +1 -1
  168. package/dist/types/loaders/playwright.js.map +1 -1
  169. package/dist/types/loaders/puppeteer.js.map +1 -1
  170. package/dist/types/md-splitter.d.ts +1 -1
  171. package/dist/types/md-splitter.js.map +1 -1
  172. package/dist/types/mdast-handlers/custom-a-handler.js.map +1 -1
  173. package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
  174. package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
  175. package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
  176. package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
  177. package/dist/types/mdast-handlers/custom-table-handler.js.map +1 -1
  178. package/dist/types/mdast-handlers/definition-list-handler.d.ts +2 -0
  179. package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -0
  180. package/dist/types/mdast-handlers/math-handler.js.map +1 -1
  181. package/dist/types/mdast-handlers/sup-sub-handler.d.ts +10 -0
  182. package/dist/types/mdast-handlers/sup-sub-handler.js.map +1 -0
  183. package/dist/types/mdast-to-markdown.js.map +1 -1
  184. package/dist/types/metadata/index.d.ts +47 -0
  185. package/dist/types/metadata/index.js.map +1 -0
  186. package/dist/types/normalize/index.d.ts +53 -0
  187. package/dist/types/normalize/index.js.map +1 -0
  188. package/dist/types/platform/client.d.ts +55 -0
  189. package/dist/types/platform/client.js.map +1 -0
  190. package/dist/types/platform/error.d.ts +22 -0
  191. package/dist/types/platform/error.js.map +1 -0
  192. package/dist/types/platform/fetch.d.ts +36 -0
  193. package/dist/types/platform/fetch.js.map +1 -0
  194. package/dist/types/platform/index.d.ts +4 -0
  195. package/dist/types/platform/index.js.map +1 -0
  196. package/dist/types/platform/types.d.ts +164 -0
  197. package/dist/types/platform/types.js.map +1 -0
  198. package/dist/types/utils/hast-fast.d.ts +101 -0
  199. package/dist/types/utils/hast-fast.js.map +1 -0
  200. package/dist/utils/hast-fast.js +246 -0
  201. package/package.json +25 -7
  202. package/dist/types/cli/commands/webforai/index.d.ts +0 -6
  203. package/dist/types/cli/commands/webforai/index.js.map +0 -1
@@ -0,0 +1,204 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var client_exports = {};
20
+ __export(client_exports, {
21
+ DEFAULT_BASE_URL: () => DEFAULT_BASE_URL,
22
+ createPlatformClient: () => createPlatformClient
23
+ });
24
+ module.exports = __toCommonJS(client_exports);
25
+ var import_error = require("./error");
26
+ var import_fetch = require("./fetch");
27
+ var import_types = require("./types");
28
+ const DEFAULT_BASE_URL = "https://platform.webforai.dev";
29
+ const TERMINAL_STATES = /* @__PURE__ */ new Set(["completed", "failed"]);
30
+ const sleep = (ms, signal) => new Promise((resolve, reject) => {
31
+ if (signal?.aborted) {
32
+ reject(signal.reason instanceof Error ? signal.reason : new Error("aborted"));
33
+ return;
34
+ }
35
+ const timer = setTimeout(() => {
36
+ signal?.removeEventListener("abort", onAbort);
37
+ resolve();
38
+ }, ms);
39
+ const onAbort = () => {
40
+ clearTimeout(timer);
41
+ reject(signal?.reason instanceof Error ? signal.reason : new Error("aborted"));
42
+ };
43
+ signal?.addEventListener("abort", onAbort, { once: true });
44
+ });
45
+ const parseRetryAfter = (body, response) => {
46
+ const fromBody = body?.error?.retryAfter;
47
+ if (typeof fromBody === "number") {
48
+ return fromBody;
49
+ }
50
+ const header = Number(response.headers.get("retry-after"));
51
+ return Number.isFinite(header) && header > 0 ? header : void 0;
52
+ };
53
+ const dashboardHint = (status, dashboardUrl) => {
54
+ if (status === 401) {
55
+ return `create or check API keys at ${dashboardUrl}`;
56
+ }
57
+ if (status === 402) {
58
+ return `manage billing, credits and the spend cap at ${dashboardUrl}`;
59
+ }
60
+ return void 0;
61
+ };
62
+ const toApiError = (response, body, dashboardUrl) => {
63
+ const envelope = body;
64
+ const code = typeof envelope?.error?.code === "string" ? envelope.error.code : "invalid_response";
65
+ let message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
66
+ const hint = code === "invalid_response" ? void 0 : dashboardHint(response.status, dashboardUrl);
67
+ if (hint && !message.includes("/dashboard")) {
68
+ message = `${message} (${hint})`;
69
+ }
70
+ return new import_error.PlatformApiError(code, message, response.status, parseRetryAfter(body, response));
71
+ };
72
+ const createPlatformClient = (options = {}) => {
73
+ const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
74
+ const dashboardUrl = `${baseUrl}/dashboard`;
75
+ const fetchImpl = (0, import_fetch.resolveFetch)(options.fetch);
76
+ const request = async (path, init) => {
77
+ const headers = { accept: "application/json" };
78
+ if (init.body !== void 0) {
79
+ headers["content-type"] = "application/json";
80
+ }
81
+ if (init.auth) {
82
+ if (!options.apiKey) {
83
+ throw new import_error.PlatformApiError(
84
+ "missing_api_key",
85
+ `This endpoint needs an API key: pass \`apiKey\` to createPlatformClient (create one at ${dashboardUrl}; 1,000 free credits/month).`,
86
+ 0
87
+ );
88
+ }
89
+ headers.authorization = `Bearer ${options.apiKey}`;
90
+ }
91
+ const response = await fetchImpl(`${baseUrl}${path}`, {
92
+ method: init.method,
93
+ headers,
94
+ body: init.body === void 0 ? void 0 : JSON.stringify(init.body),
95
+ ...init.signal ? { signal: init.signal } : {}
96
+ });
97
+ const body = await response.json().catch(() => void 0);
98
+ if (!response.ok) {
99
+ throw toApiError(response, body, dashboardUrl);
100
+ }
101
+ if (body === void 0) {
102
+ throw new import_error.PlatformApiError("invalid_response", "response body is not JSON", response.status);
103
+ }
104
+ return body;
105
+ };
106
+ const getJob = (jobId, signal) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true, signal });
107
+ const getJobResults = (jobId, opts) => {
108
+ const query = opts?.cursor ? `?cursor=${encodeURIComponent(opts.cursor)}` : "";
109
+ return request(`/v1/jobs/${encodeURIComponent(jobId)}/results${query}`, { method: "GET", auth: true });
110
+ };
111
+ const resolveStored = async (resultUrl) => {
112
+ const response = await fetchImpl(resultUrl, { method: "GET" });
113
+ if (!response.ok) {
114
+ throw new import_error.PlatformApiError(
115
+ "artifact_fetch_failed",
116
+ `stored result fetch failed (HTTP ${response.status}) \u2014 result URLs expire after ~24h`,
117
+ response.status
118
+ );
119
+ }
120
+ return await response.json();
121
+ };
122
+ const getJobOrBackOff = async (jobId, signal, interval) => {
123
+ try {
124
+ return await getJob(jobId, signal);
125
+ } catch (error) {
126
+ if (!(error instanceof import_error.PlatformApiError && error.status === 429)) {
127
+ throw error;
128
+ }
129
+ await sleep(error.retryAfter !== void 0 ? error.retryAfter * 1e3 : interval, signal);
130
+ return void 0;
131
+ }
132
+ };
133
+ return {
134
+ scrape: (opts) => request("/v1/scrape", { method: "POST", body: opts, auth: true }),
135
+ scrapeAsync: (opts) => request("/v1/scrape", { method: "POST", body: { ...opts, async: true }, auth: true }),
136
+ batch: (opts) => request("/v1/batch", { method: "POST", body: opts, auth: true }),
137
+ crawl: (opts) => request("/v1/crawl", { method: "POST", body: opts, auth: true }),
138
+ getJob,
139
+ getJobResults,
140
+ waitForJob: async (jobId, opts = {}) => {
141
+ const interval = opts.pollIntervalMs ?? 2e3;
142
+ const timeout = opts.timeoutMs ?? 10 * 60 * 1e3;
143
+ if (!Number.isFinite(interval) || interval < 0 || interval > 2147483647) {
144
+ throw new RangeError("pollIntervalMs must be between 0 and 2147483647");
145
+ }
146
+ if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647) {
147
+ throw new RangeError("timeoutMs must be greater than 0 and at most 2147483647");
148
+ }
149
+ opts.signal?.throwIfAborted();
150
+ const controller = new AbortController();
151
+ const { signal } = controller;
152
+ const onAbort = () => controller.abort(opts.signal?.reason);
153
+ opts.signal?.addEventListener("abort", onAbort, { once: true });
154
+ const timer = setTimeout(
155
+ () => controller.abort(new import_error.PlatformApiError("poll_timeout", `job ${jobId} did not finish within ${timeout}ms`, 0)),
156
+ timeout
157
+ );
158
+ let rejectOnAbort = () => {
159
+ };
160
+ const aborted = new Promise((_resolve, reject) => {
161
+ rejectOnAbort = () => reject(signal.reason);
162
+ signal.addEventListener("abort", rejectOnAbort, { once: true });
163
+ });
164
+ const poll = async () => {
165
+ for (; ; ) {
166
+ signal.throwIfAborted();
167
+ const status = await getJobOrBackOff(jobId, signal, interval);
168
+ signal.throwIfAborted();
169
+ if (status === void 0) {
170
+ continue;
171
+ }
172
+ opts.onStatus?.(status);
173
+ if (TERMINAL_STATES.has(status.status)) {
174
+ return status;
175
+ }
176
+ await sleep(interval, signal);
177
+ }
178
+ };
179
+ try {
180
+ return await Promise.race([poll(), aborted]);
181
+ } finally {
182
+ clearTimeout(timer);
183
+ opts.signal?.removeEventListener("abort", onAbort);
184
+ signal.removeEventListener("abort", rejectOnAbort);
185
+ }
186
+ },
187
+ jobResults: async function* (jobId) {
188
+ let cursor;
189
+ do {
190
+ const page = await getJobResults(jobId, { cursor });
191
+ for (const item of page.results) {
192
+ yield (0, import_types.isStoredPageStub)(item) ? await resolveStored(item.resultUrl) : item;
193
+ }
194
+ cursor = page.cursor;
195
+ } while (cursor);
196
+ },
197
+ demoScrape: (opts) => request("/v1/demo/scrape", { method: "POST", body: opts, auth: false })
198
+ };
199
+ };
200
+ // Annotate the CommonJS export names for ESM import in node:
201
+ 0 && (module.exports = {
202
+ DEFAULT_BASE_URL,
203
+ createPlatformClient
204
+ });
@@ -0,0 +1,36 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var error_exports = {};
20
+ __export(error_exports, {
21
+ PlatformApiError: () => PlatformApiError
22
+ });
23
+ module.exports = __toCommonJS(error_exports);
24
+ class PlatformApiError extends Error {
25
+ constructor(code, message, status, retryAfter) {
26
+ super(message);
27
+ this.name = "PlatformApiError";
28
+ this.code = code;
29
+ this.status = status;
30
+ this.retryAfter = retryAfter;
31
+ }
32
+ }
33
+ // Annotate the CommonJS export names for ESM import in node:
34
+ 0 && (module.exports = {
35
+ PlatformApiError
36
+ });
@@ -0,0 +1,37 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var fetch_exports = {};
20
+ __export(fetch_exports, {
21
+ resolveFetch: () => resolveFetch
22
+ });
23
+ module.exports = __toCommonJS(fetch_exports);
24
+ const resolveFetch = (custom) => {
25
+ if (custom) {
26
+ return custom;
27
+ }
28
+ const globalFetch = globalThis.fetch;
29
+ if (typeof globalFetch !== "function") {
30
+ throw new Error("No global fetch in this runtime \u2014 pass a fetch implementation to createPlatformClient({ fetch })");
31
+ }
32
+ return globalFetch.bind(globalThis);
33
+ };
34
+ // Annotate the CommonJS export names for ESM import in node:
35
+ 0 && (module.exports = {
36
+ resolveFetch
37
+ });
@@ -0,0 +1,44 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var platform_exports = {};
20
+ __export(platform_exports, {
21
+ CRAWL_SITEMAP_MODES: () => import_types.CRAWL_SITEMAP_MODES,
22
+ DEFAULT_BASE_URL: () => import_client.DEFAULT_BASE_URL,
23
+ ENGINES: () => import_types.ENGINES,
24
+ PlatformApiError: () => import_error.PlatformApiError,
25
+ REGIONS: () => import_types.REGIONS,
26
+ REQUESTED_ENGINES: () => import_types.REQUESTED_ENGINES,
27
+ createPlatformClient: () => import_client.createPlatformClient,
28
+ isStoredPageStub: () => import_types.isStoredPageStub
29
+ });
30
+ module.exports = __toCommonJS(platform_exports);
31
+ var import_client = require("./client");
32
+ var import_error = require("./error");
33
+ var import_types = require("./types");
34
+ // Annotate the CommonJS export names for ESM import in node:
35
+ 0 && (module.exports = {
36
+ CRAWL_SITEMAP_MODES,
37
+ DEFAULT_BASE_URL,
38
+ ENGINES,
39
+ PlatformApiError,
40
+ REGIONS,
41
+ REQUESTED_ENGINES,
42
+ createPlatformClient,
43
+ isStoredPageStub
44
+ });
@@ -0,0 +1,40 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var types_exports = {};
20
+ __export(types_exports, {
21
+ CRAWL_SITEMAP_MODES: () => CRAWL_SITEMAP_MODES,
22
+ ENGINES: () => ENGINES,
23
+ REGIONS: () => REGIONS,
24
+ REQUESTED_ENGINES: () => REQUESTED_ENGINES,
25
+ isStoredPageStub: () => isStoredPageStub
26
+ });
27
+ module.exports = __toCommonJS(types_exports);
28
+ const ENGINES = ["fetch", "browser", "proxy-fetch", "proxy-browser"];
29
+ const REQUESTED_ENGINES = [...ENGINES, "auto"];
30
+ const REGIONS = ["auto", "jp"];
31
+ const CRAWL_SITEMAP_MODES = ["skip", "include", "only"];
32
+ const isStoredPageStub = (item) => item.status === "ok" && "resultUrl" in item;
33
+ // Annotate the CommonJS export names for ESM import in node:
34
+ 0 && (module.exports = {
35
+ CRAWL_SITEMAP_MODES,
36
+ ENGINES,
37
+ REGIONS,
38
+ REQUESTED_ENGINES,
39
+ isStoredPageStub
40
+ });
@@ -0,0 +1,284 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var hast_fast_exports = {};
20
+ __export(hast_fast_exports, {
21
+ MetricsCollector: () => MetricsCollector,
22
+ asRoot: () => asRoot,
23
+ classList: () => classList,
24
+ cloneHast: () => cloneHast,
25
+ collectElements: () => collectElements,
26
+ findElement: () => findElement,
27
+ hasProperty: () => hasProperty,
28
+ isElement: () => isElement,
29
+ isParent: () => isParent,
30
+ isTruthyAttribute: () => isTruthyAttribute,
31
+ matchString: () => matchString,
32
+ numericProperty: () => numericProperty,
33
+ property: () => property,
34
+ pruneInPlace: () => pruneInPlace,
35
+ stringProperty: () => stringProperty,
36
+ walk: () => walk
37
+ });
38
+ module.exports = __toCommonJS(hast_fast_exports);
39
+ const EMPTY_METRICS = { text: 0, link: 0, commas: 0, paragraphs: 0, elements: 0 };
40
+ const COMMA_REGEX = /[,、,،]/g;
41
+ const PARAGRAPH_LIKE = /* @__PURE__ */ new Set(["p", "pre", "li", "blockquote", "figcaption", "dd", "dt"]);
42
+ const NON_PROSE_TAGS = /* @__PURE__ */ new Set(["script", "style", "noscript", "template", "svg", "math", "iframe", "object"]);
43
+ const isElement = (node) => typeof node === "object" && node !== null && node.type === "element";
44
+ const isParent = (node) => typeof node === "object" && node !== null && Array.isArray(node.children);
45
+ const classList = (element) => {
46
+ const className = element.properties?.className;
47
+ if (Array.isArray(className)) {
48
+ return className;
49
+ }
50
+ if (typeof className === "string") {
51
+ return className.split(/\s+/).filter(Boolean);
52
+ }
53
+ return [];
54
+ };
55
+ const matchString = (element) => {
56
+ const id = element.properties?.id;
57
+ const classes = classList(element);
58
+ if (!id && classes.length === 0) {
59
+ return element.tagName;
60
+ }
61
+ return `${element.tagName} ${typeof id === "string" ? id : ""} ${classes.join(" ")}`;
62
+ };
63
+ class MetricsCollector {
64
+ #cache = /* @__PURE__ */ new WeakMap();
65
+ metrics(node) {
66
+ if (node.type === "text") {
67
+ const value = node.value;
68
+ if (value.trim().length === 0) {
69
+ return EMPTY_METRICS;
70
+ }
71
+ return {
72
+ text: value.length,
73
+ link: 0,
74
+ commas: (value.match(COMMA_REGEX) ?? []).length,
75
+ paragraphs: 0,
76
+ elements: 0
77
+ };
78
+ }
79
+ if (!isParent(node)) {
80
+ return EMPTY_METRICS;
81
+ }
82
+ const cached = this.#cache.get(node);
83
+ if (cached) {
84
+ return cached;
85
+ }
86
+ if (isElement(node) && NON_PROSE_TAGS.has(node.tagName)) {
87
+ this.#cache.set(node, EMPTY_METRICS);
88
+ return EMPTY_METRICS;
89
+ }
90
+ let text = 0;
91
+ let link = 0;
92
+ let commas = 0;
93
+ let paragraphs = 0;
94
+ let elements = 0;
95
+ for (const child of node.children) {
96
+ const childMetrics = this.metrics(child);
97
+ text += childMetrics.text;
98
+ link += childMetrics.link;
99
+ commas += childMetrics.commas;
100
+ paragraphs += childMetrics.paragraphs;
101
+ elements += childMetrics.elements;
102
+ if (isElement(child)) {
103
+ elements += 1;
104
+ if (PARAGRAPH_LIKE.has(child.tagName)) {
105
+ paragraphs += 1;
106
+ }
107
+ }
108
+ }
109
+ if (isElement(node) && node.tagName === "a") {
110
+ link = text;
111
+ }
112
+ const metrics = { text, link, commas, paragraphs, elements };
113
+ this.#cache.set(node, metrics);
114
+ return metrics;
115
+ }
116
+ textLength(node) {
117
+ return this.metrics(node).text;
118
+ }
119
+ /** Fraction of a subtree's text that is link anchor text. `0` for empty subtrees. */
120
+ linkDensity(node) {
121
+ const { text, link } = this.metrics(node);
122
+ if (text === 0) {
123
+ return 0;
124
+ }
125
+ return Math.min(1, link / text);
126
+ }
127
+ }
128
+ const pruneInPlace = (tree, keep) => {
129
+ if (!isParent(tree)) {
130
+ return tree;
131
+ }
132
+ const stack = [tree];
133
+ while (stack.length > 0) {
134
+ const parent = stack.pop();
135
+ const children = parent.children;
136
+ let write = 0;
137
+ for (let read = 0; read < children.length; read++) {
138
+ const child = children[read];
139
+ if (!keep(child)) {
140
+ continue;
141
+ }
142
+ children[write++] = child;
143
+ if (isParent(child)) {
144
+ stack.push(child);
145
+ }
146
+ }
147
+ children.length = write;
148
+ }
149
+ return tree;
150
+ };
151
+ const walk = (tree, visit) => {
152
+ const stack = [{ node: tree, parent: void 0 }];
153
+ while (stack.length > 0) {
154
+ const { node, parent } = stack.pop();
155
+ if (visit(node, parent) === false) {
156
+ continue;
157
+ }
158
+ if (isParent(node)) {
159
+ for (let i = node.children.length - 1; i >= 0; i--) {
160
+ stack.push({ node: node.children[i], parent: node });
161
+ }
162
+ }
163
+ }
164
+ };
165
+ const collectElements = (tree, predicate) => {
166
+ const found = [];
167
+ walk(tree, (node) => {
168
+ if (isElement(node) && predicate(node)) {
169
+ found.push(node);
170
+ }
171
+ });
172
+ return found;
173
+ };
174
+ const findElement = (tree, predicate) => {
175
+ let found;
176
+ walk(tree, (node) => {
177
+ if (found) {
178
+ return false;
179
+ }
180
+ if (isElement(node) && predicate(node)) {
181
+ found = node;
182
+ return false;
183
+ }
184
+ });
185
+ return found;
186
+ };
187
+ const cloneHast = (tree) => {
188
+ try {
189
+ return structuredClone(tree);
190
+ } catch {
191
+ return manualClone(tree);
192
+ }
193
+ };
194
+ const manualClone = (node) => {
195
+ if (Array.isArray(node)) {
196
+ return node.map(manualClone);
197
+ }
198
+ if (typeof node !== "object" || node === null) {
199
+ return node;
200
+ }
201
+ const copy = {};
202
+ for (const [key, value] of Object.entries(node)) {
203
+ copy[key] = manualClone(value);
204
+ }
205
+ return copy;
206
+ };
207
+ const asRoot = (nodes) => {
208
+ if (Array.isArray(nodes)) {
209
+ return { type: "root", children: nodes };
210
+ }
211
+ return nodes;
212
+ };
213
+ const property = (element, key) => {
214
+ const properties = element.properties;
215
+ if (!properties) {
216
+ return void 0;
217
+ }
218
+ const direct = properties[key];
219
+ if (direct !== void 0) {
220
+ return direct;
221
+ }
222
+ if (!(key.includes("-") || key.includes(":"))) {
223
+ return void 0;
224
+ }
225
+ return properties[camelCaseAttribute(key)];
226
+ };
227
+ const camelCaseAttribute = (key) => key.replace(/[-:]([a-z])/g, (_, character) => character.toUpperCase());
228
+ const numericProperty = (element, key) => {
229
+ const raw = property(element, key);
230
+ if (typeof raw === "number") {
231
+ return Number.isFinite(raw) ? raw : void 0;
232
+ }
233
+ if (typeof raw === "string" && raw.length > 0) {
234
+ const parsed = Number.parseFloat(raw);
235
+ return Number.isFinite(parsed) ? parsed : void 0;
236
+ }
237
+ return void 0;
238
+ };
239
+ const stringProperty = (element, key) => {
240
+ const raw = property(element, key);
241
+ if (typeof raw === "string") {
242
+ return raw;
243
+ }
244
+ if (Array.isArray(raw)) {
245
+ return raw.join(" ");
246
+ }
247
+ if (raw === true) {
248
+ return "";
249
+ }
250
+ return void 0;
251
+ };
252
+ const hasProperty = (element, key) => property(element, key) !== void 0;
253
+ const isTruthyAttribute = (element, key) => {
254
+ const raw = property(element, key);
255
+ if (raw === void 0 || raw === null || raw === false) {
256
+ return false;
257
+ }
258
+ if (raw === true) {
259
+ return true;
260
+ }
261
+ if (typeof raw === "string") {
262
+ return raw.toLowerCase() !== "false";
263
+ }
264
+ return true;
265
+ };
266
+ // Annotate the CommonJS export names for ESM import in node:
267
+ 0 && (module.exports = {
268
+ MetricsCollector,
269
+ asRoot,
270
+ classList,
271
+ cloneHast,
272
+ collectElements,
273
+ findElement,
274
+ hasProperty,
275
+ isElement,
276
+ isParent,
277
+ isTruthyAttribute,
278
+ matchString,
279
+ numericProperty,
280
+ property,
281
+ pruneInPlace,
282
+ stringProperty,
283
+ walk
284
+ });
package/dist/constants.js CHANGED
@@ -1,6 +1,6 @@
1
1
  // src/constants.ts
2
- import { takumiExtractor } from "./extractors/presets/takumi.js";
3
- var DEFAULT_EXTRACTORS = [takumiExtractor];
2
+ import { autoExtractor } from "./extractors/presets/auto.js";
3
+ var DEFAULT_EXTRACTORS = [autoExtractor];
4
4
  export {
5
5
  DEFAULT_EXTRACTORS
6
6
  };