webforai 2.1.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/README.md +86 -4
  2. package/dist/adapters/index.js +58 -0
  3. package/dist/adapters/registry.js +149 -0
  4. package/dist/adapters/sites/declarative.js +178 -0
  5. package/dist/adapters/sites/hackernews.js +94 -0
  6. package/dist/adapters/sites/youtube.js +165 -0
  7. package/dist/adapters/types.js +0 -0
  8. package/dist/bin.js +4142 -549
  9. package/dist/cjs/adapters/index.js +86 -0
  10. package/dist/cjs/adapters/registry.js +178 -0
  11. package/dist/cjs/adapters/sites/declarative.js +202 -0
  12. package/dist/cjs/adapters/sites/hackernews.js +117 -0
  13. package/dist/cjs/adapters/sites/youtube.js +191 -0
  14. package/dist/cjs/adapters/types.js +16 -0
  15. package/dist/cjs/constants.js +2 -2
  16. package/dist/cjs/detect-client-shell.js +78 -0
  17. package/dist/cjs/extract-meta-refresh.js +71 -0
  18. package/dist/cjs/extractors/index.js +5 -0
  19. package/dist/cjs/extractors/lib/constants.js +186 -0
  20. package/dist/cjs/extractors/lib/sanitize.js +429 -0
  21. package/dist/cjs/extractors/lib/score.js +264 -0
  22. package/dist/cjs/extractors/lib/terminators.js +131 -0
  23. package/dist/cjs/extractors/lib/ui-chrome.js +119 -0
  24. package/dist/cjs/extractors/pipeExtractors.js +12 -9
  25. package/dist/cjs/extractors/presets/auto.js +44 -0
  26. package/dist/cjs/extractors/presets/minimal-filter.js +27 -48
  27. package/dist/cjs/extractors/presets/takumi.js +84 -122
  28. package/dist/cjs/html-to-markdown.js +44 -7
  29. package/dist/cjs/html-to-mdast.js +10 -1
  30. package/dist/cjs/index.js +43 -2
  31. package/dist/cjs/link-replacer.js +19 -13
  32. package/dist/cjs/loaders/cf-puppeteer.js +14 -7
  33. package/dist/cjs/loaders/fetch.js +14 -2
  34. package/dist/cjs/loaders/playwright.js +32 -30
  35. package/dist/cjs/loaders/puppeteer.js +14 -7
  36. package/dist/cjs/md-splitter.js +24 -11
  37. package/dist/cjs/mdast-handlers/custom-a-handler.js +25 -9
  38. package/dist/cjs/mdast-handlers/custom-code-handler.js +18 -7
  39. package/dist/cjs/mdast-handlers/custom-div-handler.js +12 -33
  40. package/dist/cjs/mdast-handlers/custom-img-handler.js +23 -1
  41. package/dist/cjs/mdast-handlers/custom-table-handler.js +16 -0
  42. package/dist/cjs/mdast-handlers/definition-list-handler.js +110 -0
  43. package/dist/cjs/mdast-handlers/math-handler.js +12 -3
  44. package/dist/cjs/mdast-handlers/sup-sub-handler.js +39 -0
  45. package/dist/cjs/mdast-to-markdown.js +2 -5
  46. package/dist/cjs/metadata/index.js +223 -0
  47. package/dist/cjs/normalize/index.js +177 -0
  48. package/dist/cjs/platform/client.js +204 -0
  49. package/dist/cjs/platform/error.js +36 -0
  50. package/dist/cjs/platform/fetch.js +37 -0
  51. package/dist/cjs/platform/index.js +44 -0
  52. package/dist/cjs/platform/types.js +40 -0
  53. package/dist/cjs/utils/hast-fast.js +284 -0
  54. package/dist/constants.js +2 -2
  55. package/dist/detect-client-shell.js +53 -0
  56. package/dist/extract-meta-refresh.js +47 -0
  57. package/dist/extractors/index.js +3 -0
  58. package/dist/extractors/lib/constants.js +150 -0
  59. package/dist/extractors/lib/sanitize.js +419 -0
  60. package/dist/extractors/lib/score.js +242 -0
  61. package/dist/extractors/lib/terminators.js +107 -0
  62. package/dist/extractors/lib/ui-chrome.js +96 -0
  63. package/dist/extractors/pipeExtractors.js +12 -9
  64. package/dist/extractors/presets/auto.js +20 -0
  65. package/dist/extractors/presets/minimal-filter.js +35 -47
  66. package/dist/extractors/presets/takumi.js +91 -122
  67. package/dist/html-to-markdown.js +42 -6
  68. package/dist/html-to-mdast.js +10 -1
  69. package/dist/index.js +45 -2
  70. package/dist/link-replacer.js +19 -13
  71. package/dist/loaders/cf-puppeteer.js +15 -8
  72. package/dist/loaders/fetch.js +14 -2
  73. package/dist/loaders/playwright.js +32 -30
  74. package/dist/loaders/puppeteer.js +15 -8
  75. package/dist/md-splitter.js +24 -11
  76. package/dist/mdast-handlers/custom-a-handler.js +25 -9
  77. package/dist/mdast-handlers/custom-code-handler.js +17 -7
  78. package/dist/mdast-handlers/custom-div-handler.js +12 -33
  79. package/dist/mdast-handlers/custom-img-handler.js +23 -1
  80. package/dist/mdast-handlers/custom-table-handler.js +16 -0
  81. package/dist/mdast-handlers/definition-list-handler.js +87 -0
  82. package/dist/mdast-handlers/math-handler.js +12 -3
  83. package/dist/mdast-handlers/sup-sub-handler.js +16 -0
  84. package/dist/mdast-to-markdown.js +2 -5
  85. package/dist/metadata/index.js +198 -0
  86. package/dist/normalize/index.js +159 -0
  87. package/dist/platform/client.js +182 -0
  88. package/dist/platform/error.js +13 -0
  89. package/dist/platform/fetch.js +14 -0
  90. package/dist/platform/index.js +23 -0
  91. package/dist/platform/types.js +13 -0
  92. package/dist/types/adapters/index.d.ts +24 -0
  93. package/dist/types/adapters/index.js.map +1 -0
  94. package/dist/types/adapters/registry.d.ts +35 -0
  95. package/dist/types/adapters/registry.js.map +1 -0
  96. package/dist/types/adapters/sites/declarative.d.ts +13 -0
  97. package/dist/types/adapters/sites/declarative.js.map +1 -0
  98. package/dist/types/adapters/sites/hackernews.d.ts +13 -0
  99. package/dist/types/adapters/sites/hackernews.js.map +1 -0
  100. package/dist/types/adapters/sites/youtube.d.ts +31 -0
  101. package/dist/types/adapters/sites/youtube.js.map +1 -0
  102. package/dist/types/adapters/types.d.ts +99 -0
  103. package/dist/types/adapters/types.js.map +1 -0
  104. package/dist/types/cli/bin.js.map +1 -1
  105. package/dist/types/cli/commands/jobs/files.d.ts +28 -0
  106. package/dist/types/cli/commands/jobs/files.js.map +1 -0
  107. package/dist/types/cli/commands/jobs/llms-txt.d.ts +33 -0
  108. package/dist/types/cli/commands/jobs/llms-txt.js.map +1 -0
  109. package/dist/types/cli/commands/jobs/options.d.ts +58 -0
  110. package/dist/types/cli/commands/jobs/options.js.map +1 -0
  111. package/dist/types/cli/commands/jobs/run.d.ts +44 -0
  112. package/dist/types/cli/commands/jobs/run.js.map +1 -0
  113. package/dist/types/cli/commands/skill/content.d.ts +9 -0
  114. package/dist/types/cli/commands/skill/content.js.map +1 -0
  115. package/dist/types/cli/commands/skill/index.d.ts +19 -0
  116. package/dist/types/cli/commands/skill/index.js.map +1 -0
  117. package/dist/types/cli/commands/webforai/interactive.d.ts +7 -0
  118. package/dist/types/cli/commands/webforai/interactive.js.map +1 -0
  119. package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -1
  120. package/dist/types/cli/commands/webforai/options.d.ts +62 -0
  121. package/dist/types/cli/commands/webforai/options.js.map +1 -0
  122. package/dist/types/cli/commands/webforai/run.d.ts +10 -0
  123. package/dist/types/cli/commands/webforai/run.js.map +1 -0
  124. package/dist/types/cli/constants.d.ts +14 -4
  125. package/dist/types/cli/constants.js.map +1 -1
  126. package/dist/types/cli/helpers/selectExtractMode.js.map +1 -1
  127. package/dist/types/cli/helpers/selectLoader.js.map +1 -1
  128. package/dist/types/cli/utils.d.ts +8 -1
  129. package/dist/types/cli/utils.js.map +1 -1
  130. package/dist/types/constants.d.ts +7 -1
  131. package/dist/types/constants.js.map +1 -1
  132. package/dist/types/detect-client-shell.d.ts +24 -0
  133. package/dist/types/detect-client-shell.js.map +1 -0
  134. package/dist/types/extract-meta-refresh.d.ts +29 -0
  135. package/dist/types/extract-meta-refresh.js.map +1 -0
  136. package/dist/types/extractors/index.d.ts +1 -0
  137. package/dist/types/extractors/index.js.map +1 -1
  138. package/dist/types/extractors/lib/constants.d.ts +81 -0
  139. package/dist/types/extractors/lib/constants.js.map +1 -0
  140. package/dist/types/extractors/lib/sanitize.d.ts +66 -0
  141. package/dist/types/extractors/lib/sanitize.js.map +1 -0
  142. package/dist/types/extractors/lib/score.d.ts +85 -0
  143. package/dist/types/extractors/lib/score.js.map +1 -0
  144. package/dist/types/extractors/lib/terminators.d.ts +26 -0
  145. package/dist/types/extractors/lib/terminators.js.map +1 -0
  146. package/dist/types/extractors/lib/ui-chrome.d.ts +18 -0
  147. package/dist/types/extractors/lib/ui-chrome.js.map +1 -0
  148. package/dist/types/extractors/pipeExtractors.d.ts +5 -0
  149. package/dist/types/extractors/pipeExtractors.js.map +1 -1
  150. package/dist/types/extractors/presets/auto.d.ts +30 -0
  151. package/dist/types/extractors/presets/auto.js.map +1 -0
  152. package/dist/types/extractors/presets/minimal-filter.d.ts +10 -2
  153. package/dist/types/extractors/presets/minimal-filter.js.map +1 -1
  154. package/dist/types/extractors/presets/takumi.d.ts +19 -3
  155. package/dist/types/extractors/presets/takumi.js.map +1 -1
  156. package/dist/types/extractors/types.d.ts +8 -0
  157. package/dist/types/html-to-markdown.d.ts +38 -0
  158. package/dist/types/html-to-markdown.js.map +1 -1
  159. package/dist/types/html-to-mdast.d.ts +11 -0
  160. package/dist/types/html-to-mdast.js.map +1 -1
  161. package/dist/types/index.d.ts +8 -3
  162. package/dist/types/index.js.map +1 -1
  163. package/dist/types/link-replacer.d.ts +3 -1
  164. package/dist/types/link-replacer.js.map +1 -1
  165. package/dist/types/loaders/cf-puppeteer.js.map +1 -1
  166. package/dist/types/loaders/fetch.d.ts +4 -0
  167. package/dist/types/loaders/fetch.js.map +1 -1
  168. package/dist/types/loaders/playwright.js.map +1 -1
  169. package/dist/types/loaders/puppeteer.js.map +1 -1
  170. package/dist/types/md-splitter.d.ts +1 -1
  171. package/dist/types/md-splitter.js.map +1 -1
  172. package/dist/types/mdast-handlers/custom-a-handler.js.map +1 -1
  173. package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
  174. package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
  175. package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
  176. package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
  177. package/dist/types/mdast-handlers/custom-table-handler.js.map +1 -1
  178. package/dist/types/mdast-handlers/definition-list-handler.d.ts +2 -0
  179. package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -0
  180. package/dist/types/mdast-handlers/math-handler.js.map +1 -1
  181. package/dist/types/mdast-handlers/sup-sub-handler.d.ts +10 -0
  182. package/dist/types/mdast-handlers/sup-sub-handler.js.map +1 -0
  183. package/dist/types/mdast-to-markdown.js.map +1 -1
  184. package/dist/types/metadata/index.d.ts +47 -0
  185. package/dist/types/metadata/index.js.map +1 -0
  186. package/dist/types/normalize/index.d.ts +53 -0
  187. package/dist/types/normalize/index.js.map +1 -0
  188. package/dist/types/platform/client.d.ts +55 -0
  189. package/dist/types/platform/client.js.map +1 -0
  190. package/dist/types/platform/error.d.ts +22 -0
  191. package/dist/types/platform/error.js.map +1 -0
  192. package/dist/types/platform/fetch.d.ts +36 -0
  193. package/dist/types/platform/fetch.js.map +1 -0
  194. package/dist/types/platform/index.d.ts +4 -0
  195. package/dist/types/platform/index.js.map +1 -0
  196. package/dist/types/platform/types.d.ts +164 -0
  197. package/dist/types/platform/types.js.map +1 -0
  198. package/dist/types/utils/hast-fast.d.ts +101 -0
  199. package/dist/types/utils/hast-fast.js.map +1 -0
  200. package/dist/utils/hast-fast.js +246 -0
  201. package/package.json +25 -7
  202. package/dist/types/cli/commands/webforai/index.d.ts +0 -6
  203. package/dist/types/cli/commands/webforai/index.js.map +0 -1
@@ -0,0 +1,159 @@
1
+ // src/normalize/index.ts
2
+ import {
3
+ classList,
4
+ findElement,
5
+ isElement,
6
+ numericProperty,
7
+ property,
8
+ pruneInPlace,
9
+ stringProperty,
10
+ walk
11
+ } from "../utils/hast-fast.js";
12
+ var LAZY_SRC_ATTRIBUTES = [
13
+ "data-src",
14
+ "data-original",
15
+ "data-lazy-src",
16
+ "data-original-src",
17
+ "data-actualsrc",
18
+ "data-hi-res-src",
19
+ "data-echo"
20
+ ];
21
+ var PLACEHOLDER_SRC = /^$|^data:image\/|^about:blank$|spacer\.gif$|blank\.(gif|png)$/i;
22
+ var readSrcset = (element, attribute) => {
23
+ const raw = property(element, attribute);
24
+ if (typeof raw === "string") {
25
+ return raw;
26
+ }
27
+ if (Array.isArray(raw)) {
28
+ return raw.join(", ");
29
+ }
30
+ return void 0;
31
+ };
32
+ var bestFromSrcset = (srcset) => {
33
+ let best;
34
+ for (const entry of srcset.split(",")) {
35
+ const parts = entry.trim().split(/\s+/);
36
+ const url = parts[0];
37
+ if (!url) {
38
+ continue;
39
+ }
40
+ const descriptor = parts[1];
41
+ let weight = 1;
42
+ if (descriptor?.endsWith("w")) {
43
+ weight = Number.parseFloat(descriptor) || 1;
44
+ } else if (descriptor?.endsWith("x")) {
45
+ weight = (Number.parseFloat(descriptor) || 1) * 1e3;
46
+ }
47
+ if (!best || weight > best.weight) {
48
+ best = { url, weight };
49
+ }
50
+ }
51
+ return best?.url;
52
+ };
53
+ var resolveImageSources = (tree) => {
54
+ walk(tree, (node) => {
55
+ if (!isElement(node) || node.tagName !== "img") {
56
+ return;
57
+ }
58
+ const current = stringProperty(node, "src");
59
+ if (current && !PLACEHOLDER_SRC.test(current)) {
60
+ return;
61
+ }
62
+ for (const attribute of LAZY_SRC_ATTRIBUTES) {
63
+ const candidate = stringProperty(node, attribute);
64
+ if (candidate && !PLACEHOLDER_SRC.test(candidate)) {
65
+ node.properties.src = candidate;
66
+ return;
67
+ }
68
+ }
69
+ for (const attribute of ["srcSet", "srcset", "data-srcset"]) {
70
+ const srcset = readSrcset(node, attribute);
71
+ const best = srcset ? bestFromSrcset(srcset) : void 0;
72
+ if (best) {
73
+ node.properties.src = best;
74
+ return;
75
+ }
76
+ }
77
+ });
78
+ };
79
+ var katexDuplicates = (node) => {
80
+ const classes = classList(node);
81
+ if (!(classes.includes("katex") || classes.includes("katex-display"))) {
82
+ return [];
83
+ }
84
+ const hasMathml = node.children.some(
85
+ (child) => isElement(child) && classList(child).includes("katex-mathml") && hasMathSource(child)
86
+ );
87
+ if (!hasMathml) {
88
+ return [];
89
+ }
90
+ return node.children.filter((child) => isElement(child) && classList(child).includes("katex-html"));
91
+ };
92
+ var mediawikiDuplicates = (node) => {
93
+ if (!classList(node).includes("mwe-math-element") || !hasMathSource(node)) {
94
+ return [];
95
+ }
96
+ return node.children.filter(
97
+ (child) => isElement(child) && classList(child).some((name) => name.startsWith("mwe-math-fallback-image"))
98
+ );
99
+ };
100
+ var mathjaxDuplicates = (node) => {
101
+ if (node.tagName !== "mjx-container") {
102
+ return [];
103
+ }
104
+ const hasMathml = node.children.some(
105
+ (child) => isElement(child) && child.tagName === "mjx-assistive-mml" && hasMathSource(child)
106
+ );
107
+ if (!hasMathml) {
108
+ return [];
109
+ }
110
+ return node.children.filter((child) => isElement(child) && child.tagName !== "mjx-assistive-mml");
111
+ };
112
+ var hasMathSource = (node) => Boolean(findElement(node, (element) => element.tagName === "math" && element.children.length > 0));
113
+ var dedupeRenderedMath = (tree) => {
114
+ const doomed = /* @__PURE__ */ new Set();
115
+ walk(tree, (node) => {
116
+ if (!isElement(node)) {
117
+ return;
118
+ }
119
+ for (const duplicate of [...katexDuplicates(node), ...mathjaxDuplicates(node), ...mediawikiDuplicates(node)]) {
120
+ doomed.add(duplicate);
121
+ }
122
+ });
123
+ if (doomed.size > 0) {
124
+ pruneInPlace(tree, (node) => !(isElement(node) && doomed.has(node)));
125
+ }
126
+ };
127
+ var promoteAriaHeadings = (tree) => {
128
+ walk(tree, (node) => {
129
+ if (!isElement(node) || stringProperty(node, "role") !== "heading") {
130
+ return;
131
+ }
132
+ if (/^h[1-6]$/.test(node.tagName)) {
133
+ return;
134
+ }
135
+ const level = numericProperty(node, "aria-level") ?? 2;
136
+ const clamped = Math.min(6, Math.max(1, Math.round(level)));
137
+ node.tagName = `h${clamped}`;
138
+ });
139
+ };
140
+ var normalizeHast = (tree, options = {}) => {
141
+ const { images = true, math = true, ariaHeadings = true } = options;
142
+ if (images) {
143
+ resolveImageSources(tree);
144
+ }
145
+ if (math) {
146
+ dedupeRenderedMath(tree);
147
+ }
148
+ if (ariaHeadings) {
149
+ promoteAriaHeadings(tree);
150
+ }
151
+ return tree;
152
+ };
153
+ export {
154
+ bestFromSrcset,
155
+ dedupeRenderedMath,
156
+ normalizeHast,
157
+ promoteAriaHeadings,
158
+ resolveImageSources
159
+ };
@@ -0,0 +1,182 @@
1
+ // src/platform/client.ts
2
+ import { PlatformApiError } from "./error.js";
3
+ import { resolveFetch } from "./fetch.js";
4
+ import {
5
+ isStoredPageStub
6
+ } from "./types.js";
7
+ var DEFAULT_BASE_URL = "https://platform.webforai.dev";
8
+ var TERMINAL_STATES = /* @__PURE__ */ new Set(["completed", "failed"]);
9
+ var sleep = (ms, signal) => new Promise((resolve, reject) => {
10
+ if (signal?.aborted) {
11
+ reject(signal.reason instanceof Error ? signal.reason : new Error("aborted"));
12
+ return;
13
+ }
14
+ const timer = setTimeout(() => {
15
+ signal?.removeEventListener("abort", onAbort);
16
+ resolve();
17
+ }, ms);
18
+ const onAbort = () => {
19
+ clearTimeout(timer);
20
+ reject(signal?.reason instanceof Error ? signal.reason : new Error("aborted"));
21
+ };
22
+ signal?.addEventListener("abort", onAbort, { once: true });
23
+ });
24
+ var parseRetryAfter = (body, response) => {
25
+ const fromBody = body?.error?.retryAfter;
26
+ if (typeof fromBody === "number") {
27
+ return fromBody;
28
+ }
29
+ const header = Number(response.headers.get("retry-after"));
30
+ return Number.isFinite(header) && header > 0 ? header : void 0;
31
+ };
32
+ var dashboardHint = (status, dashboardUrl) => {
33
+ if (status === 401) {
34
+ return `create or check API keys at ${dashboardUrl}`;
35
+ }
36
+ if (status === 402) {
37
+ return `manage billing, credits and the spend cap at ${dashboardUrl}`;
38
+ }
39
+ return void 0;
40
+ };
41
+ var toApiError = (response, body, dashboardUrl) => {
42
+ const envelope = body;
43
+ const code = typeof envelope?.error?.code === "string" ? envelope.error.code : "invalid_response";
44
+ let message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
45
+ const hint = code === "invalid_response" ? void 0 : dashboardHint(response.status, dashboardUrl);
46
+ if (hint && !message.includes("/dashboard")) {
47
+ message = `${message} (${hint})`;
48
+ }
49
+ return new PlatformApiError(code, message, response.status, parseRetryAfter(body, response));
50
+ };
51
+ var createPlatformClient = (options = {}) => {
52
+ const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
53
+ const dashboardUrl = `${baseUrl}/dashboard`;
54
+ const fetchImpl = resolveFetch(options.fetch);
55
+ const request = async (path, init) => {
56
+ const headers = { accept: "application/json" };
57
+ if (init.body !== void 0) {
58
+ headers["content-type"] = "application/json";
59
+ }
60
+ if (init.auth) {
61
+ if (!options.apiKey) {
62
+ throw new PlatformApiError(
63
+ "missing_api_key",
64
+ `This endpoint needs an API key: pass \`apiKey\` to createPlatformClient (create one at ${dashboardUrl}; 1,000 free credits/month).`,
65
+ 0
66
+ );
67
+ }
68
+ headers.authorization = `Bearer ${options.apiKey}`;
69
+ }
70
+ const response = await fetchImpl(`${baseUrl}${path}`, {
71
+ method: init.method,
72
+ headers,
73
+ body: init.body === void 0 ? void 0 : JSON.stringify(init.body),
74
+ ...init.signal ? { signal: init.signal } : {}
75
+ });
76
+ const body = await response.json().catch(() => void 0);
77
+ if (!response.ok) {
78
+ throw toApiError(response, body, dashboardUrl);
79
+ }
80
+ if (body === void 0) {
81
+ throw new PlatformApiError("invalid_response", "response body is not JSON", response.status);
82
+ }
83
+ return body;
84
+ };
85
+ const getJob = (jobId, signal) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true, signal });
86
+ const getJobResults = (jobId, opts) => {
87
+ const query = opts?.cursor ? `?cursor=${encodeURIComponent(opts.cursor)}` : "";
88
+ return request(`/v1/jobs/${encodeURIComponent(jobId)}/results${query}`, { method: "GET", auth: true });
89
+ };
90
+ const resolveStored = async (resultUrl) => {
91
+ const response = await fetchImpl(resultUrl, { method: "GET" });
92
+ if (!response.ok) {
93
+ throw new PlatformApiError(
94
+ "artifact_fetch_failed",
95
+ `stored result fetch failed (HTTP ${response.status}) \u2014 result URLs expire after ~24h`,
96
+ response.status
97
+ );
98
+ }
99
+ return await response.json();
100
+ };
101
+ const getJobOrBackOff = async (jobId, signal, interval) => {
102
+ try {
103
+ return await getJob(jobId, signal);
104
+ } catch (error) {
105
+ if (!(error instanceof PlatformApiError && error.status === 429)) {
106
+ throw error;
107
+ }
108
+ await sleep(error.retryAfter !== void 0 ? error.retryAfter * 1e3 : interval, signal);
109
+ return void 0;
110
+ }
111
+ };
112
+ return {
113
+ scrape: (opts) => request("/v1/scrape", { method: "POST", body: opts, auth: true }),
114
+ scrapeAsync: (opts) => request("/v1/scrape", { method: "POST", body: { ...opts, async: true }, auth: true }),
115
+ batch: (opts) => request("/v1/batch", { method: "POST", body: opts, auth: true }),
116
+ crawl: (opts) => request("/v1/crawl", { method: "POST", body: opts, auth: true }),
117
+ getJob,
118
+ getJobResults,
119
+ waitForJob: async (jobId, opts = {}) => {
120
+ const interval = opts.pollIntervalMs ?? 2e3;
121
+ const timeout = opts.timeoutMs ?? 10 * 60 * 1e3;
122
+ if (!Number.isFinite(interval) || interval < 0 || interval > 2147483647) {
123
+ throw new RangeError("pollIntervalMs must be between 0 and 2147483647");
124
+ }
125
+ if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647) {
126
+ throw new RangeError("timeoutMs must be greater than 0 and at most 2147483647");
127
+ }
128
+ opts.signal?.throwIfAborted();
129
+ const controller = new AbortController();
130
+ const { signal } = controller;
131
+ const onAbort = () => controller.abort(opts.signal?.reason);
132
+ opts.signal?.addEventListener("abort", onAbort, { once: true });
133
+ const timer = setTimeout(
134
+ () => controller.abort(new PlatformApiError("poll_timeout", `job ${jobId} did not finish within ${timeout}ms`, 0)),
135
+ timeout
136
+ );
137
+ let rejectOnAbort = () => {
138
+ };
139
+ const aborted = new Promise((_resolve, reject) => {
140
+ rejectOnAbort = () => reject(signal.reason);
141
+ signal.addEventListener("abort", rejectOnAbort, { once: true });
142
+ });
143
+ const poll = async () => {
144
+ for (; ; ) {
145
+ signal.throwIfAborted();
146
+ const status = await getJobOrBackOff(jobId, signal, interval);
147
+ signal.throwIfAborted();
148
+ if (status === void 0) {
149
+ continue;
150
+ }
151
+ opts.onStatus?.(status);
152
+ if (TERMINAL_STATES.has(status.status)) {
153
+ return status;
154
+ }
155
+ await sleep(interval, signal);
156
+ }
157
+ };
158
+ try {
159
+ return await Promise.race([poll(), aborted]);
160
+ } finally {
161
+ clearTimeout(timer);
162
+ opts.signal?.removeEventListener("abort", onAbort);
163
+ signal.removeEventListener("abort", rejectOnAbort);
164
+ }
165
+ },
166
+ jobResults: async function* (jobId) {
167
+ let cursor;
168
+ do {
169
+ const page = await getJobResults(jobId, { cursor });
170
+ for (const item of page.results) {
171
+ yield isStoredPageStub(item) ? await resolveStored(item.resultUrl) : item;
172
+ }
173
+ cursor = page.cursor;
174
+ } while (cursor);
175
+ },
176
+ demoScrape: (opts) => request("/v1/demo/scrape", { method: "POST", body: opts, auth: false })
177
+ };
178
+ };
179
+ export {
180
+ DEFAULT_BASE_URL,
181
+ createPlatformClient
182
+ };
@@ -0,0 +1,13 @@
1
+ // src/platform/error.ts
2
+ var PlatformApiError = class extends Error {
3
+ constructor(code, message, status, retryAfter) {
4
+ super(message);
5
+ this.name = "PlatformApiError";
6
+ this.code = code;
7
+ this.status = status;
8
+ this.retryAfter = retryAfter;
9
+ }
10
+ };
11
+ export {
12
+ PlatformApiError
13
+ };
@@ -0,0 +1,14 @@
1
+ // src/platform/fetch.ts
2
+ var resolveFetch = (custom) => {
3
+ if (custom) {
4
+ return custom;
5
+ }
6
+ const globalFetch = globalThis.fetch;
7
+ if (typeof globalFetch !== "function") {
8
+ throw new Error("No global fetch in this runtime \u2014 pass a fetch implementation to createPlatformClient({ fetch })");
9
+ }
10
+ return globalFetch.bind(globalThis);
11
+ };
12
+ export {
13
+ resolveFetch
14
+ };
@@ -0,0 +1,23 @@
1
+ // src/platform/index.ts
2
+ import {
3
+ DEFAULT_BASE_URL,
4
+ createPlatformClient
5
+ } from "./client.js";
6
+ import { PlatformApiError } from "./error.js";
7
+ import {
8
+ CRAWL_SITEMAP_MODES,
9
+ ENGINES,
10
+ REGIONS,
11
+ REQUESTED_ENGINES,
12
+ isStoredPageStub
13
+ } from "./types.js";
14
+ export {
15
+ CRAWL_SITEMAP_MODES,
16
+ DEFAULT_BASE_URL,
17
+ ENGINES,
18
+ PlatformApiError,
19
+ REGIONS,
20
+ REQUESTED_ENGINES,
21
+ createPlatformClient,
22
+ isStoredPageStub
23
+ };
@@ -0,0 +1,13 @@
1
+ // src/platform/types.ts
2
+ var ENGINES = ["fetch", "browser", "proxy-fetch", "proxy-browser"];
3
+ var REQUESTED_ENGINES = [...ENGINES, "auto"];
4
+ var REGIONS = ["auto", "jp"];
5
+ var CRAWL_SITEMAP_MODES = ["skip", "include", "only"];
6
+ var isStoredPageStub = (item) => item.status === "ok" && "resultUrl" in item;
7
+ export {
8
+ CRAWL_SITEMAP_MODES,
9
+ ENGINES,
10
+ REGIONS,
11
+ REQUESTED_ENGINES,
12
+ isStoredPageStub
13
+ };
@@ -0,0 +1,24 @@
1
+ import type { Nodes as Hast } from "hast";
2
+ import type { AdapterContext, SiteAdapter } from "./types";
3
+ /**
4
+ * Built-in adapters, in priority order.
5
+ *
6
+ * Host-specific adapters come first so that a site with its own entry is not claimed by a
7
+ * generic documentation-framework fingerprint that would also match it.
8
+ */
9
+ export declare const BUILTIN_ADAPTERS: SiteAdapter[];
10
+ export declare const adapterById: (id: string) => SiteAdapter | undefined;
11
+ /**
12
+ * Runs the first adapter that claims the page.
13
+ *
14
+ * Returns `undefined` when no adapter applies, or when the one that did produced too little to
15
+ * be trusted — in both cases the caller should fall back to generic extraction. An adapter
16
+ * silently returning a near-empty tree is the main way site-specific code goes wrong as sites
17
+ * are redesigned, so the result is always size-checked rather than taken on faith.
18
+ */
19
+ export declare const runAdapters: (context: AdapterContext, adapters?: SiteAdapter[]) => Hast | undefined;
20
+ export { buildPageSignature, defineSelectorAdapter, removeAll, resolveAdapter, selectFirst, } from "./registry";
21
+ export { DECLARATIVE_ADAPTERS } from "./sites/declarative";
22
+ export { hackernewsAdapter } from "./sites/hackernews";
23
+ export { youtubeAdapter, youtubeCaptionTracks, type CaptionTrack } from "./sites/youtube";
24
+ export type { AdapterContext, PageSignature, SelectorAdapterSpec, SiteAdapter } from "./types";
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/adapters/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AACnG,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAkB,CAAC,cAAc,EAAE,iBAAiB,EAAE,GAAG,oBAAoB,CAAC,CAAC;AAE5G,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,EAAU,EAA2B,EAAE,CAClE,gBAAgB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAEvD,2FAA2F;AAC3F,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAuB,EAAE,WAA0B,gBAAgB,EAAoB,EAAE;IACpH,8FAA8F;IAC9F,kFAAkF;IAClF,IAAI,OAAO,GAAG,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACtD,IAAI,aAAa,GAAG,OAAO,CAAC;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,aAAa,GAAG,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACjG,OAAO,GAAG,2BAA2B,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC;IACxB,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAM,MAAM,GAAmB,aAAa,CAAC,KAAK;QACjD,CAAC,CAAC,aAAa;QACf,CAAC,CAAC,EAAE,GAAG,aAAa,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAE1E,IAAI,SAA2B,CAAC;IAChC,IAAI,CAAC;QACJ,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,yFAAyF;IACzF,6FAA6F;IAC7F,gEAAgE;IAChE,eAAe,CAAC,SAAS,CAAC,CAAC;IAE3B,OAAO,IAAI,gBAAgB,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AACjG,CAAC,CAAC;AAEF,2DAA2D;AAC3D,OAAO,EACN,kBAAkB,EAClB,qBAAqB,EACrB,SAAS,EACT,cAAc,EACd,WAAW,GACX,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAqB,MAAM,iBAAiB,CAAC"}
@@ -0,0 +1,35 @@
1
+ import type { Element, Nodes as Hast } from "hast";
2
+ import type { AdapterContext, PageSignature, SelectorAdapterSpec, SiteAdapter } from "./types";
3
+ /**
4
+ * Builds a document summary in a single traversal.
5
+ *
6
+ * Called once per conversion and shared by every fingerprint, replacing what would otherwise be
7
+ * one full CSS-selector scan per adapter.
8
+ */
9
+ export declare const buildPageSignature: (root: Hast) => PageSignature;
10
+ /**
11
+ * Compiles a declarative spec into a runnable adapter.
12
+ *
13
+ * The compiled adapter clones nothing: `extract` is called with a tree the pipeline already owns,
14
+ * and removal happens in place on the selected container.
15
+ */
16
+ export declare const defineSelectorAdapter: (spec: SelectorAdapterSpec) => SiteAdapter;
17
+ /** Removes every node matching any of `selectors` from `root`, in place. */
18
+ export declare const removeAll: (root: Element | Hast, selectors: string[] | undefined) => void;
19
+ /** Convenience wrapper for adapters that need a single element. */
20
+ export declare const selectFirst: (root: Hast, selectors: string[]) => Element | undefined;
21
+ /**
22
+ * Finds the first adapter that claims the page.
23
+ *
24
+ * Order matters: specific hosts are registered before the generic documentation-framework
25
+ * fingerprints, so a Docusaurus-powered site with its own adapter still gets the specific one.
26
+ */
27
+ export declare const resolveAdapterByHost: (context: AdapterContext, adapters: SiteAdapter[]) => SiteAdapter | undefined;
28
+ export declare const resolveAdapterByFingerprint: (context: AdapterContext, adapters: SiteAdapter[]) => SiteAdapter | undefined;
29
+ /**
30
+ * Finds the first adapter that claims the page.
31
+ *
32
+ * Hostname matching runs before platform fingerprints: it is a string test, and a known host is
33
+ * always more reliable than a fingerprint that a host-specific site would also trigger.
34
+ */
35
+ export declare const resolveAdapter: (context: AdapterContext, adapters: SiteAdapter[]) => SiteAdapter | undefined;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"registry.js","sourceRoot":"","sources":["../../../src/adapters/registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAGhH,MAAM,eAAe,GAAkB;IACtC,OAAO,EAAE,IAAI,GAAG,EAAE;IAClB,GAAG,EAAE,IAAI,GAAG,EAAE;IACd,IAAI,EAAE,IAAI,GAAG,EAAE;IACf,SAAS,EAAE,EAAE;IACb,IAAI,EAAE,EAAE;CACR,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAU,EAAiB,EAAE;IAC/D,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO;QACR,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;QAED,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACtC,IAAI,EAAE,EAAE,CAAC;YACR,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;YACnF,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC;YACtD,SAAS,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC;YACpC,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,WAAW,EAAE,CAAC;gBACvC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAC;IAEH,OAAO;QACN,OAAO;QACP,GAAG;QACH,IAAI;QACJ,SAAS,EAAE,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE;QACjD,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE;KACxC,CAAC;AACH,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,MAAM,MAAM,GAAG,CAAC,GAAuB,EAAsB,EAAE;IAC9D,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,IAAI,CAAC;QACJ,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,IAAyB,EAAe,EAAE,CAAC,CAAC;IACjF,EAAE,EAAE,IAAI,CAAC,EAAE;IAEX,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE;QACpB,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,OAAO,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,kBAAkB,EAAE,CAAC,OAAO,EAAE,EAAE;QAC/B,+EAA+E;QAC/E,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,eAAe,CAAC;QACvD,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC;IAC/C,CAAC;IAED,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE;QACpB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,kBAAkB,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC5D,IAAI,OAAO,EAAE,CAAC;gBACb,OAAO,OAAO,CAAC;YAChB,CAAC;QACF,CAAC;QAED,4FAA4F;QAC5F,OAAO,SAAS,CAAC;IAClB,CAAC;CACD,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,CAAC,QAAgB,EAAE,IAAyB,EAAE,OAAuB,EAAoB,EAAE;IACrH,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC7B,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAClG,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAClD,IAAI,SAAS,EAAE,CAAC;QACf,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAC1E,CAAC,CAAC;AAEF,4EAA4E;AAC5E,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAoB,EAAE,SAA+B,EAAQ,EAAE;IACxF,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1C,OAAO;IACR,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAW,CAAC;IAClC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrB,CAAC;IACF,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACrB,YAAY,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtE,CAAC;AACF,CAAC,CAAC;AAEF,mEAAmE;AACnE,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAU,EAAE,SAAmB,EAAuB,EAAE;IACnF,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,KAAK,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,OAAuB,EAAE,QAAuB,EAA2B,EAAE;IACjH,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC;YACJ,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9B,OAAO,OAAO,CAAC;YAChB,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,sDAAsD;QACvD,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAC1C,OAAuB,EACvB,QAAuB,EACG,EAAE;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC;YACJ,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3C,OAAO,OAAO,CAAC;YAChB,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,YAAY;QACb,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,OAAuB,EAAE,QAAuB,EAA2B,EAAE,CAC3G,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,2BAA2B,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Selector-driven site adapters.
3
+ *
4
+ * Every selector here was verified against a real capture of the site rather than recalled, and
5
+ * each spec carries a fingerprint where one exists so that self-hosted instances of the same
6
+ * platform are covered too, not just the flagship domain.
7
+ */
8
+ import type { Nodes as Hast } from "hast";
9
+ import type { SiteAdapter } from "../types";
10
+ export declare const DECLARATIVE_ADAPTERS: SiteAdapter[];
11
+ /** Exposed for tests and for callers that want to reference one adapter by name. */
12
+ export declare const declarativeAdapterById: (id: string) => SiteAdapter | undefined;
13
+ export type { Hast };
@@ -0,0 +1 @@
1
+ {"version":3,"file":"declarative.js","sourceRoot":"","sources":["../../../../src/adapters/sites/declarative.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAGpD,mEAAmE;AACnE,MAAM,YAAY,GAAG;IACpB,UAAU;IACV,qCAAqC;IACrC,QAAQ;IACR,eAAe;IACf,gBAAgB;IAChB,oBAAoB;CACpB,CAAC;AAEF,MAAM,KAAK,GAA0B;IACpC;QACC,EAAE,EAAE,QAAQ;QACZ,KAAK,EAAE,oBAAoB;QAC3B,sFAAsF;QACtF,OAAO,EAAE,CAAC,4BAA4B,EAAE,uBAAuB,EAAE,gBAAgB,CAAC;QAClF,MAAM,EAAE,CAAC,0BAA0B,EAAE,0BAA0B,EAAE,gBAAgB,EAAE,GAAG,YAAY,CAAC;QACnG,SAAS,EAAE,EAAE;KACb;IACD;QACC,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,+FAA+F;QACtG,OAAO,EAAE,CAAC,UAAU,EAAE,oBAAoB,CAAC;QAC3C,MAAM,EAAE;YACP,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,gBAAgB;YAChB,cAAc;YACd,GAAG,YAAY;SACf;QACD,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,mBAAmB;QAC1B,OAAO,EAAE,CAAC,SAAS,EAAE,qBAAqB,EAAE,qBAAqB,CAAC;QAClE,MAAM,EAAE,YAAY;QACpB,SAAS,EAAE,EAAE;KACb;IACD;QACC,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,kBAAkB;QACzB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,MAAM,EAAE,YAAY;QACpB,SAAS,EAAE,EAAE;KACb;IACD;QACC,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,mBAAmB;QAC1B,OAAO,EAAE,CAAC,eAAe,EAAE,sBAAsB,EAAE,SAAS,CAAC;QAC7D,MAAM,EAAE,CAAC,YAAY,EAAE,GAAG,YAAY,CAAC;QACvC,SAAS,EAAE,EAAE;KACb;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,KAAK,EAAE,oBAAoB;QAC3B,oEAAoE;QACpE,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,4BAA4B,CAAC;QACjF,OAAO,EAAE,CAAC,iBAAiB,EAAE,SAAS,EAAE,8BAA8B,CAAC;QACvE,MAAM,EAAE,CAAC,iCAAiC,EAAE,oBAAoB,EAAE,GAAG,YAAY,CAAC;QAClF,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,UAAU;QACd,KAAK,EAAE,sBAAsB;QAC7B,qEAAqE;QACrE,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QACtE,OAAO,EAAE,CAAC,oBAAoB,EAAE,cAAc,EAAE,eAAe,CAAC;QAChE,MAAM,EAAE,CAAC,2BAA2B,EAAE,iBAAiB,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC;QACrF,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,kBAAkB;QACzB,OAAO,EAAE,CAAC,oCAAoC,EAAE,0BAA0B,CAAC;QAC3E,MAAM,EAAE,YAAY;QACpB,SAAS,EAAE,EAAE;KACb;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,KAAK,EAAE,6EAA6E;QACpF,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC7G,OAAO,EAAE,CAAC,gBAAgB,CAAC;QAC3B,MAAM,EAAE,CAAC,eAAe,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,GAAG,YAAY,CAAC;QACpF,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,WAAW;QACf,sFAAsF;QACtF,mEAAmE;QACnE,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAC1B,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC;YACzC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;YAC9C,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QACrC,OAAO,EAAE,CAAC,wBAAwB,EAAE,gBAAgB,EAAE,eAAe,EAAE,kBAAkB,CAAC;QAC1F,MAAM,EAAE,CAAC,aAAa,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;QAC/F,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,WAAW;QACf,KAAK,EAAE,uFAAuF;QAC9F,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACrE,OAAO,EAAE,CAAC,mBAAmB,EAAE,kBAAkB,CAAC;QAClD,MAAM,EAAE;YACP,iBAAiB;YACjB,SAAS;YACT,eAAe;YACf,kBAAkB;YAClB,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU;YACV,WAAW;YACX,gBAAgB;YAChB,QAAQ;YACR,UAAU;YACV,mBAAmB;YACnB,eAAe;YACf,cAAc;YACd,WAAW;YACX,MAAM;YACN,MAAM;YACN,kBAAkB;YAClB,GAAG,YAAY;SACf;QACD,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,KAAK,EAAE,oBAAoB;QAC3B,OAAO,EAAE,CAAC,kCAAkC,EAAE,+BAA+B,EAAE,OAAO,CAAC;QACvF,MAAM,EAAE,CAAC,8CAA8C,EAAE,6BAA6B,EAAE,GAAG,YAAY,CAAC;QACxG,0FAA0F;QAC1F,sFAAsF;QACtF,iDAAiD;QACjD,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,YAAY;QAChB,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAC1B,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACzF,OAAO,EAAE,CAAC,qBAAqB,EAAE,mBAAmB,CAAC;QACrD,MAAM,EAAE,CAAC,uBAAuB,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,GAAG,YAAY,CAAC;QAClG,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,WAAW;QACf,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;QAC7F,OAAO,EAAE,CAAC,eAAe,EAAE,SAAS,EAAE,2BAA2B,CAAC;QAClE,MAAM,EAAE,CAAC,cAAc,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,GAAG,YAAY,CAAC;QAC7E,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QACtE,OAAO,EAAE,CAAC,oBAAoB,EAAE,+BAA+B,CAAC;QAChE,MAAM,EAAE,CAAC,iBAAiB,EAAE,cAAc,EAAE,aAAa,EAAE,GAAG,YAAY,CAAC;QAC3E,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,aAAa;QACjB,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAChE,OAAO,EAAE,CAAC,cAAc,EAAE,eAAe,CAAC;QAC1C,MAAM,EAAE,CAAC,aAAa,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,GAAG,YAAY,CAAC;QAClF,SAAS,EAAE,GAAG;KACd;IACD;QACC,EAAE,EAAE,SAAS;QACb,WAAW,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QACjE,OAAO,EAAE,CAAC,oCAAoC,EAAE,cAAc,CAAC;QAC/D,MAAM,EAAE,YAAY;QACpB,SAAS,EAAE,GAAG;KACd;CACD,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAkB,KAAK,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;AAEpF,oFAAoF;AACpF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,EAAU,EAA2B,EAAE,CAC7E,oBAAoB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Hacker News adapter.
3
+ *
4
+ * HN is laid out with nested tables and spacer images: comment depth is encoded as the `indent`
5
+ * attribute on a `<td class="ind">` and as the pixel width of a transparent GIF. Generic
6
+ * extraction sees one enormous table, scores it as a single block and returns either everything
7
+ * or nothing, in both cases losing the reply structure that makes a thread readable.
8
+ *
9
+ * This adapter reconstructs the thread as nested blockquotes, which is the natural Markdown
10
+ * representation of a reply chain.
11
+ */
12
+ import type { SiteAdapter } from "../types";
13
+ export declare const hackernewsAdapter: SiteAdapter;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"hackernews.js","sourceRoot":"","sources":["../../../../src/adapters/sites/hackernews.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAGxE,uFAAuF;AACvF,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,MAAM,IAAI,GAAG,CAAC,KAAa,EAAkB,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;AAE1E,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,QAA0B,EAAW,EAAE,CAAC,CAAC;IAC1E,IAAI,EAAE,SAAS;IACf,OAAO;IACP,UAAU,EAAE,EAAE;IACd,QAAQ;CACR,CAAC,CAAC;AAEH,4FAA4F;AAC5F,MAAM,YAAY,GAAG,CAAC,GAAY,EAAU,EAAE;IAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO,CAAC,CAAC;IACV,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC/C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IACf,CAAC;IAED,uEAAuE;IACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpE,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACzD,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,IAAI,GAAG,CAAC,OAAyB,EAAE,KAAa,EAAkB,EAAE;IACzE,IAAI,OAAO,GAAqB,OAAO,CAAC;IACxC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,eAAe,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QACvE,OAAO,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAoB,EAAE;IACnD,MAAM,KAAK,GAAqB,EAAE,CAAC;IAEnC,MAAM,SAAS,GAAG,MAAM,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;IACjD,IAAI,SAAS,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAEnD,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/C,IAAI,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,GAAG,SAAS,EAAE,QAAQ,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACtE,CAAC;IACF,CAAC;IAED,4DAA4D;IAC5D,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACzC,IAAI,OAAO,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,IAAU,EAAoB,EAAE;IACrD,MAAM,KAAK,GAAqB,EAAE,CAAC;IAEnC,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,2BAA2B,EAAE,IAAI,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,SAAS,CAAC,uBAAuB;QAClC,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAEvE,MAAM,OAAO,GAAqB;YACjC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,OAAO,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;SAClC,CAAC;QAEF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAgB;IAC7C,EAAE,EAAE,YAAY;IAEhB,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QACpB,IAAI,CAAC,GAAG,EAAE,CAAC;YACV,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACJ,OAAO,yBAAyB,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAED,kBAAkB,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEjH,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE;QACrB,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,OAAO,SAAS,CAAC;QAClB,CAAC;QAED,MAAM,QAAQ,GAAkB,CAAC,GAAG,KAAK,CAAC,CAAC;QAC3C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YACjD,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAiB,CAAC;IAClD,CAAC;CACD,CAAC"}
@@ -0,0 +1,31 @@
1
+ /**
2
+ * YouTube adapter.
3
+ *
4
+ * A watch page has essentially no readable DOM: the title, channel and description live in an
5
+ * embedded `ytInitialPlayerResponse` JSON blob, and everything visible is player chrome and
6
+ * recommendation rails. Generic extraction produces an empty document, so this adapter reads the
7
+ * blob directly.
8
+ *
9
+ * Captions are intentionally not fetched. They live behind separate, short-lived, signed URLs,
10
+ * and issuing network requests from a synchronous HTML-to-Markdown conversion would be a
11
+ * surprising side effect. `youtubeCaptionTracks` exposes the track list so callers can fetch
12
+ * them deliberately.
13
+ */
14
+ import type { Element, Nodes as Hast } from "hast";
15
+ import type { SiteAdapter } from "../types";
16
+ export interface CaptionTrack {
17
+ languageCode: string;
18
+ name?: string;
19
+ url: string;
20
+ isAutomatic: boolean;
21
+ }
22
+ export declare const extractAssignedJson: (source: string, name: string) => unknown;
23
+ /**
24
+ * Lists the caption tracks advertised by a YouTube watch page.
25
+ *
26
+ * Returned URLs are signed and expire, so fetch them promptly or re-read the page.
27
+ */
28
+ export declare const youtubeCaptionTracks: (root: Hast) => CaptionTrack[];
29
+ export declare const youtubeAdapter: SiteAdapter;
30
+ /** Re-exported so callers can narrow a tree before handing it to the adapter. */
31
+ export declare const isYoutubeElement: (node: unknown) => node is Element;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../../../src/adapters/sites/youtube.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAoBnE;;;;;GAKG;AACH;;;;;GAKG;AACH,MAAM,aAAa,GAAG,CAAC,MAAc,EAAE,KAAa,EAAU,EAAE;IAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,IAAI,KAAK,GAAG,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAEhC,IAAI,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,KAAK,CAAC;QACjB,CAAC;aAAM,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YAC/B,OAAO,GAAG,IAAI,CAAC;QAChB,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAC9B,QAAQ,GAAG,CAAC,QAAQ,CAAC;QACtB,CAAC;aAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC;iBAAM,IAAI,SAAS,KAAK,GAAG,IAAI,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC;gBAC/C,OAAO,KAAK,GAAG,CAAC,CAAC;YAClB,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAc,EAAE,IAAY,EAAW,EAAE;IAC5E,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;IAChD,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAC/C,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACzC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;QAChB,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAY,EAAE,CAC3C,eAAe,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAC/E,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CACnF,CAAC;AAEH,MAAM,kBAAkB,GAAG,CAAC,IAAU,EAAuC,EAAE;IAC9E,KAAK,MAAM,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;YACjD,SAAS;QACV,CAAC;QACD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;QACtE,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1C,OAAO,MAAiC,CAAC;QAC1C,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,IAAU,EAAkB,EAAE;IAClE,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAI,MAAM,EAAE,QAAgD,EAAE,+BAEhE,CAAC;IACb,MAAM,MAAM,GAAG,QAAQ,EAAE,aAAa,CAAC;IAEvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACX,CAAC;IAED,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,KAA8B,EAAE,EAAE;QACxD,MAAM,GAAG,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,MAAM,YAAY,GAAG,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7F,IAAI,CAAC,CAAC,GAAG,IAAI,YAAY,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC;QACX,CAAC;QACD,MAAM,IAAI,GAAI,KAAK,CAAC,IAA4C,EAAE,UAAU,CAAC;QAC7E,OAAO;YACN;gBACC,GAAG;gBACH,YAAY;gBACZ,IAAI,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACjD,WAAW,EAAE,KAAK,CAAC,IAAI,KAAK,KAAK;aACjC;SACD,CAAC;IACH,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAkB,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;AAE1E,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,QAA0B,EAAW,EAAE,CAAC,CAAC;IAC1E,IAAI,EAAE,SAAS;IACf,OAAO;IACP,UAAU,EAAE,EAAE;IACd,QAAQ;CACR,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,KAAa,EAAW,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAE/F,MAAM,SAAS,GAAG,CAAC,KAAa,EAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAE1E,4EAA4E;AAC5E,MAAM,gBAAgB,GAAG,CAAC,WAAmB,EAAa,EAAE,CAC3D,WAAW;KACT,KAAK,CAAC,QAAQ,CAAC;KACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;KAC5B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;KACnC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACd,OAAO,CACN,GAAG,EACH,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAC1G,CACD,CAAC;AAEJ,MAAM,cAAc,GAAG,CAAC,OAAe,EAAU,EAAE;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,OAAO,GAAG,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;AAClG,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,OAAqB,EAAuB,EAAE;IACnE,MAAM,KAAK,GAAc,EAAE,CAAC;IAE5B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC9C,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,aAAa,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAgB;IAC1C,EAAE,EAAE,SAAS;IAEb,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QACpB,IAAI,CAAC,GAAG,EAAE,CAAC;YACV,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACJ,OAAO,uDAAuD,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC5F,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAED,6FAA6F;IAC7F,iEAAiE;IACjE,kBAAkB,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CACrC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEjF,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,EAAE,YAAwC,CAAC;QAEjE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;QAClB,CAAC;QAED,MAAM,QAAQ,GAAkB,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAE5D,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YACd,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,gBAAgB,EAAE,IAAI,EAAE,CAAC;QACrD,IAAI,WAAW,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACjF,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,uBAAuB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAiB,CAAC;IAClD,CAAC;CACD,CAAC;AAEF,iFAAiF;AACjF,MAAM,CAAC,MAAM,gBAAgB,GAAG,SAAS,CAAC"}