webforai 3.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +25 -5
  2. package/dist/bin.js +1277 -479
  3. package/dist/cjs/detect-client-shell.js +78 -0
  4. package/dist/cjs/extract-meta-refresh.js +71 -0
  5. package/dist/cjs/extractors/lib/sanitize.js +4 -0
  6. package/dist/cjs/html-to-mdast.js +2 -1
  7. package/dist/cjs/index.js +12 -0
  8. package/dist/cjs/link-replacer.js +19 -13
  9. package/dist/cjs/loaders/cf-puppeteer.js +14 -7
  10. package/dist/cjs/loaders/fetch.js +14 -2
  11. package/dist/cjs/loaders/playwright.js +32 -30
  12. package/dist/cjs/loaders/puppeteer.js +14 -7
  13. package/dist/cjs/md-splitter.js +24 -11
  14. package/dist/cjs/mdast-handlers/custom-code-handler.js +18 -7
  15. package/dist/cjs/mdast-handlers/custom-div-handler.js +12 -33
  16. package/dist/cjs/mdast-handlers/definition-list-handler.js +17 -6
  17. package/dist/cjs/mdast-to-markdown.js +2 -5
  18. package/dist/cjs/normalize/index.js +11 -2
  19. package/dist/cjs/platform/client.js +73 -16
  20. package/dist/cjs/platform/index.js +4 -0
  21. package/dist/cjs/platform/types.js +7 -1
  22. package/dist/detect-client-shell.js +53 -0
  23. package/dist/extract-meta-refresh.js +47 -0
  24. package/dist/extractors/lib/sanitize.js +4 -0
  25. package/dist/html-to-mdast.js +2 -1
  26. package/dist/index.js +14 -0
  27. package/dist/link-replacer.js +19 -13
  28. package/dist/loaders/cf-puppeteer.js +15 -8
  29. package/dist/loaders/fetch.js +14 -2
  30. package/dist/loaders/playwright.js +32 -30
  31. package/dist/loaders/puppeteer.js +15 -8
  32. package/dist/md-splitter.js +24 -11
  33. package/dist/mdast-handlers/custom-code-handler.js +17 -7
  34. package/dist/mdast-handlers/custom-div-handler.js +12 -33
  35. package/dist/mdast-handlers/definition-list-handler.js +17 -6
  36. package/dist/mdast-to-markdown.js +2 -5
  37. package/dist/normalize/index.js +12 -2
  38. package/dist/platform/client.js +73 -16
  39. package/dist/platform/index.js +4 -0
  40. package/dist/platform/types.js +5 -1
  41. package/dist/types/cli/bin.js.map +1 -1
  42. package/dist/types/cli/commands/jobs/files.d.ts +28 -0
  43. package/dist/types/cli/commands/jobs/files.js.map +1 -0
  44. package/dist/types/cli/commands/jobs/llms-txt.d.ts +33 -0
  45. package/dist/types/cli/commands/jobs/llms-txt.js.map +1 -0
  46. package/dist/types/cli/commands/jobs/options.d.ts +58 -0
  47. package/dist/types/cli/commands/jobs/options.js.map +1 -0
  48. package/dist/types/cli/commands/jobs/run.d.ts +44 -0
  49. package/dist/types/cli/commands/jobs/run.js.map +1 -0
  50. package/dist/types/cli/commands/skill/content.js.map +1 -1
  51. package/dist/types/cli/commands/webforai/interactive.d.ts +1 -1
  52. package/dist/types/cli/commands/webforai/interactive.js.map +1 -1
  53. package/dist/types/cli/commands/webforai/options.d.ts +9 -2
  54. package/dist/types/cli/commands/webforai/options.js.map +1 -1
  55. package/dist/types/cli/commands/webforai/run.js.map +1 -1
  56. package/dist/types/cli/utils.d.ts +8 -1
  57. package/dist/types/cli/utils.js.map +1 -1
  58. package/dist/types/detect-client-shell.d.ts +24 -0
  59. package/dist/types/detect-client-shell.js.map +1 -0
  60. package/dist/types/extract-meta-refresh.d.ts +29 -0
  61. package/dist/types/extract-meta-refresh.js.map +1 -0
  62. package/dist/types/extractors/lib/sanitize.js.map +1 -1
  63. package/dist/types/html-to-mdast.js.map +1 -1
  64. package/dist/types/index.d.ts +2 -0
  65. package/dist/types/index.js.map +1 -1
  66. package/dist/types/link-replacer.d.ts +3 -1
  67. package/dist/types/link-replacer.js.map +1 -1
  68. package/dist/types/loaders/cf-puppeteer.js.map +1 -1
  69. package/dist/types/loaders/fetch.d.ts +4 -0
  70. package/dist/types/loaders/fetch.js.map +1 -1
  71. package/dist/types/loaders/playwright.js.map +1 -1
  72. package/dist/types/loaders/puppeteer.js.map +1 -1
  73. package/dist/types/md-splitter.d.ts +1 -1
  74. package/dist/types/md-splitter.js.map +1 -1
  75. package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
  76. package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
  77. package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
  78. package/dist/types/mdast-handlers/definition-list-handler.d.ts +0 -11
  79. package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -1
  80. package/dist/types/mdast-to-markdown.js.map +1 -1
  81. package/dist/types/normalize/index.js.map +1 -1
  82. package/dist/types/platform/client.d.ts +2 -1
  83. package/dist/types/platform/client.js.map +1 -1
  84. package/dist/types/platform/error.d.ts +5 -3
  85. package/dist/types/platform/error.js.map +1 -1
  86. package/dist/types/platform/fetch.d.ts +2 -0
  87. package/dist/types/platform/fetch.js.map +1 -1
  88. package/dist/types/platform/index.d.ts +1 -1
  89. package/dist/types/platform/index.js.map +1 -1
  90. package/dist/types/platform/types.d.ts +35 -6
  91. package/dist/types/platform/types.js.map +1 -1
  92. package/package.json +16 -5
@@ -2,19 +2,29 @@
2
2
  import { toText } from "hast-util-to-text";
3
3
  import { trimTrailingLines } from "trim-trailing-lines";
4
4
  import { detectLanguage } from "../utils/detect-code-lang.js";
5
- var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
5
+ import { classList, isElement } from "../utils/hast-fast.js";
6
+ var LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
7
+ var LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
8
+ var attributeLanguage = (element) => {
9
+ for (const name of LANGUAGE_ATTRIBUTES) {
10
+ const value = element.properties?.[name];
11
+ if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
12
+ return value.trim();
13
+ }
14
+ }
15
+ return void 0;
16
+ };
17
+ var codeLanguage = (...elements) => elements.flatMap(classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
6
18
  var customCodeHandler = (state, node) => {
7
- const classNames = node.properties?.className || [];
8
- const codeValue = trimTrailingLines(toText(node)).trim();
9
- const classLang = classNames.map((className) => {
10
- const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
11
- return match?.[1];
12
- }).find((className) => className);
19
+ const code = node.children.find((child) => isElement(child) && child.tagName === "code");
20
+ const codeValue = trimTrailingLines(toText(node));
21
+ const classLang = codeLanguage(...isElement(code) ? [code, node] : [node]);
13
22
  const lang = classLang || detectLanguage(codeValue) || null;
14
23
  const result = { type: "code", lang, meta: null, value: codeValue };
15
24
  state.patch(node, result);
16
25
  return result;
17
26
  };
18
27
  export {
28
+ codeLanguage,
19
29
  customCodeHandler
20
30
  };
@@ -5,44 +5,23 @@ import { toString as hastToString } from "hast-util-to-string";
5
5
  import { toText } from "hast-util-to-text";
6
6
  import { trimTrailingLines } from "trim-trailing-lines";
7
7
  import { detectLanguage } from "../utils/detect-code-lang.js";
8
+ import { classList, collectElements, isElement } from "../utils/hast-fast.js";
9
+ import { codeLanguage } from "./custom-code-handler.js";
8
10
  var CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
9
11
  var CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
10
- var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
11
- var findRecursive = (array, condition, maxDepth = 3) => {
12
- if (maxDepth <= 0) {
13
- return null;
14
- }
15
- for (const value of array) {
16
- const result = condition(value);
17
- if (Array.isArray(result)) {
18
- return findRecursive(result, condition, maxDepth - 1);
19
- }
20
- if (result) {
21
- return value;
22
- }
23
- }
24
- return null;
25
- };
26
12
  var customDivHandler = (state, node) => {
27
- const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
28
- const codeBlock = findRecursive(node.children, (child) => {
29
- if (child.type !== "element") {
30
- return false;
31
- }
32
- if (child.tagName === "pre") {
33
- return true;
34
- }
35
- return child.children.filter((child2) => child2.type === "element");
36
- });
37
- if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
38
- const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
39
- const codeValue = trimTrailingLines(toText(codeBlock)).trim();
13
+ const classNames = classList(node);
14
+ if (!classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
15
+ return defaultHandlers.div(state, node);
16
+ }
17
+ const codeBlocks = collectElements(node, (element) => element.tagName === "pre");
18
+ const codeBlock = codeBlocks.length === 1 ? codeBlocks[0] : void 0;
19
+ if (codeBlock) {
20
+ const codeValue = trimTrailingLines(toText(codeBlock));
40
21
  const filenameElement = select(CODE_FILENAME_SELECTORS, node);
41
22
  const fileLang = filenameElement ? hastToString(filenameElement).match(/\.(\w+)$/)?.[1] : null;
42
- const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
43
- const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
44
- return match?.[1];
45
- }).find((className) => className);
23
+ const code = codeBlock.children.find((child) => isElement(child) && child.tagName === "code");
24
+ const classLang = codeLanguage(...isElement(code) ? [code, codeBlock, node] : [codeBlock, node]);
46
25
  const lang = fileLang || classLang || detectLanguage(codeValue) || null;
47
26
  const result = { type: "code", lang, meta: null, value: codeValue };
48
27
  state.patch(node, result);
@@ -28,6 +28,20 @@ var hoistBoundaryWhitespace = (nodes) => {
28
28
  }
29
29
  return result;
30
30
  };
31
+ var termsAndDefinitions = (parent) => {
32
+ const found = [];
33
+ for (const child of parent.children) {
34
+ if (child.type !== "element") {
35
+ continue;
36
+ }
37
+ if (child.tagName === "dt" || child.tagName === "dd") {
38
+ found.push(child);
39
+ } else if (child.tagName === "div") {
40
+ found.push(...termsAndDefinitions(child));
41
+ }
42
+ }
43
+ return found;
44
+ };
31
45
  var definitionListHandler = (state, node) => {
32
46
  const items = [];
33
47
  let current;
@@ -42,11 +56,7 @@ var definitionListHandler = (state, node) => {
42
56
  items.push({ type: "listItem", spread: false, children: [heading, ...current.definitions] });
43
57
  current = void 0;
44
58
  };
45
- for (const child of node.children) {
46
- if (child.type !== "element") {
47
- continue;
48
- }
49
- const element = child;
59
+ for (const element of termsAndDefinitions(node)) {
50
60
  if (element.tagName === "dt") {
51
61
  if (current && current.definitions.length > 0) {
52
62
  flush();
@@ -65,7 +75,8 @@ var definitionListHandler = (state, node) => {
65
75
  }
66
76
  flush();
67
77
  if (items.length === 0) {
68
- return void 0;
78
+ const fallback = state.toFlow(state.all(node));
79
+ return fallback.length > 0 ? fallback : void 0;
69
80
  }
70
81
  const result = { type: "list", ordered: false, spread: false, children: items };
71
82
  state.patch(node, result);
@@ -10,11 +10,8 @@ var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
10
10
  };
11
11
  var mdastToMarkdown = (mdast, options) => {
12
12
  const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
13
- let markdown = toMarkdown(warpRoot(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
14
- if (baseUrl) {
15
- markdown = linkReplacer(markdown, baseUrl);
16
- }
17
- return markdown;
13
+ const tree = warpRoot(mdast);
14
+ return toMarkdown(baseUrl ? linkReplacer(tree, baseUrl) : tree, toMarkdownOptions);
18
15
  };
19
16
  export {
20
17
  DEFAULT_MDAST_TO_MARKDOWN_OPTIONS,
@@ -1,6 +1,7 @@
1
1
  // src/normalize/index.ts
2
2
  import {
3
3
  classList,
4
+ findElement,
4
5
  isElement,
5
6
  numericProperty,
6
7
  property,
@@ -80,14 +81,16 @@ var katexDuplicates = (node) => {
80
81
  if (!(classes.includes("katex") || classes.includes("katex-display"))) {
81
82
  return [];
82
83
  }
83
- const hasMathml = node.children.some((child) => isElement(child) && classList(child).includes("katex-mathml"));
84
+ const hasMathml = node.children.some(
85
+ (child) => isElement(child) && classList(child).includes("katex-mathml") && hasMathSource(child)
86
+ );
84
87
  if (!hasMathml) {
85
88
  return [];
86
89
  }
87
90
  return node.children.filter((child) => isElement(child) && classList(child).includes("katex-html"));
88
91
  };
89
92
  var mediawikiDuplicates = (node) => {
90
- if (!classList(node).includes("mwe-math-element")) {
93
+ if (!classList(node).includes("mwe-math-element") || !hasMathSource(node)) {
91
94
  return [];
92
95
  }
93
96
  return node.children.filter(
@@ -98,8 +101,15 @@ var mathjaxDuplicates = (node) => {
98
101
  if (node.tagName !== "mjx-container") {
99
102
  return [];
100
103
  }
104
+ const hasMathml = node.children.some(
105
+ (child) => isElement(child) && child.tagName === "mjx-assistive-mml" && hasMathSource(child)
106
+ );
107
+ if (!hasMathml) {
108
+ return [];
109
+ }
101
110
  return node.children.filter((child) => isElement(child) && child.tagName !== "mjx-assistive-mml");
102
111
  };
112
+ var hasMathSource = (node) => Boolean(findElement(node, (element) => element.tagName === "math" && element.children.length > 0));
103
113
  var dedupeRenderedMath = (tree) => {
104
114
  const doomed = /* @__PURE__ */ new Set();
105
115
  walk(tree, (node) => {
@@ -29,14 +29,28 @@ var parseRetryAfter = (body, response) => {
29
29
  const header = Number(response.headers.get("retry-after"));
30
30
  return Number.isFinite(header) && header > 0 ? header : void 0;
31
31
  };
32
- var toApiError = (response, body) => {
32
+ var dashboardHint = (status, dashboardUrl) => {
33
+ if (status === 401) {
34
+ return `create or check API keys at ${dashboardUrl}`;
35
+ }
36
+ if (status === 402) {
37
+ return `manage billing, credits and the spend cap at ${dashboardUrl}`;
38
+ }
39
+ return void 0;
40
+ };
41
+ var toApiError = (response, body, dashboardUrl) => {
33
42
  const envelope = body;
34
43
  const code = typeof envelope?.error?.code === "string" ? envelope.error.code : "invalid_response";
35
- const message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
44
+ let message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
45
+ const hint = code === "invalid_response" ? void 0 : dashboardHint(response.status, dashboardUrl);
46
+ if (hint && !message.includes("/dashboard")) {
47
+ message = `${message} (${hint})`;
48
+ }
36
49
  return new PlatformApiError(code, message, response.status, parseRetryAfter(body, response));
37
50
  };
38
51
  var createPlatformClient = (options = {}) => {
39
52
  const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
53
+ const dashboardUrl = `${baseUrl}/dashboard`;
40
54
  const fetchImpl = resolveFetch(options.fetch);
41
55
  const request = async (path, init) => {
42
56
  const headers = { accept: "application/json" };
@@ -47,7 +61,7 @@ var createPlatformClient = (options = {}) => {
47
61
  if (!options.apiKey) {
48
62
  throw new PlatformApiError(
49
63
  "missing_api_key",
50
- "This endpoint needs an API key: pass `apiKey` to createPlatformClient (create one on the platform dashboard).",
64
+ `This endpoint needs an API key: pass \`apiKey\` to createPlatformClient (create one at ${dashboardUrl}; 1,000 free credits/month).`,
51
65
  0
52
66
  );
53
67
  }
@@ -56,18 +70,19 @@ var createPlatformClient = (options = {}) => {
56
70
  const response = await fetchImpl(`${baseUrl}${path}`, {
57
71
  method: init.method,
58
72
  headers,
59
- body: init.body === void 0 ? void 0 : JSON.stringify(init.body)
73
+ body: init.body === void 0 ? void 0 : JSON.stringify(init.body),
74
+ ...init.signal ? { signal: init.signal } : {}
60
75
  });
61
76
  const body = await response.json().catch(() => void 0);
62
77
  if (!response.ok) {
63
- throw toApiError(response, body);
78
+ throw toApiError(response, body, dashboardUrl);
64
79
  }
65
80
  if (body === void 0) {
66
81
  throw new PlatformApiError("invalid_response", "response body is not JSON", response.status);
67
82
  }
68
83
  return body;
69
84
  };
70
- const getJob = (jobId) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true });
85
+ const getJob = (jobId, signal) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true, signal });
71
86
  const getJobResults = (jobId, opts) => {
72
87
  const query = opts?.cursor ? `?cursor=${encodeURIComponent(opts.cursor)}` : "";
73
88
  return request(`/v1/jobs/${encodeURIComponent(jobId)}/results${query}`, { method: "GET", auth: true });
@@ -83,6 +98,17 @@ var createPlatformClient = (options = {}) => {
83
98
  }
84
99
  return await response.json();
85
100
  };
101
+ const getJobOrBackOff = async (jobId, signal, interval) => {
102
+ try {
103
+ return await getJob(jobId, signal);
104
+ } catch (error) {
105
+ if (!(error instanceof PlatformApiError && error.status === 429)) {
106
+ throw error;
107
+ }
108
+ await sleep(error.retryAfter !== void 0 ? error.retryAfter * 1e3 : interval, signal);
109
+ return void 0;
110
+ }
111
+ };
86
112
  return {
87
113
  scrape: (opts) => request("/v1/scrape", { method: "POST", body: opts, auth: true }),
88
114
  scrapeAsync: (opts) => request("/v1/scrape", { method: "POST", body: { ...opts, async: true }, auth: true }),
@@ -93,17 +119,48 @@ var createPlatformClient = (options = {}) => {
93
119
  waitForJob: async (jobId, opts = {}) => {
94
120
  const interval = opts.pollIntervalMs ?? 2e3;
95
121
  const timeout = opts.timeoutMs ?? 10 * 60 * 1e3;
96
- const startedAt = Date.now();
97
- for (; ; ) {
98
- const status = await getJob(jobId);
99
- opts.onStatus?.(status);
100
- if (TERMINAL_STATES.has(status.status)) {
101
- return status;
102
- }
103
- if (Date.now() - startedAt + interval > timeout) {
104
- throw new PlatformApiError("poll_timeout", `job ${jobId} still ${status.status} after ${timeout}ms`, 0);
122
+ if (!Number.isFinite(interval) || interval < 0 || interval > 2147483647) {
123
+ throw new RangeError("pollIntervalMs must be between 0 and 2147483647");
124
+ }
125
+ if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647) {
126
+ throw new RangeError("timeoutMs must be greater than 0 and at most 2147483647");
127
+ }
128
+ opts.signal?.throwIfAborted();
129
+ const controller = new AbortController();
130
+ const { signal } = controller;
131
+ const onAbort = () => controller.abort(opts.signal?.reason);
132
+ opts.signal?.addEventListener("abort", onAbort, { once: true });
133
+ const timer = setTimeout(
134
+ () => controller.abort(new PlatformApiError("poll_timeout", `job ${jobId} did not finish within ${timeout}ms`, 0)),
135
+ timeout
136
+ );
137
+ let rejectOnAbort = () => {
138
+ };
139
+ const aborted = new Promise((_resolve, reject) => {
140
+ rejectOnAbort = () => reject(signal.reason);
141
+ signal.addEventListener("abort", rejectOnAbort, { once: true });
142
+ });
143
+ const poll = async () => {
144
+ for (; ; ) {
145
+ signal.throwIfAborted();
146
+ const status = await getJobOrBackOff(jobId, signal, interval);
147
+ signal.throwIfAborted();
148
+ if (status === void 0) {
149
+ continue;
150
+ }
151
+ opts.onStatus?.(status);
152
+ if (TERMINAL_STATES.has(status.status)) {
153
+ return status;
154
+ }
155
+ await sleep(interval, signal);
105
156
  }
106
- await sleep(interval, opts.signal);
157
+ };
158
+ try {
159
+ return await Promise.race([poll(), aborted]);
160
+ } finally {
161
+ clearTimeout(timer);
162
+ opts.signal?.removeEventListener("abort", onAbort);
163
+ signal.removeEventListener("abort", rejectOnAbort);
107
164
  }
108
165
  },
109
166
  jobResults: async function* (jobId) {
@@ -5,15 +5,19 @@ import {
5
5
  } from "./client.js";
6
6
  import { PlatformApiError } from "./error.js";
7
7
  import {
8
+ CRAWL_SITEMAP_MODES,
8
9
  ENGINES,
9
10
  REGIONS,
11
+ REQUESTED_ENGINES,
10
12
  isStoredPageStub
11
13
  } from "./types.js";
12
14
  export {
15
+ CRAWL_SITEMAP_MODES,
13
16
  DEFAULT_BASE_URL,
14
17
  ENGINES,
15
18
  PlatformApiError,
16
19
  REGIONS,
20
+ REQUESTED_ENGINES,
17
21
  createPlatformClient,
18
22
  isStoredPageStub
19
23
  };
@@ -1,9 +1,13 @@
1
1
  // src/platform/types.ts
2
2
  var ENGINES = ["fetch", "browser", "proxy-fetch", "proxy-browser"];
3
- var REGIONS = ["auto", "us", "eu", "uk", "jp", "asia"];
3
+ var REQUESTED_ENGINES = [...ENGINES, "auto"];
4
+ var REGIONS = ["auto", "jp"];
5
+ var CRAWL_SITEMAP_MODES = ["skip", "include", "only"];
4
6
  var isStoredPageStub = (item) => item.status === "ok" && "resultUrl" in item;
5
7
  export {
8
+ CRAWL_SITEMAP_MODES,
6
9
  ENGINES,
7
10
  REGIONS,
11
+ REQUESTED_ENGINES,
8
12
  isStoredPageStub
9
13
  };
@@ -1 +1 @@
1
- {"version":3,"file":"bin.js","sourceRoot":"","sources":["../../../src/cli/bin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,WAAW,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAmB,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,OAAO,EAAiB,UAAU,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAExF,MAAM,aAAa,GAAG;;;;;;;;;;;;IAYlB,WAAW;IACX,gBAAgB;;;;;;;CAOnB,CAAC;AAEF,OAAO;KACL,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,gEAAgE,CAAC;KAC7E,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,eAAe,EAAE,4BAA4B,CAAC;KAC3E,WAAW,CAAC,OAAO,EAAE,aAAa,CAAC;KACnC,wBAAwB,EAAE,CAAC;AAE7B,OAAO;KACL,QAAQ,CAAC,UAAU,EAAE,mCAAmC,CAAC;KACzD,MAAM,CAAC,qBAAqB,EAAE,4CAA4C,CAAC;KAC3E,MAAM,CAAC,uBAAuB,EAAE,yBAAyB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC;KAC3G,MAAM,CAAC,mBAAmB,EAAE,oBAAoB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC;KACpG,MAAM,CAAC,yBAAyB,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KAC1G,MAAM,CAAC,eAAe,EAAE,wDAAwD,CAAC;KACjF,MAAM,CAAC,QAAQ,EAAE,gFAAgF,CAAC;KAClG,MAAM,CAAC,mBAAmB,EAAE,sFAAsF,CAAC;KACnH,MAAM,CAAC,mBAAmB,EAAE,qFAAqF,CAAC;KAClH,MAAM,CAAC,cAAc,EAAE,oEAAoE,CAAC;KAC5F,MAAM,CAAC,iBAAiB,EAAE,kCAAkC,WAAW,GAAG,CAAC;KAC3E,MAAM,CAAC,sBAAsB,EAAE,mCAAmC,gBAAgB,0BAA0B,CAAC;KAC7G,MAAM,CAAC,mBAAmB,EAAE,yCAAyC,CAAC;KACtE,MAAM,CAAC,aAAa,EAAE,wBAAwB,CAAC;KAC/C,MAAM,CAAC,KAAK,EAAE,MAA0B,EAAE,KAA2C,EAAE,EAAE;IACzF,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACvB,MAAM,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxC,OAAO;IACR,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,UAAU,CAAC,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;AACpD,CAAC,CAAC,CAAC;AAEJ,OAAO;KACL,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,iFAAiF,CAAC;KAC9F,MAAM,CAAC,WAAW,EAAE,wFAAwF,CAAC;KAC7G,MAAM,CAAC,cAAc,EAAE,oEAAoE,CAAC;KAC5F,MAAM,CAAC,sBAAsB,EAAE,2DAA2D,CAAC;KAC3F,MAAM,CAAC,WAAW,EAAE,kDAAkD,CAAC;KACvE,MAAM,CAAC,QAAQ,EAAE,yDAAyD,CAAC;KAC3E,MAAM,CAAC,OAAO,EAAE,2DAA2D,CAAC;KAC5E,MAAM,CAAC,cAAc,EAAE,mFAAmF,CAAC;KAC3G,WAAW,CAAC,OAAO,EAAE,qFAAqF,CAAC;KAC3G,MAAM,CAAC,KAAK,EAAE,KAAiB,EAAE,EAAE;IACnC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC,CAAC,CAAC;AAEJ,MAAM,IAAI,GAAG,CAAC,KAAc,EAAS,EAAE;IACtC,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;QACrC,uEAAuE;QACvE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC;AAEF,OAAO,CAAC,YAAY,EAAE,CAAC;AACvB,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC"}
1
+ {"version":3,"file":"bin.js","sourceRoot":"","sources":["../../../src/cli/bin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,cAAc,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAClE,OAAO,WAAW,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAiB,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAmB,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,OAAO,EAAiB,UAAU,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACxF,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAE/C,MAAM,aAAa,GAAG;;;;;;;;;;;;;;IAclB,WAAW;IACX,gBAAgB;;;;;;;;;;;IAWhB,gBAAgB;CACnB,CAAC;AAEF,+EAA+E;AAC/E,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,mBAAmB,CAAC,CAAC,CAAC;AAE/F,4FAA4F;AAC5F,IAAI,iBAAqC,CAAC;AAC1C,MAAM,mBAAmB,GAAG,CAAC,KAA+B,EAAE,EAAE;IAC/D,iBAAiB,GAAG,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,OAAO;KACL,IAAI,CAAC,UAAU,CAAC;IACjB,2FAA2F;KAC1F,uBAAuB,EAAE;IAC1B,8EAA8E;KAC7E,YAAY,EAAE;KACd,WAAW,CAAC,gEAAgE,CAAC;KAC7E,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,eAAe,EAAE,4BAA4B,CAAC;KAC3E,WAAW,CAAC,OAAO,EAAE,aAAa,CAAC;KACnC,wBAAwB,EAAE,CAAC;AAE7B,OAAO;KACL,QAAQ,CAAC,UAAU,EAAE,mCAAmC,CAAC;KACzD,MAAM,CAAC,qBAAqB,EAAE,4CAA4C,CAAC;KAC3E,MAAM,CAAC,uBAAuB,EAAE,yBAAyB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC;KAC3G,MAAM,CAAC,mBAAmB,EAAE,oBAAoB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC;KACpG,MAAM,CAAC,yBAAyB,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KAC1G,MAAM,CAAC,eAAe,EAAE,wDAAwD,CAAC;KACjF,MAAM,CAAC,QAAQ,EAAE,gFAAgF,CAAC;KAClG,MAAM,CACN,mBAAmB,EACnB,uHAAuH,CACvH;KACA,MAAM,CAAC,mBAAmB,EAAE,qFAAqF,CAAC;KAClH,MAAM,CAAC,cAAc,EAAE,oEAAoE,CAAC;KAC5F,MAAM,CAAC,kBAAkB,EAAE,4EAA4E,CAAC;KACxG,MAAM,CAAC,iBAAiB,EAAE,kCAAkC,WAAW,GAAG,CAAC;KAC3E,MAAM,CAAC,sBAAsB,EAAE,mCAAmC,gBAAgB,0BAA0B,CAAC;KAC7G,MAAM,CAAC,mBAAmB,EAAE,yCAAyC,CAAC;KACtE,MAAM,CAAC,aAAa,EAAE,wBAAwB,CAAC;KAC/C,MAAM,CAAC,KAAK,EAAE,MAA0B,EAAE,KAA2C,EAAE,EAAE;IACzF,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACvB,MAAM,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxC,OAAO;IACR,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,UAAU,CAAC,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;AACpD,CAAC,CAAC,CAAC;AAEJ,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,WAAqB,EAAE,EAAY,EAAE,CAAC,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,CAAC;AAE3F,gGAAgG;AAChG,MAAM,qBAAqB,GAAG,CAAC,OAAgB,EAAW,EAAE,CAC3D,OAAO;KACL,cAAc,CAAC,oBAAoB,EAAE,qEAAqE,CAAC;KAC3G,MAAM,CACN,mBAAmB,EACnB,kGAAkG,CAClG;KACA,MAAM,CAAC,mBAAmB,EAAE,iEAAiE,CAAC;KAC9F,MAAM,CAAC,yBAAyB,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;KAC/F,MAAM,CAAC,eAAe,EAAE,2DAA2D,CAAC;KACpF,MAAM,CAAC,QAAQ,EAAE,yFAAyF,CAAC;KAC3G,MAAM,CAAC,qBAAqB,EAAE,oEAAoE,CAAC;KACnG,MAAM,CAAC,iBAAiB,EAAE,kCAAkC,WAAW,GAAG,CAAC;KAC3E,MAAM,CAAC,sBAAsB,EAAE,mCAAmC,gBAAgB,0BAA0B,CAAC;KAC7G,MAAM,CAAC,aAAa,EAAE,wBAAwB,CAAC,CAAC;AAEnD,MAAM,eAAe,GAAG;;;;;CAKvB,CAAC;AAEF,qBAAqB,CACpB,OAAO;KACL,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,0EAA0E,CAAC;KACvF,QAAQ,CAAC,OAAO,EAAE,+CAA+C,CAAC,CACpE;KACC,MAAM,CAAC,iBAAiB,EAAE,2CAA2C,CAAC;KACtE,MAAM,CAAC,aAAa,EAAE,mCAAmC,CAAC;KAC1D,MAAM,CAAC,mBAAmB,EAAE,uDAAuD,EAAE,OAAO,CAAC;KAC7F,MAAM,CAAC,mBAAmB,EAAE,iDAAiD,EAAE,OAAO,CAAC;KACvF,MAAM,CACN,kBAAkB,EAClB,+FAA+F,CAC/F;KACA,MAAM,CAAC,kBAAkB,EAAE,2CAA2C,CAAC;KACvE,MAAM,CAAC,qBAAqB,EAAE,wBAAwB,CAAC;KACvD,MAAM,CAAC,YAAY,EAAE,gFAAgF,CAAC;KACtG,WAAW,CACX,OAAO,EACP,GAAG,eAAe;;;;CAInB,CACC;KACA,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,KAAe,EAAE,EAAE;IAC9C,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,aAAa,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC,CAAC,CAAC;AAEJ,qBAAqB,CACpB,OAAO;KACL,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,sEAAsE,CAAC;KACnF,QAAQ,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAC9D;KACC,MAAM,CAAC,mBAAmB,EAAE,yEAAyE,CAAC;KACtG,MAAM,CAAC,kBAAkB,EAAE,2DAA2D,CAAC;KACvF,WAAW,CACX,OAAO,EACP,GAAG,eAAe;;;;CAInB,CACC;KACA,MAAM,CAAC,KAAK,EAAE,IAAc,EAAE,KAAe,EAAE,EAAE;IACjD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,aAAa,CAAC,iBAAiB,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAC9D,CAAC,CAAC,CAAC;AAEJ,OAAO;KACL,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,iFAAiF,CAAC;KAC9F,MAAM,CAAC,WAAW,EAAE,wFAAwF,CAAC;KAC7G,MAAM,CAAC,cAAc,EAAE,oEAAoE,CAAC;KAC5F,MAAM,CAAC,sBAAsB,EAAE,2DAA2D,CAAC;KAC3F,MAAM,CAAC,WAAW,EAAE,kDAAkD,CAAC;KACvE,MAAM,CAAC,QAAQ,EAAE,yDAAyD,CAAC;KAC3E,MAAM,CAAC,OAAO,EAAE,2DAA2D,CAAC;KAC5E,MAAM,CAAC,cAAc,EAAE,mFAAmF,CAAC;KAC3G,WAAW,CAAC,OAAO,EAAE,qFAAqF,CAAC;KAC3G,MAAM,CAAC,KAAK,EAAE,KAAiB,EAAE,EAAE;IACnC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC,CAAC,CAAC;AAEJ,MAAM,IAAI,GAAG,CAAC,KAAc,EAAS,EAAE;IACtC,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;QACrC,uEAAuE;QACvE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,YAAY,gBAAgB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB,KAAK,CAAC,UAAU,IAAI,CAAC;QAC1F,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChE,oFAAoF;QACpF,IAAI,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAC/E,OAAO,CAAC,KAAK,CAAC,wCAAwC,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;QACzC,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC;AAEF,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC"}
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Maps a page URL to a relative `.md` path that mirrors the site's structure:
3
+ * `/` → `index.md`, `/docs/` → `docs/index.md`, `/docs/intro.html` → `docs/intro.md`.
4
+ * A query string adds a short stable hash (`search_q-1a2b3c4d.md`) so `?page=2` does not
5
+ * overwrite `?page=1`. `includeHost` prefixes the hostname (batch jobs span origins).
6
+ *
7
+ * The result never escapes the output directory: segments are sanitized one by one, so
8
+ * `..`, encoded slashes and absolute paths cannot survive.
9
+ */
10
+ export declare const pageRelativePath: (pageUrl: string, options?: {
11
+ includeHost?: boolean;
12
+ }) => string;
13
+ export interface PlannedFile {
14
+ url: string;
15
+ /** Path relative to the output directory, POSIX separators. */
16
+ relativePath: string;
17
+ }
18
+ /**
19
+ * Assigns every URL a distinct relative path. URLs are sorted first so the same set of pages
20
+ * always gets the same names, whatever order the job returned them in; collisions (e.g.
21
+ * `/a` and `/a.html`) take the CLI's usual `_1`, `_2` suffixes. Comparison is
22
+ * case-insensitive because macOS and Windows filesystems are.
23
+ */
24
+ export declare const planOutputFiles: (urls: readonly string[], options?: {
25
+ includeHost?: boolean;
26
+ }) => PlannedFile[];
27
+ /** Resolves a planned path inside `outputDir`, refusing anything that would land outside it. */
28
+ export declare const resolveInside: (outputDir: string, relativePath: string) => string;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"files.js","sourceRoot":"","sources":["../../../../../src/cli/commands/jobs/files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAEvD,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,gGAAgG;AAChG,MAAM,cAAc,GAAG,iCAAiC,CAAC;AAEzD,MAAM,SAAS,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAE1G,MAAM,aAAa,GAAG,CAAC,OAAe,EAAU,EAAE;IACjD,IAAI,CAAC;QACJ,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,OAAO,CAAC;IAChB,CAAC;AACF,CAAC,CAAC;AAEF,iGAAiG;AACjG,MAAM,eAAe,GAAG,CAAC,OAAe,EAAU,EAAE;IACnD,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC;SACpC,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,oCAAoC,EAAE,GAAG,CAAC;SAClD,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;IACjC,IAAI,OAAO,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC;IAChB,CAAC;IACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,GAAG,CAAC,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5E,CAAC,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,OAAe,EAAE,UAAqC,EAAE,EAAU,EAAE;IACpG,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,MAAM,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,aAAa,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAEtF,IAAI,IAAI,GAAG,aAAa,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC;IACxF,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC;IACnD,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,IAAI,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACxG,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC;AACtD,CAAC,CAAC;AAQF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAuB,EAAE,UAAqC,EAAE,EAAiB,EAAE;IAClH,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACzB,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,wBAAwB,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAClG,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC;QACtC,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,gGAAgG;AAChG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,SAAiB,EAAE,YAAoB,EAAU,EAAE;IAChF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mDAAmD,YAAY,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC,CAAC"}
@@ -0,0 +1,33 @@
1
+ /**
2
+ * llms.txt / llms-full.txt rendering for crawl output, per https://llmstxt.org:
3
+ *
4
+ * ```md
5
+ * # Site title
6
+ *
7
+ * > One-line summary
8
+ *
9
+ * ## Section
10
+ *
11
+ * - [Page title](https://url): description
12
+ * ```
13
+ *
14
+ * `llms-full.txt` is the companion convention: every page's Markdown, concatenated, so an
15
+ * agent can load a whole site in one read.
16
+ */
17
+ export interface LlmsPage {
18
+ url: string;
19
+ markdown: string;
20
+ metadata: Record<string, unknown>;
21
+ }
22
+ export interface LlmsSite {
23
+ /** The crawl's seed URL; its page supplies the site title and summary when present. */
24
+ seedUrl: string;
25
+ pages: readonly LlmsPage[];
26
+ }
27
+ /** Pages in output order: grouped by section (top level first, then A→Z), URL order inside. */
28
+ export declare const orderPages: (pages: readonly LlmsPage[]) => {
29
+ section: string;
30
+ pages: LlmsPage[];
31
+ }[];
32
+ export declare const renderLlmsTxt: (site: LlmsSite) => string;
33
+ export declare const renderLlmsFullTxt: (site: LlmsSite) => string;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"llms-txt.js","sourceRoot":"","sources":["../../../../../src/cli/commands/jobs/llms-txt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAcH,MAAM,GAAG,GAAG,CAAC,KAAc,EAAsB,EAAE,CAClD,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEnG,MAAM,YAAY,GAAG,CAAC,QAAgB,EAAsB,EAAE;IAC7D,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5D,OAAO,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1C,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,cAAc,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;AAErF,MAAM,SAAS,GAAG,CAAC,IAAc,EAAU,EAAE,CAC5C,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;AAErG,MAAM,YAAY,GAAG,CAAC,KAAa,EAAU,EAAE;IAC9C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;IACd,OAAO,GAAG,CAAC,IAAI,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,IAAc,EAAwB,EAAE;IACzD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;AACnE,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,IAAc,EAAE,IAA0B,EAAU,EAAE,CACxE,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC5B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC9E,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC;IACzB,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;AAE5B;;;GAGG;AACH,MAAM,SAAS,GAAG,CAAC,OAAe,EAAU,EAAE;IAC7C,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAClE,OAAO,EAAE,CAAC;IACX,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,CAAC;QACJ,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACR,uBAAuB;IACxB,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,+FAA+F;AAC/F,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAA0B,EAA4C,EAAE;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;SACvE,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;AAC9D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAc,EAAU,EAAE;IACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAChD,IAAI,OAAO,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;IAChC,CAAC;IACD,KAAK,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzD,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,IAAI,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5D,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC;AAC1C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAc,EAAU,EAAE;IAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7C,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClC,4EAA4E;YAC5E,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;YACxE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,WAAW,IAAI,CAAC,GAAG,OAAO,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;IACF,CAAC;IACD,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACzC,CAAC,CAAC"}
@@ -0,0 +1,58 @@
1
+ import type { CrawlSitemapMode, Region, RequestedEngine } from "../../../platform";
2
+ import { type ExtractorName } from "../../constants";
3
+ /** Server-side caps (docs/specs/platform/03_api.md); checked locally to fail before billing. */
4
+ export declare const MAX_BATCH_URLS = 100;
5
+ export declare const MAX_CRAWL_DEPTH = 5;
6
+ export declare const MAX_CRAWL_LIMIT = 500;
7
+ export type JobKind = "crawl" | "batch";
8
+ export interface JobFlags {
9
+ output?: string;
10
+ engine?: string;
11
+ region?: string;
12
+ extractor?: string;
13
+ frontmatter?: boolean;
14
+ /** `true`/`false` from `--respect-robots`/`--no-respect-robots`; undefined = server default. */
15
+ respectRobots?: boolean;
16
+ apiKey?: string;
17
+ platformUrl?: string;
18
+ json?: boolean;
19
+ debug?: boolean;
20
+ /** Seconds to wait for the job before giving up (it keeps running server-side). */
21
+ timeout?: string;
22
+ maxDepth?: string;
23
+ limit?: string;
24
+ include?: string[];
25
+ exclude?: string[];
26
+ sitemap?: string;
27
+ llmsTxt?: boolean;
28
+ file?: string;
29
+ }
30
+ export interface ResolvedJob {
31
+ kind: JobKind;
32
+ /** crawl: exactly one seed URL; batch: 1–100 URLs, de-duplicated, in input order. */
33
+ urls: string[];
34
+ outputDir: string;
35
+ engine?: RequestedEngine;
36
+ region?: Region;
37
+ extractor: ExtractorName;
38
+ frontmatter: boolean;
39
+ respectRobotsTxt?: boolean;
40
+ apiKey: string;
41
+ platformUrl?: string;
42
+ json: boolean;
43
+ debug: boolean;
44
+ timeoutMs: number;
45
+ maxDepth?: number;
46
+ limit?: number;
47
+ includePaths?: string[];
48
+ excludePaths?: string[];
49
+ sitemap?: CrawlSitemapMode;
50
+ llmsTxt: boolean;
51
+ }
52
+ /** Reads a URL list: one per line, blank lines and `#` comments ignored; `-` is stdin. */
53
+ export declare const parseUrlList: (text: string) => string[];
54
+ /**
55
+ * Turns `crawl`/`batch` arguments and flags into a validated job description. Pure except
56
+ * for reading `--file`, so the surface is unit-testable without a server.
57
+ */
58
+ export declare const resolveJobOptions: (kind: JobKind, sources: string[], flags: JobFlags, env?: Record<string, string | undefined>, readFile?: (file: string) => string) => ResolvedJob;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../../src/cli/commands/jobs/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEpF,OAAO,EAAE,WAAW,EAAE,UAAU,EAAsB,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAErE,gGAAgG;AAChG,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,CAAC;AAClC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AACjC,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAC;AACnC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,CAAC;AAoDxC,MAAM,SAAS,GAAG,CAAC,KAAa,EAAW,EAAE;IAC5C,IAAI,CAAC;QACJ,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,QAAQ,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,CAAmB,IAAY,EAAE,KAAa,EAAE,OAAqB,EAAK,EAAE;IACzF,IAAI,CAAE,OAA6B,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,UAAU,CAAC,aAAa,IAAI,KAAK,KAAK,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzF,CAAC;IACD,OAAO,KAAU,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,KAAyB,EAAE,GAAW,EAAE,GAAW,EAAsB,EAAE;IAC3G,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;QAC/D,MAAM,IAAI,UAAU,CAAC,aAAa,IAAI,KAAK,KAAK,+BAA+B,GAAG,OAAO,GAAG,GAAG,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,MAA4B,EAAwB,EAAE;IACrF,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC;YACJ,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,IAAI,UAAU,CAAC,aAAa,IAAI,aAAa,KAAK,mCAAmC,CAAC,CAAC;QAC9F,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAY,EAAY,EAAE,CACtD,IAAI;KACF,KAAK,CAAC,QAAQ,CAAC;KACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;KAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,QAAkC,EAAY,EAAE;IAClF,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,IAAI,UAAU,CAAC,uBAAuB,IAAI,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjH,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAEpG;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAChC,IAAa,EACb,OAAiB,EACjB,KAAe,EACf,MAA0C,OAAO,CAAC,GAAG,EACrD,WAAqC,eAAe,EACtC,EAAE;IAChB,MAAM,QAAQ,GAAG,IAAI,KAAK,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzF,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAErD,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,UAAU,CAAC,kCAAkC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,UAAU,CAAC,oEAAoE,CAAC,CAAC;IAC5F,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,IAAI,UAAU,CACnB,uBAAuB,cAAc,2BAA2B,IAAI,CAAC,MAAM,mBAAmB,CAC9F,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,UAAU,CAAC,wBAAwB,OAAO,GAAG,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,6DAA6D,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC/D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,kBAAkB,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,cAAc,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,uBAAuB,CAAC;IAEvG,OAAO;QACN,IAAI;QACJ,IAAI;QACJ,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,SAAS;QACnF,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM;QACrF,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,KAAK;QACvC,gBAAgB,EAAE,KAAK,CAAC,aAAa;QACrC,MAAM;QACN,WAAW;QACX,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,KAAK;QACzB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,KAAK;QAC3B,SAAS,EAAE,cAAc,GAAG,IAAI;QAChC,QAAQ,EAAE,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,EAAE,eAAe,CAAC;QACpE,KAAK,EAAE,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;QAC1D,YAAY,EAAE,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC;QAChD,YAAY,EAAE,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC;QAChD,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC,SAAS;QACzF,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,KAAK;KAC/B,CAAC;AACH,CAAC,CAAC"}
@@ -0,0 +1,44 @@
1
+ import type { JobStatus } from "../../../platform";
2
+ import type { ResolvedJob } from "./options";
3
+ export interface JobIo {
4
+ stdout: (text: string) => void;
5
+ stderr: (text: string) => void;
6
+ /** Live progress (a rewritten status line) only when stderr is an interactive terminal. */
7
+ isTTY: boolean;
8
+ sleep: (ms: number) => Promise<void>;
9
+ fetch: typeof fetch;
10
+ }
11
+ /** What `--json` prints: the job, every written file, and every failed page. */
12
+ export interface JobEnvelope {
13
+ jobId: string;
14
+ type: ResolvedJob["kind"];
15
+ status: JobStatus["status"];
16
+ outputDir: string;
17
+ credits: number;
18
+ pages: {
19
+ url: string;
20
+ file: string;
21
+ title?: string;
22
+ engine: string;
23
+ credits: number;
24
+ warning?: string;
25
+ }[];
26
+ failures: {
27
+ url: string;
28
+ code: string;
29
+ message: string;
30
+ }[];
31
+ llmsTxt?: string;
32
+ llmsFullTxt?: string;
33
+ error?: string;
34
+ }
35
+ /** The job finished but not cleanly (job failed, or no page converted) — exit code 1. */
36
+ export declare class JobIncompleteError extends Error {
37
+ }
38
+ /**
39
+ * `webforai crawl` / `webforai batch`: submit the job, wait for it (progress on stderr), page
40
+ * through the results, write one `.md` per page into the output directory (plus llms.txt
41
+ * files on request), then report. stdout carries only the written file paths — or the
42
+ * `--json` envelope — so the command composes like the single-page one.
43
+ */
44
+ export declare const runJobCommand: (job: ResolvedJob, io?: JobIo) => Promise<JobEnvelope>;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run.js","sourceRoot":"","sources":["../../../../../src/cli/commands/jobs/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAE7F,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG9D,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAW/B,MAAM,SAAS,GAAG,GAAU,EAAE,CAAC,CAAC;IAC/B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC5C,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC5C,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IACpC,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAChE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;CACxC,CAAC,CAAC;AAgBH,yFAAyF;AACzF,MAAM,OAAO,kBAAmB,SAAQ,KAAK;CAAG;AAEhD,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,MAAiB,EAAU,EAAE;IAChE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,OAAO,GAAG,IAAI,IAAI,MAAM,CAAC,KAAK,MAAM,MAAM,CAAC,MAAM,MAAM,MAAM,CAAC,SAAS,GAAG,KAAK,SAAS,MAAM,MAAM,MAAM,CAAC,OAAO,UAAU,CAAC;AAC9H,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,kBAAkB,GAAG,KAAK,EAAK,EAAS,EAAE,GAA2B,EAAE,IAAsB,EAAc,EAAE;IAClH,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;QACnC,IAAI,CAAC;YACJ,OAAO,MAAM,IAAI,EAAE,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,CAAC,KAAK,YAAY,gBAAgB,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,OAAO,GAAG,sBAAsB,EAAE,CAAC;gBAC/G,MAAM,KAAK,CAAC;YACb,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,IAAI,OAAO,EAAE,GAAG,CAAC,CAAC;YAChE,GAAG,CAAC,6BAA6B,OAAO,GAAG,CAAC,CAAC;YAC7C,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;QAChC,CAAC;IACF,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,MAAsB,EAAE,GAAgB,EAAE,EAAE;IAC3D,MAAM,MAAM,GAAG;QACd,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,OAAO,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,EAAE;KAChF,CAAC;IACF,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC,KAAK,CAAC;QACnB,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAW;QAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,GAAG,MAAM;KACT,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EAAE,EAAS,EAAE,IAAmB,EAAuB,EAAE;IACjF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,gBAAgB,CACzB,uBAAuB,EACvB,qBAAqB,IAAI,CAAC,GAAG,kCAAkC,QAAQ,CAAC,MAAM,GAAG,EACjF,QAAQ,CAAC,MAAM,CACf,CAAC;IACH,CAAC;IACD,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAgB,CAAC;AAC/C,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,cAAc,GAAG,CAAC,EAAS,EAAE,GAAgB,EAAE,EAAE;IACtD,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,MAAM,SAAS,GAAG,GAAG,EAAE;QACtB,IAAI,QAAQ,EAAE,CAAC;YACd,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzB,QAAQ,GAAG,KAAK,CAAC;QAClB,CAAC;IACF,CAAC,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAE;QAC5B,SAAS,EAAE,CAAC;QACZ,EAAE,CAAC,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,OAAkB,EAAE,EAAE;QACrC,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC;YACd,EAAE,CAAC,MAAM,CAAC,yBAAyB,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;YACtE,QAAQ,GAAG,IAAI,CAAC;QACjB,CAAC;aAAM,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACtB,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC,CAAC;IACF,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACnC,CAAC,CAAC;AAIF,MAAM,gBAAgB,GAAG,KAAK,EAC7B,MAAsB,EACtB,GAAgB,EAChB,KAAa,EACb,EAAS,EACT,QAAkB,EACG,EAAE;IACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC;IAC5C,IAAI,CAAC;QACJ,OAAO,MAAM,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CACtD,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE;YACxB,cAAc,EAAE,gBAAgB;YAChC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7C,QAAQ,EAAE,QAAQ,CAAC,MAAM;SACzB,CAAC,CACF,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,gBAAgB,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACxE,MAAM,IAAI,gBAAgB,CACzB,cAAc,EACd,GAAG,KAAK,CAAC,OAAO,mGAAmG,KAAK,WAAW,EACnI,CAAC,CACD,CAAC;QACH,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,SAAS,EAAE,CAAC;IACtB,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,KAAK,EAAE,MAAsB,EAAE,KAAa,EAAE,EAAS,EAAE,QAAkB,EAAE,EAAE;IACrG,MAAM,SAAS,GAAkB,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,IAAI,MAA0B,CAAC;IAC/B,GAAG,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACvG,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC5B,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACF,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,CAAC,QAAQ,MAAM,EAAE;IACjB,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,KAAK,EAAE,GAAgB,EAAE,SAAwB,EAAiC,EAAE;IACtG,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,KAAK,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,IAAI,eAAe,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,WAAW,EAAE,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC,EAAE,CAAC;QAC/G,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAgB,CAAC;QAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAC1D,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QAChG,OAAO,CAAC,IAAI,CAAC;YACZ,GAAG;YACH,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC;YAC5C,KAAK,EAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YAChF,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,KAAK,EAAE,GAAgB,EAAE,SAAwB,EAAE,QAAqB,EAAiB,EAAE;IACjH,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAW,EAAE,KAAK,EAAE,CAAC;IACvD,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACxD,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IACjE,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;AACnE,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,GAAgB,EAAE,QAAqB,EAAE,EAAS,EAAE,GAA2B,EAAQ,EAAE;IACxG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;SAAM,CAAC;QACP,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YACnC,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QAC7B,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxC,GAAG,CACF,GAAG,GAAG,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,KAAK,CAAC,MAAM,uBAAuB,GAAG,CAAC,SAAS,EAAE;QAC/G,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,QAAQ,CAAC,OAAO,UAAU,CAC1E,CAAC;IACF,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtB,GAAG,CAAC,yCAAyC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,YAAY,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,MAAM,GAAG,mBAAmB,EAAE,CAAC;QAClC,GAAG,CAAC,WAAW,MAAM,GAAG,mBAAmB,oBAAoB,CAAC,CAAC;IAClE,CAAC;AACF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAAE,GAAgB,EAAE,KAAY,SAAS,EAAE,EAAwB,EAAE;IACtG,MAAM,MAAM,GAAG,oBAAoB,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;IACvG,MAAM,QAAQ,GAAG,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAEzC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IACxF,QAAQ,CAAC,GAAG,CACX,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,eAAe,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,OAAO,GAAG,CACxG,CAAC;IAEF,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;IACvE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;IAElF,MAAM,QAAQ,GAAgB;QAC7B,KAAK;QACL,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK,EAAE,MAAM,UAAU,CAAC,GAAG,EAAE,SAAS,CAAC;QACvC,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/F,KAAK,EAAE,KAAK,CAAC,KAAK;KAClB,CAAC;IACF,IAAI,GAAG,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,cAAc,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAChD,CAAC;IACD,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAExC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,kBAAkB,CAAC,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,UAAU,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzG,CAAC;IACD,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,kBAAkB,CAAC,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,qBAAqB,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"content.js","sourceRoot":"","sources":["../../../../../src/cli/commands/skill/content.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC;;;;;gBAKjD,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgF9C,CAAC"}
1
+ {"version":3,"file":"content.js","sourceRoot":"","sources":["../../../../../src/cli/commands/skill/content.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC;;;;;gBAKjD,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8J9C,CAAC"}
@@ -1,4 +1,4 @@
1
- import type { CliFlags } from "./options";
1
+ import { type CliFlags } from "./options";
2
2
  /**
3
3
  * The guided flow behind `-i/--interactive` — the pre-v3 default. Prompts only for what the
4
4
  * flags did not already provide, then runs the exact same pipeline as the non-interactive