webforai 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -5
- package/dist/bin.js +1277 -479
- package/dist/cjs/detect-client-shell.js +78 -0
- package/dist/cjs/extract-meta-refresh.js +71 -0
- package/dist/cjs/extractors/lib/sanitize.js +4 -0
- package/dist/cjs/html-to-mdast.js +2 -1
- package/dist/cjs/index.js +12 -0
- package/dist/cjs/link-replacer.js +19 -13
- package/dist/cjs/loaders/cf-puppeteer.js +14 -7
- package/dist/cjs/loaders/fetch.js +14 -2
- package/dist/cjs/loaders/playwright.js +32 -30
- package/dist/cjs/loaders/puppeteer.js +14 -7
- package/dist/cjs/md-splitter.js +24 -11
- package/dist/cjs/mdast-handlers/custom-code-handler.js +18 -7
- package/dist/cjs/mdast-handlers/custom-div-handler.js +12 -33
- package/dist/cjs/mdast-handlers/definition-list-handler.js +17 -6
- package/dist/cjs/mdast-to-markdown.js +2 -5
- package/dist/cjs/normalize/index.js +11 -2
- package/dist/cjs/platform/client.js +73 -16
- package/dist/cjs/platform/index.js +4 -0
- package/dist/cjs/platform/types.js +7 -1
- package/dist/detect-client-shell.js +53 -0
- package/dist/extract-meta-refresh.js +47 -0
- package/dist/extractors/lib/sanitize.js +4 -0
- package/dist/html-to-mdast.js +2 -1
- package/dist/index.js +14 -0
- package/dist/link-replacer.js +19 -13
- package/dist/loaders/cf-puppeteer.js +15 -8
- package/dist/loaders/fetch.js +14 -2
- package/dist/loaders/playwright.js +32 -30
- package/dist/loaders/puppeteer.js +15 -8
- package/dist/md-splitter.js +24 -11
- package/dist/mdast-handlers/custom-code-handler.js +17 -7
- package/dist/mdast-handlers/custom-div-handler.js +12 -33
- package/dist/mdast-handlers/definition-list-handler.js +17 -6
- package/dist/mdast-to-markdown.js +2 -5
- package/dist/normalize/index.js +12 -2
- package/dist/platform/client.js +73 -16
- package/dist/platform/index.js +4 -0
- package/dist/platform/types.js +5 -1
- package/dist/types/cli/bin.js.map +1 -1
- package/dist/types/cli/commands/jobs/files.d.ts +28 -0
- package/dist/types/cli/commands/jobs/files.js.map +1 -0
- package/dist/types/cli/commands/jobs/llms-txt.d.ts +33 -0
- package/dist/types/cli/commands/jobs/llms-txt.js.map +1 -0
- package/dist/types/cli/commands/jobs/options.d.ts +58 -0
- package/dist/types/cli/commands/jobs/options.js.map +1 -0
- package/dist/types/cli/commands/jobs/run.d.ts +44 -0
- package/dist/types/cli/commands/jobs/run.js.map +1 -0
- package/dist/types/cli/commands/skill/content.js.map +1 -1
- package/dist/types/cli/commands/webforai/interactive.d.ts +1 -1
- package/dist/types/cli/commands/webforai/interactive.js.map +1 -1
- package/dist/types/cli/commands/webforai/options.d.ts +9 -2
- package/dist/types/cli/commands/webforai/options.js.map +1 -1
- package/dist/types/cli/commands/webforai/run.js.map +1 -1
- package/dist/types/cli/utils.d.ts +8 -1
- package/dist/types/cli/utils.js.map +1 -1
- package/dist/types/detect-client-shell.d.ts +24 -0
- package/dist/types/detect-client-shell.js.map +1 -0
- package/dist/types/extract-meta-refresh.d.ts +29 -0
- package/dist/types/extract-meta-refresh.js.map +1 -0
- package/dist/types/extractors/lib/sanitize.js.map +1 -1
- package/dist/types/html-to-mdast.js.map +1 -1
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.js.map +1 -1
- package/dist/types/link-replacer.d.ts +3 -1
- package/dist/types/link-replacer.js.map +1 -1
- package/dist/types/loaders/cf-puppeteer.js.map +1 -1
- package/dist/types/loaders/fetch.d.ts +4 -0
- package/dist/types/loaders/fetch.js.map +1 -1
- package/dist/types/loaders/playwright.js.map +1 -1
- package/dist/types/loaders/puppeteer.js.map +1 -1
- package/dist/types/md-splitter.d.ts +1 -1
- package/dist/types/md-splitter.js.map +1 -1
- package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
- package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
- package/dist/types/mdast-handlers/definition-list-handler.d.ts +0 -11
- package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -1
- package/dist/types/mdast-to-markdown.js.map +1 -1
- package/dist/types/normalize/index.js.map +1 -1
- package/dist/types/platform/client.d.ts +2 -1
- package/dist/types/platform/client.js.map +1 -1
- package/dist/types/platform/error.d.ts +5 -3
- package/dist/types/platform/error.js.map +1 -1
- package/dist/types/platform/fetch.d.ts +2 -0
- package/dist/types/platform/fetch.js.map +1 -1
- package/dist/types/platform/index.d.ts +1 -1
- package/dist/types/platform/index.js.map +1 -1
- package/dist/types/platform/types.d.ts +35 -6
- package/dist/types/platform/types.js.map +1 -1
- package/package.json +16 -5
|
@@ -2,19 +2,29 @@
|
|
|
2
2
|
import { toText } from "hast-util-to-text";
|
|
3
3
|
import { trimTrailingLines } from "trim-trailing-lines";
|
|
4
4
|
import { detectLanguage } from "../utils/detect-code-lang.js";
|
|
5
|
-
|
|
5
|
+
import { classList, isElement } from "../utils/hast-fast.js";
|
|
6
|
+
var LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
|
|
7
|
+
var LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
|
|
8
|
+
var attributeLanguage = (element) => {
|
|
9
|
+
for (const name of LANGUAGE_ATTRIBUTES) {
|
|
10
|
+
const value = element.properties?.[name];
|
|
11
|
+
if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
|
|
12
|
+
return value.trim();
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return void 0;
|
|
16
|
+
};
|
|
17
|
+
var codeLanguage = (...elements) => elements.flatMap(classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
|
|
6
18
|
var customCodeHandler = (state, node) => {
|
|
7
|
-
const
|
|
8
|
-
const codeValue = trimTrailingLines(toText(node))
|
|
9
|
-
const classLang =
|
|
10
|
-
const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
|
|
11
|
-
return match?.[1];
|
|
12
|
-
}).find((className) => className);
|
|
19
|
+
const code = node.children.find((child) => isElement(child) && child.tagName === "code");
|
|
20
|
+
const codeValue = trimTrailingLines(toText(node));
|
|
21
|
+
const classLang = codeLanguage(...isElement(code) ? [code, node] : [node]);
|
|
13
22
|
const lang = classLang || detectLanguage(codeValue) || null;
|
|
14
23
|
const result = { type: "code", lang, meta: null, value: codeValue };
|
|
15
24
|
state.patch(node, result);
|
|
16
25
|
return result;
|
|
17
26
|
};
|
|
18
27
|
export {
|
|
28
|
+
codeLanguage,
|
|
19
29
|
customCodeHandler
|
|
20
30
|
};
|
|
@@ -5,44 +5,23 @@ import { toString as hastToString } from "hast-util-to-string";
|
|
|
5
5
|
import { toText } from "hast-util-to-text";
|
|
6
6
|
import { trimTrailingLines } from "trim-trailing-lines";
|
|
7
7
|
import { detectLanguage } from "../utils/detect-code-lang.js";
|
|
8
|
+
import { classList, collectElements, isElement } from "../utils/hast-fast.js";
|
|
9
|
+
import { codeLanguage } from "./custom-code-handler.js";
|
|
8
10
|
var CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
|
|
9
11
|
var CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
|
|
10
|
-
var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
|
|
11
|
-
var findRecursive = (array, condition, maxDepth = 3) => {
|
|
12
|
-
if (maxDepth <= 0) {
|
|
13
|
-
return null;
|
|
14
|
-
}
|
|
15
|
-
for (const value of array) {
|
|
16
|
-
const result = condition(value);
|
|
17
|
-
if (Array.isArray(result)) {
|
|
18
|
-
return findRecursive(result, condition, maxDepth - 1);
|
|
19
|
-
}
|
|
20
|
-
if (result) {
|
|
21
|
-
return value;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
return null;
|
|
25
|
-
};
|
|
26
12
|
var customDivHandler = (state, node) => {
|
|
27
|
-
const classNames =
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
return child.children.filter((child2) => child2.type === "element");
|
|
36
|
-
});
|
|
37
|
-
if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
|
|
38
|
-
const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
|
|
39
|
-
const codeValue = trimTrailingLines(toText(codeBlock)).trim();
|
|
13
|
+
const classNames = classList(node);
|
|
14
|
+
if (!classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
|
|
15
|
+
return defaultHandlers.div(state, node);
|
|
16
|
+
}
|
|
17
|
+
const codeBlocks = collectElements(node, (element) => element.tagName === "pre");
|
|
18
|
+
const codeBlock = codeBlocks.length === 1 ? codeBlocks[0] : void 0;
|
|
19
|
+
if (codeBlock) {
|
|
20
|
+
const codeValue = trimTrailingLines(toText(codeBlock));
|
|
40
21
|
const filenameElement = select(CODE_FILENAME_SELECTORS, node);
|
|
41
22
|
const fileLang = filenameElement ? hastToString(filenameElement).match(/\.(\w+)$/)?.[1] : null;
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
return match?.[1];
|
|
45
|
-
}).find((className) => className);
|
|
23
|
+
const code = codeBlock.children.find((child) => isElement(child) && child.tagName === "code");
|
|
24
|
+
const classLang = codeLanguage(...isElement(code) ? [code, codeBlock, node] : [codeBlock, node]);
|
|
46
25
|
const lang = fileLang || classLang || detectLanguage(codeValue) || null;
|
|
47
26
|
const result = { type: "code", lang, meta: null, value: codeValue };
|
|
48
27
|
state.patch(node, result);
|
|
@@ -28,6 +28,20 @@ var hoistBoundaryWhitespace = (nodes) => {
|
|
|
28
28
|
}
|
|
29
29
|
return result;
|
|
30
30
|
};
|
|
31
|
+
var termsAndDefinitions = (parent) => {
|
|
32
|
+
const found = [];
|
|
33
|
+
for (const child of parent.children) {
|
|
34
|
+
if (child.type !== "element") {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
if (child.tagName === "dt" || child.tagName === "dd") {
|
|
38
|
+
found.push(child);
|
|
39
|
+
} else if (child.tagName === "div") {
|
|
40
|
+
found.push(...termsAndDefinitions(child));
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return found;
|
|
44
|
+
};
|
|
31
45
|
var definitionListHandler = (state, node) => {
|
|
32
46
|
const items = [];
|
|
33
47
|
let current;
|
|
@@ -42,11 +56,7 @@ var definitionListHandler = (state, node) => {
|
|
|
42
56
|
items.push({ type: "listItem", spread: false, children: [heading, ...current.definitions] });
|
|
43
57
|
current = void 0;
|
|
44
58
|
};
|
|
45
|
-
for (const
|
|
46
|
-
if (child.type !== "element") {
|
|
47
|
-
continue;
|
|
48
|
-
}
|
|
49
|
-
const element = child;
|
|
59
|
+
for (const element of termsAndDefinitions(node)) {
|
|
50
60
|
if (element.tagName === "dt") {
|
|
51
61
|
if (current && current.definitions.length > 0) {
|
|
52
62
|
flush();
|
|
@@ -65,7 +75,8 @@ var definitionListHandler = (state, node) => {
|
|
|
65
75
|
}
|
|
66
76
|
flush();
|
|
67
77
|
if (items.length === 0) {
|
|
68
|
-
|
|
78
|
+
const fallback = state.toFlow(state.all(node));
|
|
79
|
+
return fallback.length > 0 ? fallback : void 0;
|
|
69
80
|
}
|
|
70
81
|
const result = { type: "list", ordered: false, spread: false, children: items };
|
|
71
82
|
state.patch(node, result);
|
|
@@ -10,11 +10,8 @@ var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
|
|
|
10
10
|
};
|
|
11
11
|
var mdastToMarkdown = (mdast, options) => {
|
|
12
12
|
const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
markdown = linkReplacer(markdown, baseUrl);
|
|
16
|
-
}
|
|
17
|
-
return markdown;
|
|
13
|
+
const tree = warpRoot(mdast);
|
|
14
|
+
return toMarkdown(baseUrl ? linkReplacer(tree, baseUrl) : tree, toMarkdownOptions);
|
|
18
15
|
};
|
|
19
16
|
export {
|
|
20
17
|
DEFAULT_MDAST_TO_MARKDOWN_OPTIONS,
|
package/dist/normalize/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// src/normalize/index.ts
|
|
2
2
|
import {
|
|
3
3
|
classList,
|
|
4
|
+
findElement,
|
|
4
5
|
isElement,
|
|
5
6
|
numericProperty,
|
|
6
7
|
property,
|
|
@@ -80,14 +81,16 @@ var katexDuplicates = (node) => {
|
|
|
80
81
|
if (!(classes.includes("katex") || classes.includes("katex-display"))) {
|
|
81
82
|
return [];
|
|
82
83
|
}
|
|
83
|
-
const hasMathml = node.children.some(
|
|
84
|
+
const hasMathml = node.children.some(
|
|
85
|
+
(child) => isElement(child) && classList(child).includes("katex-mathml") && hasMathSource(child)
|
|
86
|
+
);
|
|
84
87
|
if (!hasMathml) {
|
|
85
88
|
return [];
|
|
86
89
|
}
|
|
87
90
|
return node.children.filter((child) => isElement(child) && classList(child).includes("katex-html"));
|
|
88
91
|
};
|
|
89
92
|
var mediawikiDuplicates = (node) => {
|
|
90
|
-
if (!classList(node).includes("mwe-math-element")) {
|
|
93
|
+
if (!classList(node).includes("mwe-math-element") || !hasMathSource(node)) {
|
|
91
94
|
return [];
|
|
92
95
|
}
|
|
93
96
|
return node.children.filter(
|
|
@@ -98,8 +101,15 @@ var mathjaxDuplicates = (node) => {
|
|
|
98
101
|
if (node.tagName !== "mjx-container") {
|
|
99
102
|
return [];
|
|
100
103
|
}
|
|
104
|
+
const hasMathml = node.children.some(
|
|
105
|
+
(child) => isElement(child) && child.tagName === "mjx-assistive-mml" && hasMathSource(child)
|
|
106
|
+
);
|
|
107
|
+
if (!hasMathml) {
|
|
108
|
+
return [];
|
|
109
|
+
}
|
|
101
110
|
return node.children.filter((child) => isElement(child) && child.tagName !== "mjx-assistive-mml");
|
|
102
111
|
};
|
|
112
|
+
var hasMathSource = (node) => Boolean(findElement(node, (element) => element.tagName === "math" && element.children.length > 0));
|
|
103
113
|
var dedupeRenderedMath = (tree) => {
|
|
104
114
|
const doomed = /* @__PURE__ */ new Set();
|
|
105
115
|
walk(tree, (node) => {
|
package/dist/platform/client.js
CHANGED
|
@@ -29,14 +29,28 @@ var parseRetryAfter = (body, response) => {
|
|
|
29
29
|
const header = Number(response.headers.get("retry-after"));
|
|
30
30
|
return Number.isFinite(header) && header > 0 ? header : void 0;
|
|
31
31
|
};
|
|
32
|
-
var
|
|
32
|
+
var dashboardHint = (status, dashboardUrl) => {
|
|
33
|
+
if (status === 401) {
|
|
34
|
+
return `create or check API keys at ${dashboardUrl}`;
|
|
35
|
+
}
|
|
36
|
+
if (status === 402) {
|
|
37
|
+
return `manage billing, credits and the spend cap at ${dashboardUrl}`;
|
|
38
|
+
}
|
|
39
|
+
return void 0;
|
|
40
|
+
};
|
|
41
|
+
var toApiError = (response, body, dashboardUrl) => {
|
|
33
42
|
const envelope = body;
|
|
34
43
|
const code = typeof envelope?.error?.code === "string" ? envelope.error.code : "invalid_response";
|
|
35
|
-
|
|
44
|
+
let message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
|
|
45
|
+
const hint = code === "invalid_response" ? void 0 : dashboardHint(response.status, dashboardUrl);
|
|
46
|
+
if (hint && !message.includes("/dashboard")) {
|
|
47
|
+
message = `${message} (${hint})`;
|
|
48
|
+
}
|
|
36
49
|
return new PlatformApiError(code, message, response.status, parseRetryAfter(body, response));
|
|
37
50
|
};
|
|
38
51
|
var createPlatformClient = (options = {}) => {
|
|
39
52
|
const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
|
|
53
|
+
const dashboardUrl = `${baseUrl}/dashboard`;
|
|
40
54
|
const fetchImpl = resolveFetch(options.fetch);
|
|
41
55
|
const request = async (path, init) => {
|
|
42
56
|
const headers = { accept: "application/json" };
|
|
@@ -47,7 +61,7 @@ var createPlatformClient = (options = {}) => {
|
|
|
47
61
|
if (!options.apiKey) {
|
|
48
62
|
throw new PlatformApiError(
|
|
49
63
|
"missing_api_key",
|
|
50
|
-
|
|
64
|
+
`This endpoint needs an API key: pass \`apiKey\` to createPlatformClient (create one at ${dashboardUrl}; 1,000 free credits/month).`,
|
|
51
65
|
0
|
|
52
66
|
);
|
|
53
67
|
}
|
|
@@ -56,18 +70,19 @@ var createPlatformClient = (options = {}) => {
|
|
|
56
70
|
const response = await fetchImpl(`${baseUrl}${path}`, {
|
|
57
71
|
method: init.method,
|
|
58
72
|
headers,
|
|
59
|
-
body: init.body === void 0 ? void 0 : JSON.stringify(init.body)
|
|
73
|
+
body: init.body === void 0 ? void 0 : JSON.stringify(init.body),
|
|
74
|
+
...init.signal ? { signal: init.signal } : {}
|
|
60
75
|
});
|
|
61
76
|
const body = await response.json().catch(() => void 0);
|
|
62
77
|
if (!response.ok) {
|
|
63
|
-
throw toApiError(response, body);
|
|
78
|
+
throw toApiError(response, body, dashboardUrl);
|
|
64
79
|
}
|
|
65
80
|
if (body === void 0) {
|
|
66
81
|
throw new PlatformApiError("invalid_response", "response body is not JSON", response.status);
|
|
67
82
|
}
|
|
68
83
|
return body;
|
|
69
84
|
};
|
|
70
|
-
const getJob = (jobId) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true });
|
|
85
|
+
const getJob = (jobId, signal) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true, signal });
|
|
71
86
|
const getJobResults = (jobId, opts) => {
|
|
72
87
|
const query = opts?.cursor ? `?cursor=${encodeURIComponent(opts.cursor)}` : "";
|
|
73
88
|
return request(`/v1/jobs/${encodeURIComponent(jobId)}/results${query}`, { method: "GET", auth: true });
|
|
@@ -83,6 +98,17 @@ var createPlatformClient = (options = {}) => {
|
|
|
83
98
|
}
|
|
84
99
|
return await response.json();
|
|
85
100
|
};
|
|
101
|
+
const getJobOrBackOff = async (jobId, signal, interval) => {
|
|
102
|
+
try {
|
|
103
|
+
return await getJob(jobId, signal);
|
|
104
|
+
} catch (error) {
|
|
105
|
+
if (!(error instanceof PlatformApiError && error.status === 429)) {
|
|
106
|
+
throw error;
|
|
107
|
+
}
|
|
108
|
+
await sleep(error.retryAfter !== void 0 ? error.retryAfter * 1e3 : interval, signal);
|
|
109
|
+
return void 0;
|
|
110
|
+
}
|
|
111
|
+
};
|
|
86
112
|
return {
|
|
87
113
|
scrape: (opts) => request("/v1/scrape", { method: "POST", body: opts, auth: true }),
|
|
88
114
|
scrapeAsync: (opts) => request("/v1/scrape", { method: "POST", body: { ...opts, async: true }, auth: true }),
|
|
@@ -93,17 +119,48 @@ var createPlatformClient = (options = {}) => {
|
|
|
93
119
|
waitForJob: async (jobId, opts = {}) => {
|
|
94
120
|
const interval = opts.pollIntervalMs ?? 2e3;
|
|
95
121
|
const timeout = opts.timeoutMs ?? 10 * 60 * 1e3;
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
122
|
+
if (!Number.isFinite(interval) || interval < 0 || interval > 2147483647) {
|
|
123
|
+
throw new RangeError("pollIntervalMs must be between 0 and 2147483647");
|
|
124
|
+
}
|
|
125
|
+
if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 2147483647) {
|
|
126
|
+
throw new RangeError("timeoutMs must be greater than 0 and at most 2147483647");
|
|
127
|
+
}
|
|
128
|
+
opts.signal?.throwIfAborted();
|
|
129
|
+
const controller = new AbortController();
|
|
130
|
+
const { signal } = controller;
|
|
131
|
+
const onAbort = () => controller.abort(opts.signal?.reason);
|
|
132
|
+
opts.signal?.addEventListener("abort", onAbort, { once: true });
|
|
133
|
+
const timer = setTimeout(
|
|
134
|
+
() => controller.abort(new PlatformApiError("poll_timeout", `job ${jobId} did not finish within ${timeout}ms`, 0)),
|
|
135
|
+
timeout
|
|
136
|
+
);
|
|
137
|
+
let rejectOnAbort = () => {
|
|
138
|
+
};
|
|
139
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
140
|
+
rejectOnAbort = () => reject(signal.reason);
|
|
141
|
+
signal.addEventListener("abort", rejectOnAbort, { once: true });
|
|
142
|
+
});
|
|
143
|
+
const poll = async () => {
|
|
144
|
+
for (; ; ) {
|
|
145
|
+
signal.throwIfAborted();
|
|
146
|
+
const status = await getJobOrBackOff(jobId, signal, interval);
|
|
147
|
+
signal.throwIfAborted();
|
|
148
|
+
if (status === void 0) {
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
opts.onStatus?.(status);
|
|
152
|
+
if (TERMINAL_STATES.has(status.status)) {
|
|
153
|
+
return status;
|
|
154
|
+
}
|
|
155
|
+
await sleep(interval, signal);
|
|
105
156
|
}
|
|
106
|
-
|
|
157
|
+
};
|
|
158
|
+
try {
|
|
159
|
+
return await Promise.race([poll(), aborted]);
|
|
160
|
+
} finally {
|
|
161
|
+
clearTimeout(timer);
|
|
162
|
+
opts.signal?.removeEventListener("abort", onAbort);
|
|
163
|
+
signal.removeEventListener("abort", rejectOnAbort);
|
|
107
164
|
}
|
|
108
165
|
},
|
|
109
166
|
jobResults: async function* (jobId) {
|
package/dist/platform/index.js
CHANGED
|
@@ -5,15 +5,19 @@ import {
|
|
|
5
5
|
} from "./client.js";
|
|
6
6
|
import { PlatformApiError } from "./error.js";
|
|
7
7
|
import {
|
|
8
|
+
CRAWL_SITEMAP_MODES,
|
|
8
9
|
ENGINES,
|
|
9
10
|
REGIONS,
|
|
11
|
+
REQUESTED_ENGINES,
|
|
10
12
|
isStoredPageStub
|
|
11
13
|
} from "./types.js";
|
|
12
14
|
export {
|
|
15
|
+
CRAWL_SITEMAP_MODES,
|
|
13
16
|
DEFAULT_BASE_URL,
|
|
14
17
|
ENGINES,
|
|
15
18
|
PlatformApiError,
|
|
16
19
|
REGIONS,
|
|
20
|
+
REQUESTED_ENGINES,
|
|
17
21
|
createPlatformClient,
|
|
18
22
|
isStoredPageStub
|
|
19
23
|
};
|
package/dist/platform/types.js
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
// src/platform/types.ts
|
|
2
2
|
var ENGINES = ["fetch", "browser", "proxy-fetch", "proxy-browser"];
|
|
3
|
-
var
|
|
3
|
+
var REQUESTED_ENGINES = [...ENGINES, "auto"];
|
|
4
|
+
var REGIONS = ["auto", "jp"];
|
|
5
|
+
var CRAWL_SITEMAP_MODES = ["skip", "include", "only"];
|
|
4
6
|
var isStoredPageStub = (item) => item.status === "ok" && "resultUrl" in item;
|
|
5
7
|
export {
|
|
8
|
+
CRAWL_SITEMAP_MODES,
|
|
6
9
|
ENGINES,
|
|
7
10
|
REGIONS,
|
|
11
|
+
REQUESTED_ENGINES,
|
|
8
12
|
isStoredPageStub
|
|
9
13
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bin.js","sourceRoot":"","sources":["../../../src/cli/bin.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"bin.js","sourceRoot":"","sources":["../../../src/cli/bin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,cAAc,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAClE,OAAO,WAAW,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAiB,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAmB,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,OAAO,EAAiB,UAAU,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACxF,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAE/C,MAAM,aAAa,GAAG;;;;;;;;;;;;;;IAclB,WAAW;IACX,gBAAgB;;;;;;;;;;;IAWhB,gBAAgB;CACnB,CAAC;AAEF,+EAA+E;AAC/E,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,mBAAmB,CAAC,CAAC,CAAC;AAE/F,4FAA4F;AAC5F,IAAI,iBAAqC,CAAC;AAC1C,MAAM,mBAAmB,GAAG,CAAC,KAA+B,EAAE,EAAE;IAC/D,iBAAiB,GAAG,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,OAAO;KACL,IAAI,CAAC,UAAU,CAAC;IACjB,2FAA2F;KAC1F,uBAAuB,EAAE;IAC1B,8EAA8E;KAC7E,YAAY,EAAE;KACd,WAAW,CAAC,gEAAgE,CAAC;KAC7E,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,eAAe,EAAE,4BAA4B,CAAC;KAC3E,WAAW,CAAC,OAAO,EAAE,aAAa,CAAC;KACnC,wBAAwB,EAAE,CAAC;AAE7B,OAAO;KACL,QAAQ,CAAC,UAAU,EAAE,mCAAmC,CAAC;KACzD,MAAM,CAAC,qBAAqB,EAAE,4CAA4C,CAAC;KAC3E,MAAM,CAAC,uBAAuB,EAAE,yBAAyB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC;KAC3G,MAAM,CAAC,mBAAmB,EAAE,oBAAoB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC;KACpG,MAAM,CAAC,yBAAyB,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KAC1G,MAAM,CAAC,eAAe,EAAE,wDAAwD,CAAC;KACjF,MAAM,CAAC,QAAQ,EAAE,gFAAgF,CAAC;KAClG,MAAM,CACN,mBAAmB,EACnB,uHAAuH,CACvH;KACA,MAAM,CAAC,mBAAmB,EAAE,qFAAqF,CAAC;KAClH,MAAM,CAAC,cAAc,EAAE,oEAAoE,CAAC;KAC5F,MAAM,CAAC,kBAAkB,EAAE,4EAA4E,CAAC;KACxG,MAAM,CAAC,iBAAiB,EAAE,kCAAkC,WAAW,GAAG,CAAC;KAC3E,MAAM,CAAC,sBAAsB,EAAE,mCAAmC,gBAAgB,0BAA0B,CAAC;KAC7G,MAAM,CAAC,mBAAmB,EAAE,yCAAyC,CAAC;KACtE,MAAM,CAAC,aAAa,EAAE,wBAAwB,CAAC;KAC/C,MAAM,CAAC,KAAK,EAAE,MAA0B,EAAE,KAA2C,EAAE,EAAE;IACzF,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACvB,MAAM,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxC,OAAO;IACR,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,UAAU,CAAC,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;AACpD,CAAC,CAAC,CAAC;AAEJ,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,WAAqB,EAAE,EAAY,EAAE,CAAC,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,CAAC;AAE3F,gGAAgG;AAChG,MAAM,qBAAqB,GAAG,CAAC,OAAgB,EAAW,EAAE,CAC3D,OAAO;KACL,cAAc,CAAC,oBAAoB,EAAE,qEAAqE,CAAC;KAC3G,MAAM,CACN,mBAAmB,EACnB,kGAAkG,CAClG;KACA,MAAM,CAAC,mBAAmB,EAAE,iEAAiE,CAAC;KAC9F,MAAM,CAAC,yBAAyB,EAAE,mCAAmC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;KAC/F,MAAM,CAAC,eAAe,EAAE,2DAA2D,CAAC;KACpF,MAAM,CAAC,QAAQ,EAAE,yFAAyF,CAAC;KAC3G,MAAM,CAAC,qBAAqB,EAAE,oEAAoE,CAAC;KACnG,MAAM,CAAC,iBAAiB,EAAE,kCAAkC,WAAW,GAAG,CAAC;KAC3E,MAAM,CAAC,sBAAsB,EAAE,mCAAmC,gBAAgB,0BAA0B,CAAC;KAC7G,MAAM,CAAC,aAAa,EAAE,wBAAwB,CAAC,CAAC;AAEnD,MAAM,eAAe,GAAG;;;;;CAKvB,CAAC;AAEF,qBAAqB,CACpB,OAAO;KACL,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,0EAA0E,CAAC;KACvF,QAAQ,CAAC,OAAO,EAAE,+CAA+C,CAAC,CACpE;KACC,MAAM,CAAC,iBAAiB,EAAE,2CAA2C,CAAC;KACtE,MAAM,CAAC,aAAa,EAAE,mCAAmC,CAAC;KAC1D,MAAM,CAAC,mBAAmB,EAAE,uDAAuD,EAAE,OAAO,CAAC;KAC7F,MAAM,CAAC,mBAAmB,EAAE,iDAAiD,EAAE,OAAO,CAAC;KACvF,MAAM,CACN,kBAAkB,EAClB,+FAA+F,CAC/F;KACA,MAAM,CAAC,kBAAkB,EAAE,2CAA2C,CAAC;KACvE,MAAM,CAAC,qBAAqB,EAAE,wBAAwB,CAAC;KACvD,MAAM,CAAC,YAAY,EAAE,gFAAgF,CAAC;KACtG,WAAW,CACX,OAAO,EACP,GAAG,eAAe;;;;CAInB,CACC;KACA,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,KAAe,EAAE,EAAE;IAC9C,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,aAAa,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC,CAAC,CAAC;AAEJ,qBAAqB,CACpB,OAAO;KACL,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,sEAAsE,CAAC;KACnF,QAAQ,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAC9D;KACC,MAAM,CAAC,mBAAmB,EAAE,yEAAyE,CAAC;KACtG,MAAM,CAAC,kBAAkB,EAAE,2DAA2D,CAAC;KACvF,WAAW,CACX,OAAO,EACP,GAAG,eAAe;;;;CAInB,CACC;KACA,MAAM,CAAC,KAAK,EAAE,IAAc,EAAE,KAAe,EAAE,EAAE;IACjD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,aAAa,CAAC,iBAAiB,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAC9D,CAAC,CAAC,CAAC;AAEJ,OAAO;KACL,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,iFAAiF,CAAC;KAC9F,MAAM,CAAC,WAAW,EAAE,wFAAwF,CAAC;KAC7G,MAAM,CAAC,cAAc,EAAE,oEAAoE,CAAC;KAC5F,MAAM,CAAC,sBAAsB,EAAE,2DAA2D,CAAC;KAC3F,MAAM,CAAC,WAAW,EAAE,kDAAkD,CAAC;KACvE,MAAM,CAAC,QAAQ,EAAE,yDAAyD,CAAC;KAC3E,MAAM,CAAC,OAAO,EAAE,2DAA2D,CAAC;KAC5E,MAAM,CAAC,cAAc,EAAE,mFAAmF,CAAC;KAC3G,WAAW,CAAC,OAAO,EAAE,qFAAqF,CAAC;KAC3G,MAAM,CAAC,KAAK,EAAE,KAAiB,EAAE,EAAE;IACnC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC,CAAC,CAAC;AAEJ,MAAM,IAAI,GAAG,CAAC,KAAc,EAAS,EAAE;IACtC,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;QACrC,uEAAuE;QACvE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,YAAY,gBAAgB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB,KAAK,CAAC,UAAU,IAAI,CAAC;QAC1F,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChE,oFAAoF;QACpF,IAAI,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAC/E,OAAO,CAAC,KAAK,CAAC,wCAAwC,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;QACzC,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC;AAEF,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Maps a page URL to a relative `.md` path that mirrors the site's structure:
|
|
3
|
+
* `/` → `index.md`, `/docs/` → `docs/index.md`, `/docs/intro.html` → `docs/intro.md`.
|
|
4
|
+
* A query string adds a short stable hash (`search_q-1a2b3c4d.md`) so `?page=2` does not
|
|
5
|
+
* overwrite `?page=1`. `includeHost` prefixes the hostname (batch jobs span origins).
|
|
6
|
+
*
|
|
7
|
+
* The result never escapes the output directory: segments are sanitized one by one, so
|
|
8
|
+
* `..`, encoded slashes and absolute paths cannot survive.
|
|
9
|
+
*/
|
|
10
|
+
export declare const pageRelativePath: (pageUrl: string, options?: {
|
|
11
|
+
includeHost?: boolean;
|
|
12
|
+
}) => string;
|
|
13
|
+
export interface PlannedFile {
|
|
14
|
+
url: string;
|
|
15
|
+
/** Path relative to the output directory, POSIX separators. */
|
|
16
|
+
relativePath: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Assigns every URL a distinct relative path. URLs are sorted first so the same set of pages
|
|
20
|
+
* always gets the same names, whatever order the job returned them in; collisions (e.g.
|
|
21
|
+
* `/a` and `/a.html`) take the CLI's usual `_1`, `_2` suffixes. Comparison is
|
|
22
|
+
* case-insensitive because macOS and Windows filesystems are.
|
|
23
|
+
*/
|
|
24
|
+
export declare const planOutputFiles: (urls: readonly string[], options?: {
|
|
25
|
+
includeHost?: boolean;
|
|
26
|
+
}) => PlannedFile[];
|
|
27
|
+
/** Resolves a planned path inside `outputDir`, refusing anything that would land outside it. */
|
|
28
|
+
export declare const resolveInside: (outputDir: string, relativePath: string) => string;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"files.js","sourceRoot":"","sources":["../../../../../src/cli/commands/jobs/files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAEvD,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,gGAAgG;AAChG,MAAM,cAAc,GAAG,iCAAiC,CAAC;AAEzD,MAAM,SAAS,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAE1G,MAAM,aAAa,GAAG,CAAC,OAAe,EAAU,EAAE;IACjD,IAAI,CAAC;QACJ,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,OAAO,CAAC;IAChB,CAAC;AACF,CAAC,CAAC;AAEF,iGAAiG;AACjG,MAAM,eAAe,GAAG,CAAC,OAAe,EAAU,EAAE;IACnD,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC;SACpC,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,oCAAoC,EAAE,GAAG,CAAC;SAClD,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;IACjC,IAAI,OAAO,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC;IAChB,CAAC;IACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,GAAG,CAAC,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5E,CAAC,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,OAAe,EAAE,UAAqC,EAAE,EAAU,EAAE;IACpG,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,MAAM,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,aAAa,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAEtF,IAAI,IAAI,GAAG,aAAa,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC;IACxF,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC;IACnD,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,IAAI,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACxG,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC;AACtD,CAAC,CAAC;AAQF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAuB,EAAE,UAAqC,EAAE,EAAiB,EAAE;IAClH,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACzB,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,wBAAwB,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAClG,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC;QACtC,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,gGAAgG;AAChG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,SAAiB,EAAE,YAAoB,EAAU,EAAE;IAChF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mDAAmD,YAAY,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* llms.txt / llms-full.txt rendering for crawl output, per https://llmstxt.org:
|
|
3
|
+
*
|
|
4
|
+
* ```md
|
|
5
|
+
* # Site title
|
|
6
|
+
*
|
|
7
|
+
* > One-line summary
|
|
8
|
+
*
|
|
9
|
+
* ## Section
|
|
10
|
+
*
|
|
11
|
+
* - [Page title](https://url): description
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* `llms-full.txt` is the companion convention: every page's Markdown, concatenated, so an
|
|
15
|
+
* agent can load a whole site in one read.
|
|
16
|
+
*/
|
|
17
|
+
export interface LlmsPage {
|
|
18
|
+
url: string;
|
|
19
|
+
markdown: string;
|
|
20
|
+
metadata: Record<string, unknown>;
|
|
21
|
+
}
|
|
22
|
+
export interface LlmsSite {
|
|
23
|
+
/** The crawl's seed URL; its page supplies the site title and summary when present. */
|
|
24
|
+
seedUrl: string;
|
|
25
|
+
pages: readonly LlmsPage[];
|
|
26
|
+
}
|
|
27
|
+
/** Pages in output order: grouped by section (top level first, then A→Z), URL order inside. */
|
|
28
|
+
export declare const orderPages: (pages: readonly LlmsPage[]) => {
|
|
29
|
+
section: string;
|
|
30
|
+
pages: LlmsPage[];
|
|
31
|
+
}[];
|
|
32
|
+
export declare const renderLlmsTxt: (site: LlmsSite) => string;
|
|
33
|
+
export declare const renderLlmsFullTxt: (site: LlmsSite) => string;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llms-txt.js","sourceRoot":"","sources":["../../../../../src/cli/commands/jobs/llms-txt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAcH,MAAM,GAAG,GAAG,CAAC,KAAc,EAAsB,EAAE,CAClD,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEnG,MAAM,YAAY,GAAG,CAAC,QAAgB,EAAsB,EAAE;IAC7D,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5D,OAAO,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1C,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,cAAc,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;AAErF,MAAM,SAAS,GAAG,CAAC,IAAc,EAAU,EAAE,CAC5C,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;AAErG,MAAM,YAAY,GAAG,CAAC,KAAa,EAAU,EAAE;IAC9C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;IACd,OAAO,GAAG,CAAC,IAAI,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,IAAc,EAAwB,EAAE;IACzD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;AACnE,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,IAAc,EAAE,IAA0B,EAAU,EAAE,CACxE,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC5B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC9E,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC;IACzB,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;AAE5B;;;GAGG;AACH,MAAM,SAAS,GAAG,CAAC,OAAe,EAAU,EAAE;IAC7C,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAClE,OAAO,EAAE,CAAC;IACX,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,CAAC;QACJ,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACR,uBAAuB;IACxB,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,+FAA+F;AAC/F,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAA0B,EAA4C,EAAE;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;SACvE,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;AAC9D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAc,EAAU,EAAE;IACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAChD,IAAI,OAAO,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;IAChC,CAAC;IACD,KAAK,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzD,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,IAAI,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5D,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC;AAC1C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAc,EAAU,EAAE;IAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7C,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClC,4EAA4E;YAC5E,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;YACxE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,WAAW,IAAI,CAAC,GAAG,OAAO,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;IACF,CAAC;IACD,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;AACzC,CAAC,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { CrawlSitemapMode, Region, RequestedEngine } from "../../../platform";
|
|
2
|
+
import { type ExtractorName } from "../../constants";
|
|
3
|
+
/** Server-side caps (docs/specs/platform/03_api.md); checked locally to fail before billing. */
|
|
4
|
+
export declare const MAX_BATCH_URLS = 100;
|
|
5
|
+
export declare const MAX_CRAWL_DEPTH = 5;
|
|
6
|
+
export declare const MAX_CRAWL_LIMIT = 500;
|
|
7
|
+
export type JobKind = "crawl" | "batch";
|
|
8
|
+
export interface JobFlags {
|
|
9
|
+
output?: string;
|
|
10
|
+
engine?: string;
|
|
11
|
+
region?: string;
|
|
12
|
+
extractor?: string;
|
|
13
|
+
frontmatter?: boolean;
|
|
14
|
+
/** `true`/`false` from `--respect-robots`/`--no-respect-robots`; undefined = server default. */
|
|
15
|
+
respectRobots?: boolean;
|
|
16
|
+
apiKey?: string;
|
|
17
|
+
platformUrl?: string;
|
|
18
|
+
json?: boolean;
|
|
19
|
+
debug?: boolean;
|
|
20
|
+
/** Seconds to wait for the job before giving up (it keeps running server-side). */
|
|
21
|
+
timeout?: string;
|
|
22
|
+
maxDepth?: string;
|
|
23
|
+
limit?: string;
|
|
24
|
+
include?: string[];
|
|
25
|
+
exclude?: string[];
|
|
26
|
+
sitemap?: string;
|
|
27
|
+
llmsTxt?: boolean;
|
|
28
|
+
file?: string;
|
|
29
|
+
}
|
|
30
|
+
export interface ResolvedJob {
|
|
31
|
+
kind: JobKind;
|
|
32
|
+
/** crawl: exactly one seed URL; batch: 1–100 URLs, de-duplicated, in input order. */
|
|
33
|
+
urls: string[];
|
|
34
|
+
outputDir: string;
|
|
35
|
+
engine?: RequestedEngine;
|
|
36
|
+
region?: Region;
|
|
37
|
+
extractor: ExtractorName;
|
|
38
|
+
frontmatter: boolean;
|
|
39
|
+
respectRobotsTxt?: boolean;
|
|
40
|
+
apiKey: string;
|
|
41
|
+
platformUrl?: string;
|
|
42
|
+
json: boolean;
|
|
43
|
+
debug: boolean;
|
|
44
|
+
timeoutMs: number;
|
|
45
|
+
maxDepth?: number;
|
|
46
|
+
limit?: number;
|
|
47
|
+
includePaths?: string[];
|
|
48
|
+
excludePaths?: string[];
|
|
49
|
+
sitemap?: CrawlSitemapMode;
|
|
50
|
+
llmsTxt: boolean;
|
|
51
|
+
}
|
|
52
|
+
/** Reads a URL list: one per line, blank lines and `#` comments ignored; `-` is stdin. */
|
|
53
|
+
export declare const parseUrlList: (text: string) => string[];
|
|
54
|
+
/**
|
|
55
|
+
* Turns `crawl`/`batch` arguments and flags into a validated job description. Pure except
|
|
56
|
+
* for reading `--file`, so the surface is unit-testable without a server.
|
|
57
|
+
*/
|
|
58
|
+
export declare const resolveJobOptions: (kind: JobKind, sources: string[], flags: JobFlags, env?: Record<string, string | undefined>, readFile?: (file: string) => string) => ResolvedJob;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../../src/cli/commands/jobs/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEpF,OAAO,EAAE,WAAW,EAAE,UAAU,EAAsB,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAErE,gGAAgG;AAChG,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,CAAC;AAClC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AACjC,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAC;AACnC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,CAAC;AAoDxC,MAAM,SAAS,GAAG,CAAC,KAAa,EAAW,EAAE;IAC5C,IAAI,CAAC;QACJ,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,QAAQ,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,CAAmB,IAAY,EAAE,KAAa,EAAE,OAAqB,EAAK,EAAE;IACzF,IAAI,CAAE,OAA6B,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,UAAU,CAAC,aAAa,IAAI,KAAK,KAAK,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzF,CAAC;IACD,OAAO,KAAU,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,KAAyB,EAAE,GAAW,EAAE,GAAW,EAAsB,EAAE;IAC3G,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;QAC/D,MAAM,IAAI,UAAU,CAAC,aAAa,IAAI,KAAK,KAAK,+BAA+B,GAAG,OAAO,GAAG,GAAG,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,MAA4B,EAAwB,EAAE;IACrF,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC;YACJ,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,IAAI,UAAU,CAAC,aAAa,IAAI,aAAa,KAAK,mCAAmC,CAAC,CAAC;QAC9F,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAY,EAAY,EAAE,CACtD,IAAI;KACF,KAAK,CAAC,QAAQ,CAAC;KACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;KAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,QAAkC,EAAY,EAAE;IAClF,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,IAAI,UAAU,CAAC,uBAAuB,IAAI,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjH,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAEpG;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAChC,IAAa,EACb,OAAiB,EACjB,KAAe,EACf,MAA0C,OAAO,CAAC,GAAG,EACrD,WAAqC,eAAe,EACtC,EAAE;IAChB,MAAM,QAAQ,GAAG,IAAI,KAAK,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzF,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAErD,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,UAAU,CAAC,kCAAkC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,UAAU,CAAC,oEAAoE,CAAC,CAAC;IAC5F,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAClC,MAAM,IAAI,UAAU,CACnB,uBAAuB,cAAc,2BAA2B,IAAI,CAAC,MAAM,mBAAmB,CAC9F,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,UAAU,CAAC,wBAAwB,OAAO,GAAG,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,6DAA6D,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC/D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,kBAAkB,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,cAAc,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,uBAAuB,CAAC;IAEvG,OAAO;QACN,IAAI;QACJ,IAAI;QACJ,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,SAAS;QACnF,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM;QACrF,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,KAAK;QACvC,gBAAgB,EAAE,KAAK,CAAC,aAAa;QACrC,MAAM;QACN,WAAW;QACX,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,KAAK;QACzB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,KAAK;QAC3B,SAAS,EAAE,cAAc,GAAG,IAAI;QAChC,QAAQ,EAAE,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,EAAE,eAAe,CAAC;QACpE,KAAK,EAAE,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;QAC1D,YAAY,EAAE,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC;QAChD,YAAY,EAAE,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC;QAChD,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC,SAAS;QACzF,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,KAAK;KAC/B,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { JobStatus } from "../../../platform";
|
|
2
|
+
import type { ResolvedJob } from "./options";
|
|
3
|
+
export interface JobIo {
|
|
4
|
+
stdout: (text: string) => void;
|
|
5
|
+
stderr: (text: string) => void;
|
|
6
|
+
/** Live progress (a rewritten status line) only when stderr is an interactive terminal. */
|
|
7
|
+
isTTY: boolean;
|
|
8
|
+
sleep: (ms: number) => Promise<void>;
|
|
9
|
+
fetch: typeof fetch;
|
|
10
|
+
}
|
|
11
|
+
/** What `--json` prints: the job, every written file, and every failed page. */
|
|
12
|
+
export interface JobEnvelope {
|
|
13
|
+
jobId: string;
|
|
14
|
+
type: ResolvedJob["kind"];
|
|
15
|
+
status: JobStatus["status"];
|
|
16
|
+
outputDir: string;
|
|
17
|
+
credits: number;
|
|
18
|
+
pages: {
|
|
19
|
+
url: string;
|
|
20
|
+
file: string;
|
|
21
|
+
title?: string;
|
|
22
|
+
engine: string;
|
|
23
|
+
credits: number;
|
|
24
|
+
warning?: string;
|
|
25
|
+
}[];
|
|
26
|
+
failures: {
|
|
27
|
+
url: string;
|
|
28
|
+
code: string;
|
|
29
|
+
message: string;
|
|
30
|
+
}[];
|
|
31
|
+
llmsTxt?: string;
|
|
32
|
+
llmsFullTxt?: string;
|
|
33
|
+
error?: string;
|
|
34
|
+
}
|
|
35
|
+
/** The job finished but not cleanly (job failed, or no page converted) — exit code 1. */
|
|
36
|
+
export declare class JobIncompleteError extends Error {
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* `webforai crawl` / `webforai batch`: submit the job, wait for it (progress on stderr), page
|
|
40
|
+
* through the results, write one `.md` per page into the output directory (plus llms.txt
|
|
41
|
+
* files on request), then report. stdout carries only the written file paths — or the
|
|
42
|
+
* `--json` envelope — so the command composes like the single-page one.
|
|
43
|
+
*/
|
|
44
|
+
export declare const runJobCommand: (job: ResolvedJob, io?: JobIo) => Promise<JobEnvelope>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../../../src/cli/commands/jobs/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAE7F,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG9D,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAW/B,MAAM,SAAS,GAAG,GAAU,EAAE,CAAC,CAAC;IAC/B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC5C,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC5C,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IACpC,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAChE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;CACxC,CAAC,CAAC;AAgBH,yFAAyF;AACzF,MAAM,OAAO,kBAAmB,SAAQ,KAAK;CAAG;AAEhD,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,MAAiB,EAAU,EAAE;IAChE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,OAAO,GAAG,IAAI,IAAI,MAAM,CAAC,KAAK,MAAM,MAAM,CAAC,MAAM,MAAM,MAAM,CAAC,SAAS,GAAG,KAAK,SAAS,MAAM,MAAM,MAAM,CAAC,OAAO,UAAU,CAAC;AAC9H,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,kBAAkB,GAAG,KAAK,EAAK,EAAS,EAAE,GAA2B,EAAE,IAAsB,EAAc,EAAE;IAClH,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;QACnC,IAAI,CAAC;YACJ,OAAO,MAAM,IAAI,EAAE,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,CAAC,KAAK,YAAY,gBAAgB,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,OAAO,GAAG,sBAAsB,EAAE,CAAC;gBAC/G,MAAM,KAAK,CAAC;YACb,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,IAAI,OAAO,EAAE,GAAG,CAAC,CAAC;YAChE,GAAG,CAAC,6BAA6B,OAAO,GAAG,CAAC,CAAC;YAC7C,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;QAChC,CAAC;IACF,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,MAAsB,EAAE,GAAgB,EAAE,EAAE;IAC3D,MAAM,MAAM,GAAG;QACd,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,OAAO,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,EAAE;KAChF,CAAC;IACF,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC,KAAK,CAAC;QACnB,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAW;QAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,GAAG,MAAM;KACT,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EAAE,EAAS,EAAE,IAAmB,EAAuB,EAAE;IACjF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,gBAAgB,CACzB,uBAAuB,EACvB,qBAAqB,IAAI,CAAC,GAAG,kCAAkC,QAAQ,CAAC,MAAM,GAAG,EACjF,QAAQ,CAAC,MAAM,CACf,CAAC;IACH,CAAC;IACD,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAgB,CAAC;AAC/C,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,cAAc,GAAG,CAAC,EAAS,EAAE,GAAgB,EAAE,EAAE;IACtD,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,MAAM,SAAS,GAAG,GAAG,EAAE;QACtB,IAAI,QAAQ,EAAE,CAAC;YACd,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzB,QAAQ,GAAG,KAAK,CAAC;QAClB,CAAC;IACF,CAAC,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAE;QAC5B,SAAS,EAAE,CAAC;QACZ,EAAE,CAAC,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,OAAkB,EAAE,EAAE;QACrC,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC;YACd,EAAE,CAAC,MAAM,CAAC,yBAAyB,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;YACtE,QAAQ,GAAG,IAAI,CAAC;QACjB,CAAC;aAAM,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACtB,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC,CAAC;IACF,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACnC,CAAC,CAAC;AAIF,MAAM,gBAAgB,GAAG,KAAK,EAC7B,MAAsB,EACtB,GAAgB,EAChB,KAAa,EACb,EAAS,EACT,QAAkB,EACG,EAAE;IACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC;IAC5C,IAAI,CAAC;QACJ,OAAO,MAAM,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CACtD,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE;YACxB,cAAc,EAAE,gBAAgB;YAChC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7C,QAAQ,EAAE,QAAQ,CAAC,MAAM;SACzB,CAAC,CACF,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,gBAAgB,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACxE,MAAM,IAAI,gBAAgB,CACzB,cAAc,EACd,GAAG,KAAK,CAAC,OAAO,mGAAmG,KAAK,WAAW,EACnI,CAAC,CACD,CAAC;QACH,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,SAAS,EAAE,CAAC;IACtB,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,KAAK,EAAE,MAAsB,EAAE,KAAa,EAAE,EAAS,EAAE,QAAkB,EAAE,EAAE;IACrG,MAAM,SAAS,GAAkB,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,IAAI,MAA0B,CAAC;IAC/B,GAAG,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACvG,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAC5B,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACF,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,CAAC,QAAQ,MAAM,EAAE;IACjB,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,KAAK,EAAE,GAAgB,EAAE,SAAwB,EAAiC,EAAE;IACtG,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,KAAK,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,IAAI,eAAe,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,WAAW,EAAE,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC,EAAE,CAAC;QAC/G,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAgB,CAAC;QAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAC1D,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QAChG,OAAO,CAAC,IAAI,CAAC;YACZ,GAAG;YACH,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC;YAC5C,KAAK,EAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YAChF,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,KAAK,EAAE,GAAgB,EAAE,SAAwB,EAAE,QAAqB,EAAiB,EAAE;IACjH,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAW,EAAE,KAAK,EAAE,CAAC;IACvD,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACxD,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IACjE,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;AACnE,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,GAAgB,EAAE,QAAqB,EAAE,EAAS,EAAE,GAA2B,EAAQ,EAAE;IACxG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;SAAM,CAAC;QACP,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YACnC,EAAE,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QAC7B,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxC,GAAG,CACF,GAAG,GAAG,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,KAAK,CAAC,MAAM,uBAAuB,GAAG,CAAC,SAAS,EAAE;QAC/G,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,QAAQ,CAAC,OAAO,UAAU,CAC1E,CAAC;IACF,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtB,GAAG,CAAC,yCAAyC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,YAAY,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,MAAM,GAAG,mBAAmB,EAAE,CAAC;QAClC,GAAG,CAAC,WAAW,MAAM,GAAG,mBAAmB,oBAAoB,CAAC,CAAC;IAClE,CAAC;AACF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAAE,GAAgB,EAAE,KAAY,SAAS,EAAE,EAAwB,EAAE;IACtG,MAAM,MAAM,GAAG,oBAAoB,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;IACvG,MAAM,QAAQ,GAAG,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAEzC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IACxF,QAAQ,CAAC,GAAG,CACX,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,eAAe,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,OAAO,GAAG,CACxG,CAAC;IAEF,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;IACvE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;IAElF,MAAM,QAAQ,GAAgB;QAC7B,KAAK;QACL,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK,EAAE,MAAM,UAAU,CAAC,GAAG,EAAE,SAAS,CAAC;QACvC,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/F,KAAK,EAAE,KAAK,CAAC,KAAK;KAClB,CAAC;IACF,IAAI,GAAG,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,cAAc,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAChD,CAAC;IACD,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAExC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,kBAAkB,CAAC,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,UAAU,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzG,CAAC;IACD,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,kBAAkB,CAAC,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,qBAAqB,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"content.js","sourceRoot":"","sources":["../../../../../src/cli/commands/skill/content.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC;;;;;gBAKjD,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE
|
|
1
|
+
{"version":3,"file":"content.js","sourceRoot":"","sources":["../../../../../src/cli/commands/skill/content.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC;;;;;gBAKjD,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8J9C,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type CliFlags } from "./options";
|
|
2
2
|
/**
|
|
3
3
|
* The guided flow behind `-i/--interactive` — the pre-v3 default. Prompts only for what the
|
|
4
4
|
* flags did not already provide, then runs the exact same pipeline as the non-interactive
|