ssqch 0.12.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/completions/bash +66 -8
- package/bundled/completions/elvish +23 -4
- package/bundled/completions/fish +36 -25
- package/bundled/completions/powershell +25 -4
- package/bundled/completions/zsh +44 -10
- package/bundled/docs/agents.md +2 -2
- package/bundled/docs/build-summary.md +1 -1
- package/bundled/docs/comparing-builds.md +2 -2
- package/bundled/docs/configuring.md +3 -6
- package/bundled/docs/deploying.md +1 -1
- package/bundled/docs/field-guide.md +4 -4
- package/bundled/docs/github-pages.md +1 -0
- package/bundled/docs/index.md +1 -1
- package/bundled/docs/install.md +1 -1
- package/bundled/docs/known-gaps.md +1 -1
- package/bundled/docs/markdown.md +8 -12
- package/bundled/docs/nav.yml +5 -1
- package/bundled/docs/netlify.md +4 -2
- package/bundled/docs/patrol.md +48 -0
- package/bundled/docs/plugins.md +6 -6
- package/bundled/docs/robots.md +85 -0
- package/bundled/docs/seo.md +2 -38
- package/bundled/docs/serving-markdown.md +142 -0
- package/bundled/docs/template-languages.md +1 -1
- package/bundled/docs/templates.md +1 -1
- package/bundled/docs/trailheads/from-hugo.md +5 -3
- package/bundled/docs/trailheads/from-jekyll.md +54 -13
- package/bundled/docs/trailheads/from-mkdocs.md +15 -12
- package/bundled/docs/trailheads/from-vitepress.md +13 -11
- package/bundled/docs/trailheads.md +3 -3
- package/bundled/docs/where-values-come-from.md +5 -5
- package/bundled/docs/writing-plugins.md +69 -0
- package/bundled/ssqch.d.ts +118 -44
- package/bundled/starter/engines/index.edge +1 -0
- package/bundled/starter/engines/index.ejs +1 -0
- package/bundled/starter/engines/index.hbs +1 -0
- package/bundled/starter/engines/index.liquid +1 -0
- package/bundled/starter/engines/layout.edge +7 -0
- package/bundled/starter/engines/layout.ejs +4 -0
- package/bundled/starter/engines/layout.hbs +4 -0
- package/bundled/starter/engines/layout.liquid +4 -0
- package/bundled/starter/src/_data/site.json +1 -0
- package/bundled/starter/src/_includes/layout.jinja +4 -0
- package/bundled/starter/src/index.jinja +1 -0
- package/bundled/starter/ssqch.config.js +14 -1
- package/bundled/starter-docs/engines/edge/base.edge +7 -0
- package/bundled/starter-docs/engines/edge/doc.edge +4 -0
- package/bundled/starter-docs/engines/ejs/base.ejs +4 -0
- package/bundled/starter-docs/engines/ejs/doc.ejs +4 -0
- package/bundled/starter-docs/engines/hbs/base.hbs +4 -0
- package/bundled/starter-docs/engines/hbs/doc.hbs +3 -0
- package/bundled/starter-docs/engines/liquid/base.liquid +4 -0
- package/bundled/starter-docs/engines/liquid/doc.liquid +4 -0
- package/bundled/starter-docs/src/_data/site.json +1 -0
- package/bundled/starter-docs/src/_includes/base.jinja +4 -0
- package/bundled/starter-docs/src/_includes/doc.jinja +4 -0
- package/bundled/starter-docs/src/assets/docs.css +4 -0
- package/bundled/starter-docs/src/assets/docs.js +17 -0
- package/bundled/starter-docs/ssqch.config.js +14 -1
- package/lib/assets.js +9 -9
- package/lib/chunk-25WERNDD.js +112 -0
- package/lib/{chunk-4ZMNLWE4.js → chunk-2BJFGKOE.js} +2 -2
- package/lib/{chunk-M2BBE2C3.js → chunk-3ZTRIO6O.js} +18 -17
- package/lib/{chunk-57HLSKZS.js → chunk-4AZFCSGJ.js} +2 -2
- package/lib/{chunk-ULQHJ4TS.js → chunk-55WXDGO4.js} +3 -3
- package/lib/{chunk-JX43SJHU.js → chunk-6FY77I4M.js} +24 -22
- package/lib/{chunk-RY3KLUQ3.js → chunk-6OUCRDYI.js} +267 -202
- package/lib/{chunk-EVFNYUTR.js → chunk-6U5VIQVY.js} +5 -5
- package/lib/{chunk-N3IZ44XY.js → chunk-7U5GWTEH.js} +88 -54
- package/lib/{chunk-VTYQ3JKL.js → chunk-AIJNG6MJ.js} +2 -2
- package/lib/{chunk-IIBZPHAW.js → chunk-C2Q7JKFI.js} +8 -8
- package/lib/{chunk-6CSH757V.js → chunk-DFJFHULZ.js} +33 -8
- package/lib/{chunk-XVV7JJK7.js → chunk-ENWDM2RW.js} +40 -39
- package/lib/{chunk-JUALMFW3.js → chunk-ER72RPD6.js} +1 -1
- package/lib/{chunk-RFREB3SI.js → chunk-F3JPNFCZ.js} +7 -7
- package/lib/{chunk-LLOA3QXV.js → chunk-FENJHLNM.js} +3 -2
- package/lib/{chunk-UU43WIZG.js → chunk-FJML7INB.js} +4 -2
- package/lib/{chunk-IZCF4UH3.js → chunk-FQZDUDFK.js} +1 -1
- package/lib/chunk-GM5SHCTU.js +272 -0
- package/lib/{chunk-GVHQ37JE.js → chunk-GSNRAGKQ.js} +10 -5
- package/lib/{chunk-L2IY4P6O.js → chunk-GU7BBYWB.js} +3 -3
- package/lib/{chunk-RGYJQYW3.js → chunk-HDKU6LRI.js} +32 -5
- package/lib/{chunk-IQUGLVZY.js → chunk-K5WZ2VYK.js} +4 -4
- package/lib/{chunk-JVT5ZRRH.js → chunk-OOKBXQAY.js} +5 -5
- package/lib/{chunk-MOLTDO5A.js → chunk-SNQOHNXE.js} +1 -1
- package/lib/{chunk-Z3XH4KWX.js → chunk-SPIFZBIW.js} +1 -1
- package/lib/{chunk-FNNXRVSU.js → chunk-T5IUI7W7.js} +10 -10
- package/lib/{chunk-5HEZ3WUQ.js → chunk-TGSVSG2G.js} +2 -2
- package/lib/{chunk-3GROJ3TY.js → chunk-U2KEMVXO.js} +24 -17
- package/lib/{chunk-4B3N44SN.js → chunk-UHRBPZ4A.js} +3 -3
- package/lib/{chunk-GWWA5Y3X.js → chunk-ULCT3PAV.js} +1 -1
- package/lib/{chunk-7RGEWDLF.js → chunk-UZNUS7RP.js} +10 -8
- package/lib/{chunk-DZ5F5XSI.js → chunk-V3G2QGL3.js} +10 -105
- package/lib/{chunk-DOHOKUGB.js → chunk-VILOEVK7.js} +50 -26
- package/lib/{chunk-DXMDXHV7.js → chunk-WGAIE3DD.js} +15 -14
- package/lib/{chunk-G3RFQDKO.js → chunk-WUCRAV36.js} +8 -6
- package/lib/{chunk-52ROOMBZ.js → chunk-XYAWJTHM.js} +9 -9
- package/lib/{chunk-L3K7MMFC.js → chunk-YA23NDMW.js} +2 -2
- package/lib/{chunk-R7PBREXL.js → chunk-YNJNMQWU.js} +8 -5
- package/lib/{chunk-AK7ZKSZP.js → chunk-ZMG25MXF.js} +17 -16
- package/lib/cli.js +56 -34
- package/lib/elements.js +9 -9
- package/lib/from-hugo.js +36 -0
- package/lib/from-jekyll.js +41 -0
- package/lib/from-mkdocs.js +49 -0
- package/lib/from-vitepress.js +49 -0
- package/lib/image.js +10 -10
- package/lib/links.js +9 -9
- package/lib/llms.js +619 -54
- package/lib/minify.js +10 -10
- package/lib/redirects.js +9 -9
- package/lib/robots.js +87 -0
- package/lib/rss.js +22 -12
- package/lib/sass.js +10 -10
- package/lib/sitemap.js +22 -33
- package/lib/ssqch.js +5 -5
- package/lib/vue.js +14 -13
- package/package.json +21 -1
- package/types/from-hugo.d.ts +5 -0
- package/types/from-jekyll.d.ts +11 -0
- package/types/from-mkdocs.d.ts +11 -0
- package/types/from-vitepress.d.ts +5 -0
- package/types/index.d.ts +45 -29
- package/types/llms.d.ts +18 -8
- package/types/robots.d.ts +13 -0
- package/types/sass.d.ts +1 -1
- package/types/sitemap.d.ts +3 -5
- package/wasm/markdown.wasm +0 -0
- /package/lib/{chunk-JX43SJHU.js.LEGAL.txt → chunk-6FY77I4M.js.LEGAL.txt} +0 -0
- /package/lib/{chunk-R7PBREXL.js.LEGAL.txt → chunk-YNJNMQWU.js.LEGAL.txt} +0 -0
package/lib/llms.js
CHANGED
|
@@ -1,31 +1,35 @@
|
|
|
1
1
|
import { createRequire as __ssqchCreateRequire } from "node:module";
|
|
2
2
|
const require = __ssqchCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
decodeEntities
|
|
5
|
+
} from "./chunk-25WERNDD.js";
|
|
3
6
|
import {
|
|
4
7
|
Options
|
|
5
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-SNQOHNXE.js";
|
|
6
9
|
import {
|
|
7
10
|
BUILTINS,
|
|
8
11
|
debugString
|
|
9
|
-
} from "./chunk-
|
|
10
|
-
import "./chunk-
|
|
11
|
-
import "./chunk-
|
|
12
|
+
} from "./chunk-VILOEVK7.js";
|
|
13
|
+
import "./chunk-ULCT3PAV.js";
|
|
14
|
+
import "./chunk-6U5VIQVY.js";
|
|
12
15
|
import "./chunk-AD3HAW3W.js";
|
|
13
16
|
import "./chunk-YCOTX2VM.js";
|
|
14
17
|
import "./chunk-DDV6JEC2.js";
|
|
15
18
|
import {
|
|
16
|
-
|
|
17
|
-
} from "./chunk-
|
|
18
|
-
import "./chunk-FVZ5JBRI.js";
|
|
19
|
+
currentRendering
|
|
20
|
+
} from "./chunk-6OUCRDYI.js";
|
|
19
21
|
import {
|
|
20
22
|
globSet
|
|
21
|
-
} from "./chunk-
|
|
22
|
-
import "./chunk-
|
|
23
|
-
import "./chunk-
|
|
23
|
+
} from "./chunk-SPIFZBIW.js";
|
|
24
|
+
import "./chunk-FVZ5JBRI.js";
|
|
25
|
+
import "./chunk-FJML7INB.js";
|
|
26
|
+
import "./chunk-4AZFCSGJ.js";
|
|
24
27
|
import "./chunk-CMUWRPSQ.js";
|
|
25
28
|
import {
|
|
26
29
|
bail,
|
|
27
|
-
|
|
28
|
-
|
|
30
|
+
warn,
|
|
31
|
+
warning
|
|
32
|
+
} from "./chunk-YNJNMQWU.js";
|
|
29
33
|
import {
|
|
30
34
|
compareText
|
|
31
35
|
} from "./chunk-VJWQSYEG.js";
|
|
@@ -34,25 +38,486 @@ import "./chunk-I3PS26CI.js";
|
|
|
34
38
|
// src/modules/llms.ts
|
|
35
39
|
import fs from "node:fs";
|
|
36
40
|
import path from "node:path";
|
|
41
|
+
|
|
42
|
+
// src/html-markdown.ts
|
|
43
|
+
var VOID = /* @__PURE__ */ new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]);
|
|
44
|
+
var RAW = /* @__PURE__ */ new Set(["script", "style", "textarea", "title", "xmp", "noscript", "template", "iframe", "noembed", "noframes"]);
|
|
45
|
+
var SKIP = /* @__PURE__ */ new Set(["script", "style", "noscript", "template", "head", "nav", "form", "button", "select", "input", "textarea", "iframe", "canvas", "svg", "math", "video", "audio", "object", "embed", "dialog", "meta", "link", "title"]);
|
|
46
|
+
var BLOCK = /* @__PURE__ */ new Set(["address", "article", "aside", "blockquote", "body", "details", "dialog", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "li", "main", "nav", "ol", "p", "pre", "section", "summary", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul", "html", "caption"]);
|
|
47
|
+
var CLOSES_P = /* @__PURE__ */ new Set(["address", "article", "aside", "blockquote", "details", "div", "dl", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "main", "nav", "ol", "p", "pre", "section", "table", "ul"]);
|
|
48
|
+
var TAG = /<(\/?)([A-Za-z][A-Za-z0-9-]*)((?:\s+[^\s"'>/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>`]+))?)*)\s*(\/?)>/y;
|
|
49
|
+
var ATTR = /([^\s"'>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
|
|
50
|
+
function parse(html) {
|
|
51
|
+
const root = { name: "#root", attrs: /* @__PURE__ */ new Map(), children: [] };
|
|
52
|
+
const stack = [root];
|
|
53
|
+
const top = () => stack[stack.length - 1];
|
|
54
|
+
const close = (name, stopAt = /* @__PURE__ */ new Set()) => {
|
|
55
|
+
for (let i = stack.length - 1; i > 0; i--) {
|
|
56
|
+
if (stopAt.has(stack[i].name)) return;
|
|
57
|
+
if (stack[i].name === name) {
|
|
58
|
+
stack.length = i;
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
let at = 0;
|
|
64
|
+
let text = "";
|
|
65
|
+
const flush = () => {
|
|
66
|
+
if (text) top().children.push(decodeEntities(text));
|
|
67
|
+
text = "";
|
|
68
|
+
};
|
|
69
|
+
while (at < html.length) {
|
|
70
|
+
const lt = html.indexOf("<", at);
|
|
71
|
+
if (lt === -1) {
|
|
72
|
+
text += html.slice(at);
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
75
|
+
text += html.slice(at, lt);
|
|
76
|
+
at = lt;
|
|
77
|
+
if (html.startsWith("<!--", at)) {
|
|
78
|
+
const end = html.indexOf("-->", at + 4);
|
|
79
|
+
at = end === -1 ? html.length : end + 3;
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
if (html[at + 1] === "!" || html[at + 1] === "?") {
|
|
83
|
+
const end = html.indexOf(">", at);
|
|
84
|
+
at = end === -1 ? html.length : end + 1;
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
TAG.lastIndex = at;
|
|
88
|
+
const m = TAG.exec(html);
|
|
89
|
+
if (!m) {
|
|
90
|
+
text += "<";
|
|
91
|
+
at++;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
flush();
|
|
95
|
+
at = TAG.lastIndex;
|
|
96
|
+
const name = m[2].toLowerCase();
|
|
97
|
+
if (m[1]) {
|
|
98
|
+
if (name === "p" && !stack.some((e) => e.name === "p")) continue;
|
|
99
|
+
close(name);
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
const attrs = /* @__PURE__ */ new Map();
|
|
103
|
+
for (const a of m[3].matchAll(ATTR)) attrs.set(a[1].toLowerCase(), decodeEntities(a[2] ?? a[3] ?? a[4] ?? ""));
|
|
104
|
+
if (CLOSES_P.has(name)) close("p", /* @__PURE__ */ new Set(["button", "td", "th", "li", "blockquote", "div", "section", "article", "main"]));
|
|
105
|
+
if (name === "li") close("li", /* @__PURE__ */ new Set(["ul", "ol"]));
|
|
106
|
+
if (name === "dt" || name === "dd") {
|
|
107
|
+
close("dt", /* @__PURE__ */ new Set(["dl"]));
|
|
108
|
+
close("dd", /* @__PURE__ */ new Set(["dl"]));
|
|
109
|
+
}
|
|
110
|
+
if (name === "tr") close("tr", /* @__PURE__ */ new Set(["table", "thead", "tbody", "tfoot"]));
|
|
111
|
+
if (name === "td" || name === "th") {
|
|
112
|
+
close("td", /* @__PURE__ */ new Set(["tr", "table"]));
|
|
113
|
+
close("th", /* @__PURE__ */ new Set(["tr", "table"]));
|
|
114
|
+
}
|
|
115
|
+
if (name === "thead" || name === "tbody" || name === "tfoot") {
|
|
116
|
+
close("thead", /* @__PURE__ */ new Set(["table"]));
|
|
117
|
+
close("tbody", /* @__PURE__ */ new Set(["table"]));
|
|
118
|
+
}
|
|
119
|
+
const el = { name, attrs, children: [] };
|
|
120
|
+
top().children.push(el);
|
|
121
|
+
if (VOID.has(name) || m[4]) continue;
|
|
122
|
+
if (RAW.has(name)) {
|
|
123
|
+
const end = html.toLowerCase().indexOf(`</${name}`, at);
|
|
124
|
+
const inner = html.slice(at, end === -1 ? html.length : end);
|
|
125
|
+
if (name === "textarea" || name === "title") el.children.push(decodeEntities(inner));
|
|
126
|
+
at = end === -1 ? html.length : html.indexOf(">", end) + 1 || html.length;
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
stack.push(el);
|
|
130
|
+
}
|
|
131
|
+
flush();
|
|
132
|
+
return root;
|
|
133
|
+
}
|
|
134
|
+
function find(el, name) {
|
|
135
|
+
for (const c of el.children) {
|
|
136
|
+
if (typeof c === "string") continue;
|
|
137
|
+
if (c.name === name) return c;
|
|
138
|
+
const inner = find(c, name);
|
|
139
|
+
if (inner) return inner;
|
|
140
|
+
}
|
|
141
|
+
return void 0;
|
|
142
|
+
}
|
|
143
|
+
var hidden = (el) => el.attrs.has("hidden") || el.attrs.get("aria-hidden") === "true";
|
|
144
|
+
var skipped = (el) => SKIP.has(el.name) || hidden(el);
|
|
145
|
+
var isBlock = (n) => typeof n !== "string" && BLOCK.has(n.name);
|
|
146
|
+
function textOf(n) {
|
|
147
|
+
if (typeof n === "string") return n;
|
|
148
|
+
if (n.name === "br") return "\n";
|
|
149
|
+
return n.children.map(textOf).join("");
|
|
150
|
+
}
|
|
151
|
+
function escape(text) {
|
|
152
|
+
return text.replace(/\\/g, "\\\\").replace(/[*`[\]]/g, "\\$&").replace(/(^|[^\p{L}\p{N}])_|_(?=[^\p{L}\p{N}]|$)/gu, (m) => m.replace("_", "\\_")).replace(/<(?=[A-Za-z/!?])/g, "\\<").replace(/&(?=#?[A-Za-z0-9]+;)/g, "\\&");
|
|
153
|
+
}
|
|
154
|
+
function escapeLineStart(line) {
|
|
155
|
+
return line.replace(/^(\s*)(#{1,6}(?=\s|$)|>|[-+](?=\s)|(\d+)([.)])(?=\s)|=+\s*$|~~~)/, (_, space, mark, digits, dot) => digits ? `${space}${digits}\\${dot}` : `${space}\\${mark}`);
|
|
156
|
+
}
|
|
157
|
+
var selfLink = (el) => el.name === "a" && (el.attrs.get("href") ?? "").startsWith("#") && !/[\p{L}\p{N}]/u.test(textOf(el));
|
|
158
|
+
function inline(nodes, inHeading = false) {
|
|
159
|
+
let out = "";
|
|
160
|
+
for (const n of nodes) {
|
|
161
|
+
if (typeof n === "string") {
|
|
162
|
+
out += escape(n.replace(/\s+/g, " "));
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
if (skipped(n)) continue;
|
|
166
|
+
switch (n.name) {
|
|
167
|
+
case "br":
|
|
168
|
+
out += "\\\n";
|
|
169
|
+
break;
|
|
170
|
+
case "img": {
|
|
171
|
+
const alt = n.attrs.get("alt");
|
|
172
|
+
if (alt === "" || !n.attrs.get("src")) break;
|
|
173
|
+
out += `)})`;
|
|
174
|
+
break;
|
|
175
|
+
}
|
|
176
|
+
case "a": {
|
|
177
|
+
if (inHeading && selfLink(n)) break;
|
|
178
|
+
const text = inline(n.children, inHeading).trim();
|
|
179
|
+
const href = n.attrs.get("href");
|
|
180
|
+
if (!href || !text) {
|
|
181
|
+
out += text;
|
|
182
|
+
break;
|
|
183
|
+
}
|
|
184
|
+
const title = n.attrs.get("title");
|
|
185
|
+
out += `[${text}](${destination(href)}${title ? ` "${title.replace(/"/g, '\\"')}"` : ""})`;
|
|
186
|
+
break;
|
|
187
|
+
}
|
|
188
|
+
case "strong":
|
|
189
|
+
case "b": {
|
|
190
|
+
const t = inline(n.children, inHeading).trim();
|
|
191
|
+
if (t) out += `**${t}**`;
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
case "em":
|
|
195
|
+
case "i":
|
|
196
|
+
case "cite":
|
|
197
|
+
case "dfn": {
|
|
198
|
+
const t = inline(n.children, inHeading).trim();
|
|
199
|
+
if (t) out += `*${t}*`;
|
|
200
|
+
break;
|
|
201
|
+
}
|
|
202
|
+
case "del":
|
|
203
|
+
case "s":
|
|
204
|
+
case "strike": {
|
|
205
|
+
const t = inline(n.children, inHeading).trim();
|
|
206
|
+
if (t) out += `~~${t}~~`;
|
|
207
|
+
break;
|
|
208
|
+
}
|
|
209
|
+
case "code":
|
|
210
|
+
case "kbd":
|
|
211
|
+
case "samp":
|
|
212
|
+
case "tt":
|
|
213
|
+
out += codeSpan(textOf(n));
|
|
214
|
+
break;
|
|
215
|
+
default:
|
|
216
|
+
out += inline(n.children, inHeading);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return out;
|
|
220
|
+
}
|
|
221
|
+
var destination = (href) => /[\s()<>]/.test(href) ? `<${href.replace(/[<>]/g, (c) => encodeURIComponent(c))}>` : href;
|
|
222
|
+
function codeSpan(text) {
|
|
223
|
+
const code = text.replace(/\n/g, " ");
|
|
224
|
+
if (!code) return "";
|
|
225
|
+
const longest = Math.max(0, ...[...code.matchAll(/`+/g)].map((m) => m[0].length));
|
|
226
|
+
const ticks = "`".repeat(longest + 1);
|
|
227
|
+
const pad = code.startsWith("`") || code.endsWith("`") || code.startsWith(" ") && code.endsWith(" ") && code.trim() ? " " : "";
|
|
228
|
+
return `${ticks}${pad}${code}${pad}${ticks}`;
|
|
229
|
+
}
|
|
230
|
+
function paragraph(text) {
|
|
231
|
+
return text.replace(/[ \t]+/g, " ").split("\n").map((l) => escapeLineStart(l.trim())).join("\n").trim();
|
|
232
|
+
}
|
|
233
|
+
function languageOf(pre) {
|
|
234
|
+
const code = pre.children.find((c) => typeof c !== "string" && c.name === "code");
|
|
235
|
+
for (const el of [code, pre]) {
|
|
236
|
+
const m = /(?:^|\s)(?:language|lang)-([\w+#.-]+)/.exec(el?.attrs.get("class") ?? "") ?? (el?.attrs.get("data-lang") ? [, el.attrs.get("data-lang")] : null);
|
|
237
|
+
if (m?.[1]) return m[1];
|
|
238
|
+
}
|
|
239
|
+
return "";
|
|
240
|
+
}
|
|
241
|
+
function fence(text, lang) {
|
|
242
|
+
const body = text.replace(/\n$/, "");
|
|
243
|
+
const longest = Math.max(2, ...[...body.matchAll(/^\s*(`{3,})/gm)].map((m) => m[1].length));
|
|
244
|
+
const marks = "`".repeat(longest + 1);
|
|
245
|
+
return `${marks}${lang}
|
|
246
|
+
${body}
|
|
247
|
+
${marks}`;
|
|
248
|
+
}
|
|
249
|
+
var indent = (text, by) => text.split("\n").map((l, i) => i === 0 || l === "" ? l : by + l).join("\n");
|
|
250
|
+
function blocks(nodes) {
|
|
251
|
+
const out = [];
|
|
252
|
+
let run = [];
|
|
253
|
+
const endRun = () => {
|
|
254
|
+
const text = paragraph(inline(run));
|
|
255
|
+
if (text) out.push(text);
|
|
256
|
+
run = [];
|
|
257
|
+
};
|
|
258
|
+
for (const n of nodes) {
|
|
259
|
+
if (typeof n === "string" || !isBlock(n)) {
|
|
260
|
+
if (typeof n !== "string" && skipped(n)) continue;
|
|
261
|
+
run.push(n);
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
endRun();
|
|
265
|
+
if (skipped(n)) continue;
|
|
266
|
+
out.push(...block(n));
|
|
267
|
+
}
|
|
268
|
+
endRun();
|
|
269
|
+
return out;
|
|
270
|
+
}
|
|
271
|
+
function block(el) {
|
|
272
|
+
const name = el.name;
|
|
273
|
+
const heading = /^h([1-6])$/.exec(name);
|
|
274
|
+
if (heading) {
|
|
275
|
+
const text = inline(el.children, true).replace(/\s+/g, " ").trim();
|
|
276
|
+
return text ? [`${"#".repeat(Number(heading[1]))} ${text}`] : [];
|
|
277
|
+
}
|
|
278
|
+
switch (name) {
|
|
279
|
+
case "p": {
|
|
280
|
+
const t = paragraph(inline(el.children));
|
|
281
|
+
return t ? [t] : [];
|
|
282
|
+
}
|
|
283
|
+
case "hr":
|
|
284
|
+
return ["---"];
|
|
285
|
+
case "pre":
|
|
286
|
+
return [fence(textOf(el), languageOf(el))];
|
|
287
|
+
case "blockquote": {
|
|
288
|
+
const inner = blocks(el.children).join("\n\n");
|
|
289
|
+
return inner ? [inner.split("\n").map((l) => l ? `> ${l}` : ">").join("\n")] : [];
|
|
290
|
+
}
|
|
291
|
+
case "ul":
|
|
292
|
+
case "ol":
|
|
293
|
+
return list(el);
|
|
294
|
+
case "table":
|
|
295
|
+
return table(el);
|
|
296
|
+
case "dl": {
|
|
297
|
+
const out = [];
|
|
298
|
+
for (const c of el.children) {
|
|
299
|
+
if (typeof c === "string" || skipped(c)) continue;
|
|
300
|
+
if (c.name === "dt") {
|
|
301
|
+
const t = inline(c.children).trim();
|
|
302
|
+
if (t) out.push(`**${t}**`);
|
|
303
|
+
} else if (c.name === "dd") out.push(...blocks(c.children));
|
|
304
|
+
else out.push(...blocks([c]));
|
|
305
|
+
}
|
|
306
|
+
return out;
|
|
307
|
+
}
|
|
308
|
+
case "summary": {
|
|
309
|
+
const t = inline(el.children).trim();
|
|
310
|
+
return t ? [`**${t}**`] : [];
|
|
311
|
+
}
|
|
312
|
+
default:
|
|
313
|
+
return blocks(el.children);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
function list(el) {
|
|
317
|
+
const ordered = el.name === "ol";
|
|
318
|
+
let n = ordered ? Number(el.attrs.get("start") ?? 1) || 1 : 0;
|
|
319
|
+
const items = [];
|
|
320
|
+
for (const c of el.children) {
|
|
321
|
+
if (typeof c === "string" || c.name !== "li" || skipped(c)) continue;
|
|
322
|
+
const marker = ordered ? `${n++}. ` : "- ";
|
|
323
|
+
const inner = blocks(c.children);
|
|
324
|
+
const loose = c.children.some((x) => typeof x !== "string" && x.name === "p");
|
|
325
|
+
const body = inner.join(loose ? "\n\n" : "\n");
|
|
326
|
+
items.push(marker + indent(body, " ".repeat(marker.length)));
|
|
327
|
+
}
|
|
328
|
+
return items.length ? [items.join("\n")] : [];
|
|
329
|
+
}
|
|
330
|
+
function table(el) {
|
|
331
|
+
const rows = [];
|
|
332
|
+
const walk = (e, head) => {
|
|
333
|
+
for (const c of e.children) {
|
|
334
|
+
if (typeof c === "string" || skipped(c)) continue;
|
|
335
|
+
if (c.name === "tr") {
|
|
336
|
+
const cells = c.children.filter((x) => typeof x !== "string" && (x.name === "td" || x.name === "th"));
|
|
337
|
+
rows.push({ cells: cells.map((x) => inline(x.children).replace(/\s+/g, " ").trim().replace(/\|/g, "\\|")), head: head || cells.length > 0 && cells.every((x) => x.name === "th") });
|
|
338
|
+
} else if (c.name === "thead") walk(c, true);
|
|
339
|
+
else if (c.name === "tbody" || c.name === "tfoot") walk(c, false);
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
walk(el, false);
|
|
343
|
+
if (!rows.length) return [];
|
|
344
|
+
const width = Math.max(...rows.map((r) => r.cells.length));
|
|
345
|
+
const line = (cells) => `| ${Array.from({ length: width }, (_, i) => cells[i] ?? "").join(" | ")} |`;
|
|
346
|
+
const [first, ...rest] = rows[0].head ? rows : [{ cells: [], head: true }, ...rows];
|
|
347
|
+
const caption = find(el, "caption");
|
|
348
|
+
const out = caption ? [paragraph(inline(caption.children))].filter(Boolean) : [];
|
|
349
|
+
out.push([line(first.cells), `|${" --- |".repeat(width)}`, ...rest.map((r) => line(r.cells))].join("\n"));
|
|
350
|
+
return out;
|
|
351
|
+
}
|
|
352
|
+
function htmlToMarkdown(html) {
|
|
353
|
+
const root = parse(html);
|
|
354
|
+
const part = find(root, "article") ?? find(root, "main") ?? find(root, "body") ?? root;
|
|
355
|
+
const text = blocks(part.children).join("\n\n").trim();
|
|
356
|
+
return text ? `${text}
|
|
357
|
+
` : "";
|
|
358
|
+
}
|
|
359
|
+
var INLINE_LINK = /(\]\()(\s*)(<[^>\n]*>|(?:[^\s()\\]|\\.|\((?:[^\s()\\]|\\.)*\))+)/g;
|
|
360
|
+
var REFERENCE = /^( {0,3}\[[^\]\n]+\]:[ \t]*)(<[^>\n]*>|\S+)/;
|
|
361
|
+
var HTML_URL = /(<(?:a|img|source|link)\b[^>]*?\s(?:href|src)\s*=\s*)("([^"]*)"|'([^']*)')/gi;
|
|
362
|
+
function rewriteLinks(markdown, to) {
|
|
363
|
+
let fenced;
|
|
364
|
+
return markdown.split("\n").map((line) => {
|
|
365
|
+
const marker = /^\s{0,3}(`{3,}|~{3,})/.exec(line)?.[1];
|
|
366
|
+
if (fenced !== void 0) {
|
|
367
|
+
if (marker && marker[0] === fenced[0] && marker.length >= fenced.length && line.trim() === marker) fenced = void 0;
|
|
368
|
+
return line;
|
|
369
|
+
}
|
|
370
|
+
if (marker) {
|
|
371
|
+
fenced = marker;
|
|
372
|
+
return line;
|
|
373
|
+
}
|
|
374
|
+
const ref = REFERENCE.exec(line);
|
|
375
|
+
if (ref) {
|
|
376
|
+
const href = ref[2].startsWith("<") ? ref[2].slice(1, -1) : ref[2];
|
|
377
|
+
return ref[1] + destination(to(href)) + line.slice(ref[0].length);
|
|
378
|
+
}
|
|
379
|
+
return line.split(/(`+[^`]*`+)/).map((part, i) => i % 2 === 1 ? part : part.replace(INLINE_LINK, (_, open, space, dest) => open + space + destination(to(dest.startsWith("<") ? dest.slice(1, -1) : dest.replace(/\\(.)/g, "$1")))).replace(HTML_URL, (_, before, _q, d, s) => `${before}"${to(d ?? s ?? "").replace(/"/g, """)}"`)).join("");
|
|
380
|
+
}).join("\n");
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// src/modules/llms.ts
|
|
37
384
|
function llmsPlugin(options = {}) {
|
|
38
385
|
return { name: "ssqch/llms", builtin: "ssqch/llms", options };
|
|
39
386
|
}
|
|
40
387
|
function copyUrl(url) {
|
|
41
|
-
if (url
|
|
42
|
-
|
|
43
|
-
|
|
388
|
+
if (url.endsWith("/")) return `${url}index.md`;
|
|
389
|
+
return url.replace(/\.html?$/, "") + ".md";
|
|
390
|
+
}
|
|
391
|
+
function folderName(folder) {
|
|
392
|
+
const name = folder.replace(/[-_]+/g, " ");
|
|
393
|
+
return name.charAt(0).toUpperCase() + name.slice(1);
|
|
44
394
|
}
|
|
395
|
+
var BUDGET = 5e4;
|
|
45
396
|
function sectionOf(url) {
|
|
46
397
|
const first = url.split("/").filter(Boolean);
|
|
47
398
|
if (first.length < 2 && !url.endsWith("/")) return "Pages";
|
|
48
399
|
if (first.length === 0) return "Pages";
|
|
49
|
-
|
|
50
|
-
return name.charAt(0).toUpperCase() + name.slice(1);
|
|
400
|
+
return folderName(first[0]);
|
|
51
401
|
}
|
|
52
|
-
var
|
|
402
|
+
var grouped = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
403
|
+
var shown = (v) => typeof v === "string" ? debugString(v) : JSON.stringify(v) ?? typeof v;
|
|
53
404
|
var oneLine = (s) => typeof s === "string" ? s.replace(/\s+/g, " ").trim() : "";
|
|
405
|
+
var isPageUrl = (url) => url.endsWith("/") || url.endsWith(".html");
|
|
406
|
+
var keyOf = (url) => url.replace(/\/index\.html$/, "/");
|
|
407
|
+
function pageText(markdown) {
|
|
408
|
+
const out = [];
|
|
409
|
+
let fenced;
|
|
410
|
+
let raw;
|
|
411
|
+
for (const line of markdown.split("\n")) {
|
|
412
|
+
if (raw !== void 0) {
|
|
413
|
+
if (new RegExp(`</${raw}\\s*>`, "i").test(line)) raw = void 0;
|
|
414
|
+
continue;
|
|
415
|
+
}
|
|
416
|
+
const marker = /^\s{0,3}(`{3,}|~{3,})/.exec(line)?.[1];
|
|
417
|
+
if (fenced !== void 0) {
|
|
418
|
+
if (marker && marker[0] === fenced[0] && marker.length >= fenced.length && line.trim() === marker) fenced = void 0;
|
|
419
|
+
out.push(line);
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
if (marker) {
|
|
423
|
+
fenced = marker;
|
|
424
|
+
out.push(line);
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
427
|
+
const open = /^\s{0,3}<(script|style)\b/i.exec(line);
|
|
428
|
+
if (open) {
|
|
429
|
+
if (!new RegExp(`</${open[1]}\\s*>`, "i").test(line)) raw = open[1].toLowerCase();
|
|
430
|
+
continue;
|
|
431
|
+
}
|
|
432
|
+
if (/^\s*(?:<img\b[^>]*\balt=(?:""|'')[^>]*>\s*)+$/i.test(line)) continue;
|
|
433
|
+
if (line.trim() === "" && (out.length === 0 || out[out.length - 1].trim() === "")) continue;
|
|
434
|
+
out.push(line);
|
|
435
|
+
}
|
|
436
|
+
return out.join("\n").replace(/^\s+/, "");
|
|
437
|
+
}
|
|
438
|
+
function publishedImages(markdown, outputPath, base, output) {
|
|
439
|
+
let html;
|
|
440
|
+
try {
|
|
441
|
+
html = fs.readFileSync(outputPath, "utf8");
|
|
442
|
+
} catch {
|
|
443
|
+
return markdown;
|
|
444
|
+
}
|
|
445
|
+
const attr = (tag, name) => new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i").exec(tag)?.slice(1).find((v) => v !== void 0);
|
|
446
|
+
const byAlt = /* @__PURE__ */ new Map();
|
|
447
|
+
for (const [tag] of html.matchAll(/<img\b[^>]*>/gi)) {
|
|
448
|
+
const alt = decodeEntities(attr(tag, "alt") ?? "");
|
|
449
|
+
let src = attr(tag, "src");
|
|
450
|
+
if (src === void 0 || /^[a-z][a-z0-9+.-]*:/i.test(src)) continue;
|
|
451
|
+
src = decodeEntities(src);
|
|
452
|
+
if (base && src.startsWith(`${base}/`)) src = src.slice(base.length);
|
|
453
|
+
byAlt.set(alt, [...byAlt.get(alt) ?? [], src]);
|
|
454
|
+
}
|
|
455
|
+
const byText = /* @__PURE__ */ new Map();
|
|
456
|
+
for (const [, href, text] of html.matchAll(/<a\b[^>]*\shref\s*=\s*"([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi)) {
|
|
457
|
+
const key = decodeEntities(text.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
|
458
|
+
if (key) byText.set(key, [...byText.get(key) ?? [], base && href.startsWith(`${base}/`) ? href.slice(base.length) : href]);
|
|
459
|
+
}
|
|
460
|
+
if (!byAlt.size && !byText.size) return markdown;
|
|
461
|
+
const next = (alt) => byAlt.get(alt)?.shift();
|
|
462
|
+
const became = /* @__PURE__ */ new Map();
|
|
463
|
+
const out = !byAlt.size ? markdown : markdown.replace(/(!\[([^\]]*)\]\()(<[^>]*>|[^\s)]+)|<img\b[^>]*>/gi, (whole, open, alt, dest) => {
|
|
464
|
+
if (open !== void 0) {
|
|
465
|
+
const src2 = next((alt ?? "").replace(/\\(.)/g, "$1"));
|
|
466
|
+
if (src2 === void 0) return whole;
|
|
467
|
+
became.set((dest ?? "").replace(/^<|>$/g, ""), src2);
|
|
468
|
+
return open + src2;
|
|
469
|
+
}
|
|
470
|
+
const src = next(decodeEntities(attr(whole, "alt") ?? ""));
|
|
471
|
+
if (src === void 0) return whole;
|
|
472
|
+
const was = attr(whole, "src");
|
|
473
|
+
if (was !== void 0) became.set(decodeEntities(was), src);
|
|
474
|
+
return whole.replace(/(\ssrc\s*=\s*)(?:"[^"]*"|'[^']*'|[^\s>]+)/i, `$1"${src}"`);
|
|
475
|
+
});
|
|
476
|
+
const missing = (href) => href.startsWith("/") && !href.startsWith("//") && !fs.existsSync(path.join(output, decodeURIComponent(href.split(/[?#]/)[0])));
|
|
477
|
+
return out.replace(/(\[([^\]]*)\]\()([^\s)]+)/g, (whole, open, text, href) => {
|
|
478
|
+
if (became.has(href)) return open + became.get(href);
|
|
479
|
+
const instead = missing(href) ? byText.get(text.replace(/\\(.)/g, "$1").replace(/[*_`]/g, "").trim())?.find((h) => !missing(h)) : void 0;
|
|
480
|
+
return instead === void 0 ? whole : open + instead;
|
|
481
|
+
}).replace(/(<a\b[^>]*\shref\s*=\s*)(?:"([^"]*)"|'([^']*)')/gi, (whole, open, d, q) => {
|
|
482
|
+
const href = decodeEntities(d ?? q ?? "");
|
|
483
|
+
return became.has(href) ? `${open}"${became.get(href)}"` : whole;
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
function listing(head, items, budget, file) {
|
|
487
|
+
const render = (list2) => {
|
|
488
|
+
const sections = /* @__PURE__ */ new Map();
|
|
489
|
+
for (const item of list2) sections.set(item.section, [...sections.get(item.section) ?? [], item.line]);
|
|
490
|
+
const optional = sections.get("Optional");
|
|
491
|
+
if (optional) {
|
|
492
|
+
sections.delete("Optional");
|
|
493
|
+
sections.set("Optional", optional);
|
|
494
|
+
}
|
|
495
|
+
let txt2 = head;
|
|
496
|
+
for (const [name, lines] of sections) txt2 += `
|
|
497
|
+
## ${name}
|
|
498
|
+
|
|
499
|
+
${lines.join("")}`;
|
|
500
|
+
return txt2;
|
|
501
|
+
};
|
|
502
|
+
let txt = render(items);
|
|
503
|
+
if (txt.length <= budget) return txt;
|
|
504
|
+
const moved = items.map((item) => ({ ...item }));
|
|
505
|
+
let over = txt.length - budget + (items.some((i) => i.section === "Optional") ? 0 : "\n## Optional\n\n".length);
|
|
506
|
+
const most = items.reduce((n, i) => n + (i.section === "Optional" ? 0 : i.line.length - i.short.length), 0);
|
|
507
|
+
if (most < over) {
|
|
508
|
+
warn(warning("build", `llmsPlugin: ${file} is ${grouped(txt.length)} characters, over the budget of ${grouped(budget)} even without its pages' notes: give a part of the site an llms.txt of its own (paths: ["/docs/"]), or raise budget`));
|
|
509
|
+
return txt;
|
|
510
|
+
}
|
|
511
|
+
for (let i = moved.length - 1; i >= 0 && over > 0; i--) {
|
|
512
|
+
if (moved[i].section === "Optional") continue;
|
|
513
|
+
over -= moved[i].line.length - moved[i].short.length;
|
|
514
|
+
moved[i] = { section: "Optional", line: moved[i].short, short: moved[i].short };
|
|
515
|
+
}
|
|
516
|
+
return render(moved);
|
|
517
|
+
}
|
|
54
518
|
BUILTINS.set("ssqch/llms", (config, given) => {
|
|
55
|
-
|
|
519
|
+
if (given !== null && typeof given === "object" && Object.hasOwn(given, "hint")) bail('`hint` is gone: a build adds no markup of its own. Link the copy from your layout\'s <head> instead: <link rel="alternate" type="text/markdown" href="{{ markdownCopy(page.url) }}">');
|
|
520
|
+
const o = new Options(given, ["title", "description", "details", "url", "exclude", "copies", "full", "section", "sort", "paths", "budget"]);
|
|
56
521
|
const url = o.string("url");
|
|
57
522
|
if (url !== void 0 && url !== "" && !/^https?:\/\//.test(url)) bail(`url must be the site's address, like "https://example.com" (got ${debugString(url)})`);
|
|
58
523
|
const exclude = o.list("exclude") ?? [];
|
|
@@ -60,42 +525,105 @@ BUILTINS.set("ssqch/llms", (config, given) => {
|
|
|
60
525
|
const excluded = exclude.length ? globSet("exclude", exclude) : () => false;
|
|
61
526
|
const copies = o.bool("copies") ?? true;
|
|
62
527
|
const full = o.bool("full") ?? true;
|
|
63
|
-
const hint = o.bool("hint") ?? false;
|
|
64
528
|
const section = o.get("section");
|
|
65
529
|
if (section !== void 0 && typeof section !== "function") bail(`section must be a function that gets each page and returns its section's name, like (page) => page.data.category (got ${typeof section})`);
|
|
530
|
+
const sort = o.get("sort");
|
|
531
|
+
if (sort !== void 0 && typeof sort !== "function") bail(`sort must be a function that compares two pages, like (a, b) => a.data.order - b.data.order (got ${typeof sort})`);
|
|
66
532
|
const title = o.string("title");
|
|
67
533
|
const description = o.string("description");
|
|
534
|
+
const details = o.string("details");
|
|
68
535
|
const address = (url ?? "").replace(/\/+$/, "");
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
}
|
|
536
|
+
const givenBudget = o.get("budget");
|
|
537
|
+
if (givenBudget !== void 0 && givenBudget !== false && !(typeof givenBudget === "number" && Number.isInteger(givenBudget) && givenBudget > 0)) bail(`budget must be a number of characters, like 50000, or false for no limit (got ${shown(givenBudget)})`);
|
|
538
|
+
const budget = givenBudget === false ? Infinity : givenBudget ?? BUDGET;
|
|
539
|
+
const paths = [];
|
|
540
|
+
const givenPaths = o.get("paths");
|
|
541
|
+
if (givenPaths !== void 0 && !Array.isArray(givenPaths)) bail(`paths must be a list of URL paths that get an llms.txt of their own, like ["/docs/"] (got ${shown(givenPaths)})`);
|
|
542
|
+
for (const item of givenPaths ?? []) {
|
|
543
|
+
const one = typeof item === "string" ? { path: item } : item;
|
|
544
|
+
if (one === null || typeof one !== "object" || typeof one.path !== "string") bail(`paths: each one is a URL path like "/docs/", or { path: "/docs/", title, description, details } (got ${shown(item)})`);
|
|
545
|
+
const { path: at, ...rest } = one;
|
|
546
|
+
for (const [key, value] of Object.entries(rest)) {
|
|
547
|
+
if (!["title", "description", "details"].includes(key)) bail(`paths: ${shown(at)} has \`${key}\`, which isn't an option; it takes title, description and details`);
|
|
548
|
+
if (typeof value !== "string") bail(`paths: ${shown(at)}'s ${key} must be a string (got ${shown(value)})`);
|
|
549
|
+
}
|
|
550
|
+
const folder = at;
|
|
551
|
+
if (!folder.startsWith("/") || !folder.endsWith("/") || folder === "/") bail(`paths: ${shown(folder)} must be a folder's URL path, starting and ending with /, like "/docs/" (/llms.txt is the site's)`);
|
|
552
|
+
if (paths.some((f) => f.path === folder)) bail(`paths: ${shown(folder)} is listed twice`);
|
|
553
|
+
paths.push({ path: folder, ...rest });
|
|
78
554
|
}
|
|
555
|
+
paths.sort((a, b) => b.path.length - a.path.length || compareText(a.path, b.path));
|
|
556
|
+
const coveringFile = (pageUrl) => paths.find((f) => pageUrl.startsWith(f.path))?.path ?? "/";
|
|
557
|
+
const listedUrl = (pageUrl) => isPageUrl(pageUrl) && !excluded(pageUrl) && pageUrl !== "/404.html" && pageUrl !== "/404/";
|
|
558
|
+
const listed = (p) => p.outputPath.endsWith(".html") && listedUrl(p.url) && p.data.llms !== false;
|
|
559
|
+
const sources = /* @__PURE__ */ new Map();
|
|
560
|
+
config.markdownSources = sources;
|
|
561
|
+
const converted = /* @__PURE__ */ new Map();
|
|
562
|
+
const titles = /* @__PURE__ */ new Map();
|
|
563
|
+
config.htmlTransforms.push(["llms", (html, page) => {
|
|
564
|
+
if (copies || full) {
|
|
565
|
+
if (page.outputPath.endsWith(".html") && listedUrl(page.url)) {
|
|
566
|
+
if (!sources.has(page.url)) converted.set(page.url, htmlToMarkdown(html));
|
|
567
|
+
const title2 = /<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1];
|
|
568
|
+
if (title2) titles.set(page.url, decodeEntities(title2).replace(/\s+/g, " ").trim());
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
return html;
|
|
572
|
+
}]);
|
|
573
|
+
config.filters.set("markdownCopy", (value) => {
|
|
574
|
+
const pageUrl = value === void 0 || value === null ? "" : String(value);
|
|
575
|
+
if (!copies || !listedUrl(pageUrl)) return "";
|
|
576
|
+
const data = currentRendering()?.data;
|
|
577
|
+
const page = data?.page;
|
|
578
|
+
if (page && page.url === pageUrl && data?.llms === false) return "";
|
|
579
|
+
return copyUrl(pageUrl);
|
|
580
|
+
});
|
|
581
|
+
config.filters.set("llmsTxt", (value) => `${coveringFile(value === void 0 || value === null ? "/" : String(value))}llms.txt`);
|
|
79
582
|
config.generators.push(["ssqch/llms", (config2, pages) => {
|
|
80
583
|
const base = config2.base.replace(/\/$/, "");
|
|
81
|
-
const
|
|
584
|
+
const output = path.resolve(config2.root, config2.output);
|
|
585
|
+
const whole = (u) => `${address}${base}${u}`;
|
|
82
586
|
const files = [];
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
587
|
+
const chosen = pages.filter(listed).sort((a, b) => compareText(a.url, b.url));
|
|
588
|
+
if (typeof sort === "function") {
|
|
589
|
+
try {
|
|
590
|
+
chosen.sort(sort);
|
|
591
|
+
} catch (e) {
|
|
592
|
+
bail(`sort(): ${e instanceof Error ? e.message : String(e)}`);
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
const withCopy = new Set(chosen);
|
|
596
|
+
const byUrl = new Map(pages.map((p) => [keyOf(p.url), p]));
|
|
597
|
+
const byFile = new Map(pages.map((p) => [path.resolve(p.inputPath), p]));
|
|
598
|
+
const linkTo = (page) => copies && withCopy.has(page) ? copyUrl(page.url) : page.url;
|
|
599
|
+
const target = (href, from) => {
|
|
600
|
+
if (href === "" || href.startsWith("#") || href.startsWith("//") || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(href)) return href;
|
|
601
|
+
const cut = href.search(/[?#]/);
|
|
602
|
+
const [file, rest] = cut === -1 ? [href, ""] : [href.slice(0, cut), href.slice(cut)];
|
|
603
|
+
if (/\.(?:md|markdown)$/i.test(file) && !file.startsWith("/")) {
|
|
604
|
+
const page2 = byFile.get(path.resolve(path.dirname(from.inputPath), decodeURIComponent(file)));
|
|
605
|
+
if (page2) return whole(linkTo(page2)) + rest;
|
|
606
|
+
}
|
|
607
|
+
let resolved;
|
|
87
608
|
try {
|
|
88
|
-
|
|
609
|
+
resolved = new URL(file, `https://ssqch.invalid${from.url}`).pathname;
|
|
89
610
|
} catch {
|
|
90
|
-
|
|
611
|
+
return href;
|
|
91
612
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
613
|
+
const page = byUrl.get(keyOf(resolved)) ?? byUrl.get(keyOf(`${resolved}/`));
|
|
614
|
+
return whole(page ? linkTo(page) : resolved) + rest;
|
|
615
|
+
};
|
|
616
|
+
const entries = [];
|
|
617
|
+
for (const p of chosen) {
|
|
618
|
+
const source = sources.get(p.url);
|
|
619
|
+
let body = source !== void 0 ? pageText(source) : converted.get(p.url) ?? "";
|
|
620
|
+
const pageTitle = oneLine(p.data.title) || /^#\s+(.+)$/m.exec(body)?.[1]?.trim() || titles.get(p.url) || p.url;
|
|
621
|
+
if (!/^#\s/.test(body) && oneLine(p.data.title)) body = `# ${oneLine(p.data.title)}
|
|
96
622
|
|
|
97
623
|
${body}`;
|
|
98
|
-
|
|
624
|
+
body = publishedImages(body, p.outputPath, base, output);
|
|
625
|
+
body = rewriteLinks(body, (href) => target(href, p));
|
|
626
|
+
let name;
|
|
99
627
|
if (typeof section === "function") {
|
|
100
628
|
let out;
|
|
101
629
|
try {
|
|
@@ -107,30 +635,67 @@ ${body}`;
|
|
|
107
635
|
}
|
|
108
636
|
entries.push({ url: p.url, copy: copyUrl(p.url), title: pageTitle, description: oneLine(p.data.description), section: name, markdown: body.replace(/\s+$/, "") + "\n" });
|
|
109
637
|
}
|
|
638
|
+
sources.clear();
|
|
639
|
+
converted.clear();
|
|
640
|
+
titles.clear();
|
|
110
641
|
const site = pages[0]?.data.site;
|
|
111
642
|
const siteTitle = title ?? (oneLine(site !== null && typeof site === "object" ? site.title : void 0) || entries.find((e) => e.url === "/")?.title || "Pages");
|
|
112
|
-
let
|
|
643
|
+
let head = `# ${siteTitle}
|
|
113
644
|
`;
|
|
114
|
-
if (description)
|
|
645
|
+
if (description) head += `
|
|
115
646
|
> ${oneLine(description)}
|
|
116
647
|
`;
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
for (const [name, list] of sections) {
|
|
120
|
-
txt += `
|
|
121
|
-
## ${name}
|
|
122
|
-
|
|
648
|
+
if (details?.trim()) head += `
|
|
649
|
+
${details.trim()}
|
|
123
650
|
`;
|
|
124
|
-
|
|
651
|
+
const written = paths.filter((f) => {
|
|
652
|
+
if (entries.some((e) => coveringFile(e.url) === f.path)) return true;
|
|
653
|
+
warn(warning("build", `llmsPlugin: no listed pages under ${f.path}, so ${f.path}llms.txt isn't written (paths in ssqch.config.js)`));
|
|
654
|
+
return false;
|
|
655
|
+
});
|
|
656
|
+
const fileFor = (pageUrl) => written.find((f) => pageUrl.startsWith(f.path))?.path ?? "/";
|
|
657
|
+
const parentOf = (inner) => written.find((f) => f.path !== inner && inner.startsWith(f.path))?.path ?? "/";
|
|
658
|
+
const nameOf = (f) => f.title ?? folderName(f.path.split("/").filter(Boolean).at(-1) ?? "");
|
|
659
|
+
const link = (title2, href, note) => `- [${title2.replace(/([[\]])/g, "\\$1")}](${href})${note ? `: ${note}` : ""}
|
|
125
660
|
`;
|
|
661
|
+
for (const file of [{ path: "/" }, ...written]) {
|
|
662
|
+
const items = [];
|
|
663
|
+
const listedFiles = /* @__PURE__ */ new Set();
|
|
664
|
+
for (const e of entries) {
|
|
665
|
+
let at = fileFor(e.url);
|
|
666
|
+
if (at === file.path) {
|
|
667
|
+
const name2 = e.section ?? sectionOf(`/${e.url.slice(file.path.length)}`);
|
|
668
|
+
items.push({ section: name2, line: link(e.title, whole(copies ? e.copy : e.url), e.description), short: link(e.title, whole(copies ? e.copy : e.url), "") });
|
|
669
|
+
continue;
|
|
670
|
+
}
|
|
671
|
+
if (file.path !== "/" && !at.startsWith(file.path)) continue;
|
|
672
|
+
while (at !== "/" && parentOf(at) !== file.path) at = parentOf(at);
|
|
673
|
+
if (at === "/" || listedFiles.has(at)) continue;
|
|
674
|
+
listedFiles.add(at);
|
|
675
|
+
const inner = written.find((f) => f.path === at);
|
|
676
|
+
const name = e.section ?? sectionOf(`/${e.url.slice(file.path.length)}`);
|
|
677
|
+
items.push({ section: name, line: link(nameOf(inner), whole(`${inner.path}llms.txt`), oneLine(inner.description)), short: link(nameOf(inner), whole(`${inner.path}llms.txt`), "") });
|
|
678
|
+
}
|
|
679
|
+
let top = head;
|
|
680
|
+
if (file.path !== "/") {
|
|
681
|
+
top = `# ${file.title ?? `${siteTitle}: ${nameOf(file)}`}
|
|
682
|
+
`;
|
|
683
|
+
if (file.description) top += `
|
|
684
|
+
> ${oneLine(file.description)}
|
|
685
|
+
`;
|
|
686
|
+
if (file.details?.trim()) top += `
|
|
687
|
+
${file.details.trim()}
|
|
688
|
+
`;
|
|
689
|
+
}
|
|
690
|
+
files.push({ path: `${file.path}llms.txt`, contents: listing(top, items, budget, `${file.path}llms.txt`) });
|
|
126
691
|
}
|
|
127
|
-
files.push({ path: "/llms.txt", contents: txt });
|
|
128
692
|
if (copies) for (const e of entries) files.push({ path: e.copy, contents: e.markdown });
|
|
129
|
-
if (full) files.push({ path: "/llms-full.txt", contents:
|
|
130
|
-
|
|
693
|
+
if (full) files.push({ path: "/llms-full.txt", contents: `${head}
|
|
694
|
+
${entries.map((e) => `---
|
|
695
|
+
url: ${whole(e.url)}
|
|
131
696
|
---
|
|
132
697
|
|
|
133
|
-
${e.markdown}`).join("\n") });
|
|
698
|
+
${e.markdown}`).join("\n")}` });
|
|
134
699
|
return files;
|
|
135
700
|
}]);
|
|
136
701
|
});
|