steerdown 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +24 -2
- package/dist/anchors.js +9 -0
- package/dist/build.js +266 -0
- package/dist/cli.js +319 -0
- package/dist/concepts.js +174 -0
- package/dist/embeddings.js +92 -0
- package/dist/emit.js +174 -0
- package/dist/errors.js +8 -0
- package/dist/graph.js +45 -0
- package/dist/http.js +70 -0
- package/dist/jev.js +6 -0
- package/dist/judge.js +29 -0
- package/dist/origins.js +90 -0
- package/dist/pick.js +32 -0
- package/dist/rank.js +92 -0
- package/dist/scores.js +177 -0
- package/dist/sections.js +93 -0
- package/dist/similarity.js +37 -0
- package/dist/skill.js +45 -0
- package/dist/split.js +399 -0
- package/dist/steerdown.js +7 -0
- package/dist/urls.js +16 -0
- package/package.json +44 -3
- package/skills/steerdown-concepts/SKILL.md +44 -0
package/dist/origins.js
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { embeddingCachePath, embedTexts, textHash } from "./embeddings.js";
|
|
2
|
+
import { hash, judgedState, round } from "./judge.js";
|
|
3
|
+
import { candidatesBySection } from "./rank.js";
|
|
4
|
+
import { sentenceKey, splitSentences } from "./split.js";
|
|
5
|
+
/** What a link target is compared by: the page title, heading and the text Jev reads. */
|
|
6
|
+
function targetText(section) {
|
|
7
|
+
const { page_title, heading, text } = judgedState(section);
|
|
8
|
+
const title = heading === "" ? page_title : `${page_title}: ${heading}`;
|
|
9
|
+
return `${title}\n\n${text}`;
|
|
10
|
+
}
|
|
11
|
+
function dot(first, second) {
|
|
12
|
+
let sum = 0;
|
|
13
|
+
for (let index = 0; index < first.length; index++) {
|
|
14
|
+
sum += first[index] * second[index];
|
|
15
|
+
}
|
|
16
|
+
return sum;
|
|
17
|
+
}
|
|
18
|
+
/** The position and similarity of the sentence nearest to the target. */
|
|
19
|
+
function nearestSentence(sentences, target) {
|
|
20
|
+
let best = { position: 0, similarity: -Infinity };
|
|
21
|
+
sentences.forEach((sentence, position) => {
|
|
22
|
+
const similarity = dot(sentence, target);
|
|
23
|
+
if (similarity > best.similarity) {
|
|
24
|
+
best = { position, similarity };
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
return best;
|
|
28
|
+
}
|
|
29
|
+
/** For every showable link, the source sentence nearest to the target, unless it is below the minimum. */
|
|
30
|
+
function pickOrigins({ sentencesBySection, targets, links, vectors, minSimilarity, }) {
|
|
31
|
+
const vectorOf = (text) => vectors.get(textHash(text));
|
|
32
|
+
const origins = [];
|
|
33
|
+
let belowMinimum = 0;
|
|
34
|
+
links.forEach((candidates, from) => {
|
|
35
|
+
const sentences = sentencesBySection[from];
|
|
36
|
+
if (sentences.length === 0) {
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const sentenceVectors = sentences.map(vectorOf);
|
|
40
|
+
for (const { to } of candidates) {
|
|
41
|
+
const { position, similarity } = nearestSentence(sentenceVectors, vectorOf(targets[to]));
|
|
42
|
+
if (minSimilarity !== undefined && similarity < minSimilarity) {
|
|
43
|
+
belowMinimum++;
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
origins.push([
|
|
47
|
+
from,
|
|
48
|
+
to,
|
|
49
|
+
position,
|
|
50
|
+
hash(sentenceKey(sentences[position])),
|
|
51
|
+
round(similarity),
|
|
52
|
+
]);
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
return { origins, belowMinimum };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Finds the sentence each showable link grows out of: the sentence of the source section whose
|
|
59
|
+
* embedding is nearest to the target's. Without an embedding client it finds none. A link whose
|
|
60
|
+
* nearest sentence is below `minSimilarity` gets none.
|
|
61
|
+
*/
|
|
62
|
+
export async function findOrigins({ client, sections, graphSections, edges, scopes, graphPath, minSimilarity, }) {
|
|
63
|
+
if (client === undefined) {
|
|
64
|
+
return { fields: {}, report: undefined, failure: undefined };
|
|
65
|
+
}
|
|
66
|
+
const links = candidatesBySection(graphSections, edges, scopes);
|
|
67
|
+
const sentencesBySection = sections.map((section) => splitSentences(section.blocks));
|
|
68
|
+
const targets = sections.map(targetText);
|
|
69
|
+
const neededTargets = new Set(links.flat().map(({ to }) => to));
|
|
70
|
+
const texts = [
|
|
71
|
+
...links.flatMap((candidates, from) => candidates.length > 0 ? sentencesBySection[from] : []),
|
|
72
|
+
...[...neededTargets].map((to) => targets[to]),
|
|
73
|
+
];
|
|
74
|
+
const { vectors, progress } = await embedTexts({
|
|
75
|
+
client,
|
|
76
|
+
texts,
|
|
77
|
+
cachePath: embeddingCachePath(graphPath),
|
|
78
|
+
});
|
|
79
|
+
const { requests, embedded, failure } = progress;
|
|
80
|
+
if (failure !== undefined) {
|
|
81
|
+
return { fields: {}, report: { links: 0, belowMinimum: 0, embedded, requests }, failure };
|
|
82
|
+
}
|
|
83
|
+
const picked = pickOrigins({ sentencesBySection, targets, links, vectors, minSimilarity });
|
|
84
|
+
const { origins, belowMinimum } = picked;
|
|
85
|
+
return {
|
|
86
|
+
fields: { embeddingModel: client.model, origins },
|
|
87
|
+
report: { links: origins.length, belowMinimum, embedded, requests },
|
|
88
|
+
failure: undefined,
|
|
89
|
+
};
|
|
90
|
+
}
|
package/dist/pick.js
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The variety rules for a section's links, shared by the build and the browser script. Keep it
|
|
3
|
+
* free of Node and DOM imports.
|
|
4
|
+
*/
|
|
5
|
+
const linksPerSection = 3;
|
|
6
|
+
/** True when a candidate is not yet picked and adds neither a second prerequisite nor a page already linked. */
|
|
7
|
+
function fitsWithPicked(candidate, picked) {
|
|
8
|
+
const isPicked = picked.includes(candidate);
|
|
9
|
+
const isSecondPrerequisite = candidate.relation === "prerequisite" &&
|
|
10
|
+
picked.some((link) => link.relation === "prerequisite");
|
|
11
|
+
const sharesPage = picked.some((link) => link.page === candidate.page);
|
|
12
|
+
return !isPicked && !isSecondPrerequisite && !sharesPage;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Picks up to three links from candidates sorted best first: the best `next_step` whenever there
|
|
16
|
+
* is one, then the best rest, with at most one `prerequisite` and one link per page. Returns them
|
|
17
|
+
* in the candidates' order.
|
|
18
|
+
*/
|
|
19
|
+
export function pickLinks(candidates) {
|
|
20
|
+
const picked = [];
|
|
21
|
+
const nextStep = candidates.find((candidate) => candidate.relation === "next_step");
|
|
22
|
+
if (nextStep !== undefined) {
|
|
23
|
+
picked.push(nextStep);
|
|
24
|
+
}
|
|
25
|
+
for (const candidate of candidates) {
|
|
26
|
+
const hasRoom = picked.length < linksPerSection;
|
|
27
|
+
if (hasRoom && fitsWithPicked(candidate, picked)) {
|
|
28
|
+
picked.push(candidate);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return candidates.filter((candidate) => picked.includes(candidate));
|
|
32
|
+
}
|
package/dist/rank.js
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { relations } from "./graph.js";
|
|
2
|
+
import { pickLinks } from "./pick.js";
|
|
3
|
+
/**
|
|
4
|
+
* Labels below this probability are left out.
|
|
5
|
+
* Above it, independent runs agree on nearly every label.
|
|
6
|
+
*/
|
|
7
|
+
const minimumProbability = 0.6;
|
|
8
|
+
/** Largest share of sections whose links may point to one target. */
|
|
9
|
+
const largestTargetShare = 0.1;
|
|
10
|
+
/** Sorts candidates by falling probability, then by target index. */
|
|
11
|
+
function mostProbableFirst(first, second) {
|
|
12
|
+
return second.probability - first.probability || first.to - second.to;
|
|
13
|
+
}
|
|
14
|
+
/** The scope a page belongs to: the longest scope path that is the page or a folder holding it. */
|
|
15
|
+
export function scopeOf(page, scopes) {
|
|
16
|
+
let found;
|
|
17
|
+
for (const path of scopes) {
|
|
18
|
+
const scope = path.replace(/\/+$/, "");
|
|
19
|
+
const holdsPage = page === scope || page.startsWith(`${scope}/`);
|
|
20
|
+
if (holdsPage && (found === undefined || scope.length > found.length)) {
|
|
21
|
+
found = scope;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return found;
|
|
25
|
+
}
|
|
26
|
+
/** Each section's judged links that may be shown, most probable first. */
|
|
27
|
+
export function candidatesBySection(sections, edges, scopes) {
|
|
28
|
+
const sectionScopes = sections.map((section) => scopeOf(section.page, scopes));
|
|
29
|
+
const candidates = sections.map(() => []);
|
|
30
|
+
for (const [from, to, relation, probabilities] of edges) {
|
|
31
|
+
const probability = probabilities[relations.indexOf(relation)];
|
|
32
|
+
const isShown = relation !== "unrelated" && probability !== undefined;
|
|
33
|
+
if (!isShown || probability < minimumProbability) {
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
const fromScope = sectionScopes[from];
|
|
37
|
+
const toScope = sectionScopes[to];
|
|
38
|
+
// Sections from two different scopes cover different products, so only another way to the
|
|
39
|
+
// same goal is a useful link between them.
|
|
40
|
+
const crossesScopes = fromScope !== undefined && toScope !== undefined && fromScope !== toScope;
|
|
41
|
+
if (crossesScopes && relation !== "alternative") {
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
candidates[from].push({ to, relation, probability, page: sections[to].page });
|
|
45
|
+
}
|
|
46
|
+
for (const list of candidates) {
|
|
47
|
+
list.sort(mostProbableFirst);
|
|
48
|
+
}
|
|
49
|
+
return candidates;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Finds targets in the links of more sections than the limit. Each stays with the sections that
|
|
53
|
+
* judged it most probable and is removed from the other sections' candidates.
|
|
54
|
+
* Returns the sections that lost a candidate and need to pick again.
|
|
55
|
+
*/
|
|
56
|
+
function dropTargetsOverLimit(picks, candidates, mostSectionsPerTarget) {
|
|
57
|
+
const holders = new Map();
|
|
58
|
+
picks.forEach((links, from) => {
|
|
59
|
+
for (const link of links) {
|
|
60
|
+
const list = holders.get(link.to) ?? [];
|
|
61
|
+
list.push({ from, probability: link.probability });
|
|
62
|
+
holders.set(link.to, list);
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
const losers = new Set();
|
|
66
|
+
for (const [to, list] of holders) {
|
|
67
|
+
if (list.length <= mostSectionsPerTarget) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
list.sort((first, second) => second.probability - first.probability || first.from - second.from);
|
|
71
|
+
for (const { from } of list.slice(mostSectionsPerTarget)) {
|
|
72
|
+
candidates[from] = candidates[from].filter((candidate) => candidate.to !== to);
|
|
73
|
+
losers.add(from);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return losers;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Ranks each section's links. A target may appear in the links of at most a tenth of the
|
|
80
|
+
* sections. Sections that lose a target pick again until no target is over the limit.
|
|
81
|
+
*/
|
|
82
|
+
export function rankLinks(sections, edges, scopes) {
|
|
83
|
+
const candidates = candidatesBySection(sections, edges, scopes);
|
|
84
|
+
const mostSectionsPerTarget = Math.max(1, Math.floor(largestTargetShare * sections.length));
|
|
85
|
+
let picks = candidates.map((list) => pickLinks(list));
|
|
86
|
+
let repick = dropTargetsOverLimit(picks, candidates, mostSectionsPerTarget);
|
|
87
|
+
while (repick.size > 0) {
|
|
88
|
+
picks = picks.map((links, from) => (repick.has(from) ? pickLinks(candidates[from]) : links));
|
|
89
|
+
repick = dropTargetsOverLimit(picks, candidates, mostSectionsPerTarget);
|
|
90
|
+
}
|
|
91
|
+
return picks.map((links) => links.map((link) => [link.to, link.relation]));
|
|
92
|
+
}
|
package/dist/scores.js
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { runPooled } from "./http.js";
|
|
2
|
+
import { askJev } from "./jev.js";
|
|
3
|
+
import { hash, judgedState, model, requestsInFlight, round } from "./judge.js";
|
|
4
|
+
/** The score question's levels, from 0 to 3. */
|
|
5
|
+
const scoreLevels = [
|
|
6
|
+
"Not about it at all.",
|
|
7
|
+
"Mentions it in passing.",
|
|
8
|
+
"Covers part of it.",
|
|
9
|
+
"Is mainly about it.",
|
|
10
|
+
];
|
|
11
|
+
/** The input step lists a section's scores from "Mentions it in passing" up. */
|
|
12
|
+
export const mentionedScore = 1;
|
|
13
|
+
/** A section is covered well by a concept that scores at least "Covers part of it". */
|
|
14
|
+
export const coveredScore = 2;
|
|
15
|
+
/** A concept that covers more than this share of all sections is over-broad. */
|
|
16
|
+
const overBroadShare = 0.1;
|
|
17
|
+
/** A concept that covers this many sections or fewer is never over-broad, however few there are. */
|
|
18
|
+
const fewestOverBroadSections = 5;
|
|
19
|
+
function scoreInstructions(concept) {
|
|
20
|
+
return `How much is this documentation section about ${concept.label}: ${concept.definition}`;
|
|
21
|
+
}
|
|
22
|
+
const scoreQuestionHash = hash([
|
|
23
|
+
model,
|
|
24
|
+
scoreInstructions({ label: "", definition: "" }),
|
|
25
|
+
scoreLevels,
|
|
26
|
+
]);
|
|
27
|
+
const noScoreRequests = {
|
|
28
|
+
requests: 0,
|
|
29
|
+
inputTokens: 0,
|
|
30
|
+
scored: 0,
|
|
31
|
+
failure: undefined,
|
|
32
|
+
};
|
|
33
|
+
/** The hash of what Jev reads about a concept, so a changed definition is scored again. */
|
|
34
|
+
export function conceptHash(concept) {
|
|
35
|
+
return hash([concept.label, concept.definition]);
|
|
36
|
+
}
|
|
37
|
+
function cellKey(sectionHash, conceptHash) {
|
|
38
|
+
return `${sectionHash}>${conceptHash}`;
|
|
39
|
+
}
|
|
40
|
+
/** Scores from an earlier graph, keyed by section and concept hashes, when its question matches. */
|
|
41
|
+
function reusableCells(previous) {
|
|
42
|
+
const cells = new Map();
|
|
43
|
+
const isReusable = previous?.scoreQuestion === scoreQuestionHash;
|
|
44
|
+
if (!isReusable || previous.concepts === undefined || previous.scores === undefined) {
|
|
45
|
+
return cells;
|
|
46
|
+
}
|
|
47
|
+
for (const [section, concept, score, confidence] of previous.scores) {
|
|
48
|
+
const sectionHash = previous.sections[section]?.hash;
|
|
49
|
+
const previousConcept = previous.concepts[concept];
|
|
50
|
+
if (sectionHash !== undefined && previousConcept !== undefined) {
|
|
51
|
+
cells.set(cellKey(sectionHash, previousConcept.hash), { score, confidence });
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return cells;
|
|
55
|
+
}
|
|
56
|
+
/** Plans one request per section that has concepts without a reusable score. */
|
|
57
|
+
export function planScores(concepts, hashes, previous) {
|
|
58
|
+
const graphConcepts = concepts.map((concept) => ({
|
|
59
|
+
id: concept.id,
|
|
60
|
+
label: concept.label,
|
|
61
|
+
hash: conceptHash(concept),
|
|
62
|
+
}));
|
|
63
|
+
const cells = reusableCells(previous);
|
|
64
|
+
const jobs = [];
|
|
65
|
+
hashes.forEach((sectionHash, section) => {
|
|
66
|
+
const missing = [];
|
|
67
|
+
graphConcepts.forEach((concept, index) => {
|
|
68
|
+
if (!cells.has(cellKey(sectionHash, concept.hash))) {
|
|
69
|
+
missing.push(index);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
if (missing.length > 0) {
|
|
73
|
+
jobs.push({ section, concepts: missing });
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
const toScore = jobs.reduce((count, job) => count + job.concepts.length, 0);
|
|
77
|
+
return { definitions: concepts, concepts: graphConcepts, jobs, cells, toScore };
|
|
78
|
+
}
|
|
79
|
+
/** Asks Jev to score each planned section against its concepts, stopping at the first failure. */
|
|
80
|
+
async function scoreSections({ plan, sections, hashes, client, }) {
|
|
81
|
+
const concepts = plan.definitions;
|
|
82
|
+
const progress = { requests: 0, inputTokens: 0, scored: 0, failure: undefined };
|
|
83
|
+
try {
|
|
84
|
+
await runPooled(plan.jobs, requestsInFlight, async (job) => {
|
|
85
|
+
const questions = {};
|
|
86
|
+
for (const index of job.concepts) {
|
|
87
|
+
const concept = concepts[index];
|
|
88
|
+
questions[concept.id] = {
|
|
89
|
+
type: "score",
|
|
90
|
+
instructions: scoreInstructions(concept),
|
|
91
|
+
criteria: scoreLevels,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
const state = judgedState(sections[job.section]);
|
|
95
|
+
const response = await askJev(client, { model, state, questions });
|
|
96
|
+
progress.requests++;
|
|
97
|
+
progress.inputTokens += response.usage.input_tokens;
|
|
98
|
+
for (const index of job.concepts) {
|
|
99
|
+
const answer = response.answers[concepts[index].id];
|
|
100
|
+
if (answer !== undefined) {
|
|
101
|
+
const key = cellKey(hashes[job.section], plan.concepts[index].hash);
|
|
102
|
+
plan.cells.set(key, { score: round(answer.score), confidence: round(answer.confidence) });
|
|
103
|
+
progress.scored++;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
progress.failure = error;
|
|
110
|
+
}
|
|
111
|
+
return progress;
|
|
112
|
+
}
|
|
113
|
+
/** The graph's score fields: every known score, in order of section and then concept. */
|
|
114
|
+
export function graphScores(plan, hashes) {
|
|
115
|
+
const scores = [];
|
|
116
|
+
hashes.forEach((sectionHash, section) => {
|
|
117
|
+
plan.concepts.forEach((concept, index) => {
|
|
118
|
+
const cell = plan.cells.get(cellKey(sectionHash, concept.hash));
|
|
119
|
+
if (cell !== undefined) {
|
|
120
|
+
scores.push([section, index, cell.score, cell.confidence]);
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
return { scoreQuestion: scoreQuestionHash, concepts: plan.concepts, scores };
|
|
125
|
+
}
|
|
126
|
+
/** How many sections each concept covers well, in the order of `concepts`. */
|
|
127
|
+
export function coveredCounts(concepts, scores) {
|
|
128
|
+
const counts = concepts.map(() => 0);
|
|
129
|
+
for (const [, concept, score] of scores) {
|
|
130
|
+
if (score >= coveredScore) {
|
|
131
|
+
counts[concept]++;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return counts;
|
|
135
|
+
}
|
|
136
|
+
/** Concepts that cover too many sections to tell them apart. */
|
|
137
|
+
export function overBroadConcepts(sectionCount, concepts, scores) {
|
|
138
|
+
const counts = coveredCounts(concepts, scores);
|
|
139
|
+
const overBroad = [];
|
|
140
|
+
concepts.forEach((concept, index) => {
|
|
141
|
+
const sections = counts[index];
|
|
142
|
+
if (sections > fewestOverBroadSections && sections > sectionCount * overBroadShare) {
|
|
143
|
+
overBroad.push({ id: concept.id, sections });
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
return overBroad;
|
|
147
|
+
}
|
|
148
|
+
/** The indexes of sections no concept covers well. */
|
|
149
|
+
export function uncoveredSections(sectionCount, scores) {
|
|
150
|
+
const covered = new Set();
|
|
151
|
+
for (const [section, , score] of scores) {
|
|
152
|
+
if (score >= coveredScore) {
|
|
153
|
+
covered.add(section);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return Array.from({ length: sectionCount }, (_, index) => index).filter((index) => !covered.has(index));
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Scores the sections against the concepts, unless there is no concept file or judging failed,
|
|
160
|
+
* and collects the results.
|
|
161
|
+
*/
|
|
162
|
+
export async function scoreConcepts({ plan, judgingFailed, sections, hashes, client, }) {
|
|
163
|
+
if (plan === undefined) {
|
|
164
|
+
return { fields: {}, progress: noScoreRequests, report: undefined };
|
|
165
|
+
}
|
|
166
|
+
const progress = judgingFailed
|
|
167
|
+
? noScoreRequests
|
|
168
|
+
: await scoreSections({ plan, sections, hashes, client });
|
|
169
|
+
const fields = graphScores(plan, hashes);
|
|
170
|
+
const total = sections.length * plan.concepts.length;
|
|
171
|
+
const overBroad = overBroadConcepts(sections.length, fields.concepts, fields.scores);
|
|
172
|
+
return {
|
|
173
|
+
fields,
|
|
174
|
+
progress,
|
|
175
|
+
report: { total, reused: total - plan.toScore, scored: progress.scored, overBroad },
|
|
176
|
+
};
|
|
177
|
+
}
|
package/dist/sections.js
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join, relative, sep } from "node:path";
|
|
3
|
+
import { parse } from "parse5";
|
|
4
|
+
import { documentTitle, firstElement, isLinkHub, judgedText, pagefindBodies, splitContent, } from "./split.js";
|
|
5
|
+
const parse5Adapter = {
|
|
6
|
+
name: (node) => ("tagName" in node ? node.tagName : undefined),
|
|
7
|
+
attribute: (node, name) => "attrs" in node
|
|
8
|
+
? node.attrs.find((attribute) => attribute.name === name && !attribute.namespace)?.value
|
|
9
|
+
: undefined,
|
|
10
|
+
children: (node) => ("childNodes" in node ? node.childNodes : []),
|
|
11
|
+
text: (node) => node.nodeName === "#text" ? node.value : undefined,
|
|
12
|
+
};
|
|
13
|
+
const ignoredDirectoryNames = new Set(["node_modules"]);
|
|
14
|
+
/** Every file under a directory, leaving out hidden entries and dependency folders. */
|
|
15
|
+
function listFiles(directory) {
|
|
16
|
+
const files = [];
|
|
17
|
+
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
18
|
+
const isIgnored = entry.name.startsWith(".") || ignoredDirectoryNames.has(entry.name);
|
|
19
|
+
if (isIgnored) {
|
|
20
|
+
continue;
|
|
21
|
+
}
|
|
22
|
+
const path = join(directory, entry.name);
|
|
23
|
+
if (entry.isDirectory()) {
|
|
24
|
+
for (const file of listFiles(path)) {
|
|
25
|
+
files.push(file);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
else if (entry.isFile()) {
|
|
29
|
+
files.push(path);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return files;
|
|
33
|
+
}
|
|
34
|
+
/** True when a page is an excluded file or lies inside an excluded folder. */
|
|
35
|
+
function isExcludedPage(page, excluded) {
|
|
36
|
+
const excludedPaths = excluded.map((path) => path.replace(/\/+$/, ""));
|
|
37
|
+
return excludedPaths.some((path) => page === path || page.startsWith(`${path}/`));
|
|
38
|
+
}
|
|
39
|
+
/** Lists the HTML pages of a built site as sorted paths relative to it, with forward slashes. */
|
|
40
|
+
export function findHtmlPages(directory, excluded) {
|
|
41
|
+
const pages = listFiles(directory).map((path) => relative(directory, path).split(sep).join("/"));
|
|
42
|
+
return pages.filter((page) => page.endsWith(".html") && !isExcludedPage(page, excluded)).sort();
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The elements that hold a page's content: its `data-pagefind-body` elements when the site marks
|
|
46
|
+
* any, otherwise `main`, then `body`.
|
|
47
|
+
*/
|
|
48
|
+
function contentRoots(parsed, siteMarksBodies) {
|
|
49
|
+
if (siteMarksBodies) {
|
|
50
|
+
return parsed.bodies;
|
|
51
|
+
}
|
|
52
|
+
const root = firstElement(parsed.document, parse5Adapter, "main") ??
|
|
53
|
+
firstElement(parsed.document, parse5Adapter, "body");
|
|
54
|
+
return root === undefined ? [] : [root];
|
|
55
|
+
}
|
|
56
|
+
/** Splits one parsed page into sections, leaving out link hubs. */
|
|
57
|
+
function readPage(parsed, roots, result) {
|
|
58
|
+
const { page, document } = parsed;
|
|
59
|
+
const split = splitContent(roots, parse5Adapter);
|
|
60
|
+
const pageTitle = split.title || documentTitle(document, parse5Adapter) || page;
|
|
61
|
+
for (const heading of split.idlessHeadings) {
|
|
62
|
+
result.idlessHeadings.push({ page, heading });
|
|
63
|
+
}
|
|
64
|
+
for (const { anchor, heading, blocks } of split.sections) {
|
|
65
|
+
const id = anchor === "" ? page : `${page}#${anchor}`;
|
|
66
|
+
if (isLinkHub(blocks)) {
|
|
67
|
+
result.skipped.push({ id, reason: "mainly a list of links" });
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
result.sections.push({ id, page, pageTitle, heading, blocks, text: judgedText(blocks) });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Reads and splits every page, leaving out link hubs. When any page marks `data-pagefind-body`,
|
|
75
|
+
* pages without it are left out.
|
|
76
|
+
*/
|
|
77
|
+
export function readSections(directory, pages) {
|
|
78
|
+
const parsedPages = pages.map((page) => {
|
|
79
|
+
const document = parse(readFileSync(join(directory, page), "utf8"));
|
|
80
|
+
return { page, document, bodies: pagefindBodies(document, parse5Adapter) };
|
|
81
|
+
});
|
|
82
|
+
const siteMarksBodies = parsedPages.some((parsed) => parsed.bodies.length > 0);
|
|
83
|
+
const result = { sections: [], skipped: [], idlessHeadings: [] };
|
|
84
|
+
for (const parsed of parsedPages) {
|
|
85
|
+
const roots = contentRoots(parsed, siteMarksBodies);
|
|
86
|
+
if (roots.length === 0) {
|
|
87
|
+
result.skipped.push({ id: parsed.page, reason: "no data-pagefind-body element" });
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
readPage(parsed, roots, result);
|
|
91
|
+
}
|
|
92
|
+
return result;
|
|
93
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
const wordPattern = /[\p{L}\p{N}_]+/gu;
|
|
2
|
+
/** TF-IDF vectors of the given texts, normalized so a dot product is their cosine similarity. */
|
|
3
|
+
export function tfidfVectors(texts) {
|
|
4
|
+
const words = texts.map((text) => text.toLowerCase().match(wordPattern) ?? []);
|
|
5
|
+
const documentFrequency = new Map();
|
|
6
|
+
for (const textWords of words) {
|
|
7
|
+
for (const word of new Set(textWords)) {
|
|
8
|
+
documentFrequency.set(word, (documentFrequency.get(word) ?? 0) + 1);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
return words.map((textWords) => {
|
|
12
|
+
const vector = new Map();
|
|
13
|
+
for (const word of textWords) {
|
|
14
|
+
vector.set(word, (vector.get(word) ?? 0) + 1);
|
|
15
|
+
}
|
|
16
|
+
let squaredLength = 0;
|
|
17
|
+
for (const [word, count] of vector) {
|
|
18
|
+
const weight = count * Math.log(texts.length / documentFrequency.get(word));
|
|
19
|
+
vector.set(word, weight);
|
|
20
|
+
squaredLength += weight * weight;
|
|
21
|
+
}
|
|
22
|
+
const length = Math.sqrt(squaredLength) || 1;
|
|
23
|
+
for (const [word, weight] of vector) {
|
|
24
|
+
vector.set(word, weight / length);
|
|
25
|
+
}
|
|
26
|
+
return vector;
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
/** The cosine similarity of two normalized vectors. */
|
|
30
|
+
export function cosine(first, second) {
|
|
31
|
+
const [smaller, larger] = first.size < second.size ? [first, second] : [second, first];
|
|
32
|
+
let sum = 0;
|
|
33
|
+
for (const [word, weight] of smaller) {
|
|
34
|
+
sum += weight * (larger.get(word) ?? 0);
|
|
35
|
+
}
|
|
36
|
+
return sum;
|
|
37
|
+
}
|
package/dist/skill.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
import { ReportedError } from "./errors.js";
|
|
4
|
+
/** The concept skill folder that ships with this package. */
|
|
5
|
+
export const skillDirectory = fileURLToPath(new URL("../skills/steerdown-concepts", import.meta.url));
|
|
6
|
+
/** On Windows npx is `npx.cmd`, which Node only starts through a shell. */
|
|
7
|
+
const needsShell = process.platform === "win32";
|
|
8
|
+
/** Starts npx with one argument per array item, or on Windows as one command line for the shell. */
|
|
9
|
+
function startNpx(args) {
|
|
10
|
+
if (needsShell) {
|
|
11
|
+
// Node warns when an argument array is combined with a shell (DEP0190), so the line is built here.
|
|
12
|
+
// Windows paths cannot contain double quotes, so quoting each argument keeps spaces inside it.
|
|
13
|
+
const commandLine = ["npx", ...args.map((arg) => `"${arg}"`)].join(" ");
|
|
14
|
+
return spawn(commandLine, { stdio: "inherit", shell: true });
|
|
15
|
+
}
|
|
16
|
+
return spawn("npx", args, { stdio: "inherit" });
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Installs the concept skill into the current project with the skills CLI, whose output goes
|
|
20
|
+
* straight to the terminal. Throws with the skills CLI's exit code when it fails.
|
|
21
|
+
*/
|
|
22
|
+
export function installSkill() {
|
|
23
|
+
const child = startNpx(["--yes", "skills", "add", skillDirectory, "-y"]);
|
|
24
|
+
return new Promise((resolve, reject) => {
|
|
25
|
+
child.once("error", (error) => {
|
|
26
|
+
if (error.code === "ENOENT") {
|
|
27
|
+
reject(new ReportedError("npx was not found. skill install needs npx, which comes with npm."));
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
reject(error);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
child.once("close", (code) => {
|
|
34
|
+
if (code === 0) {
|
|
35
|
+
resolve();
|
|
36
|
+
}
|
|
37
|
+
else if (code === null) {
|
|
38
|
+
reject(new ReportedError("the skills CLI was stopped."));
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
reject(new ReportedError(`the skills CLI exited with code ${code}.`, code));
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
}
|