steerdown 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +24 -2
- package/dist/anchors.js +9 -0
- package/dist/build.js +266 -0
- package/dist/cli.js +319 -0
- package/dist/concepts.js +174 -0
- package/dist/embeddings.js +92 -0
- package/dist/emit.js +174 -0
- package/dist/errors.js +8 -0
- package/dist/graph.js +45 -0
- package/dist/http.js +70 -0
- package/dist/jev.js +6 -0
- package/dist/judge.js +29 -0
- package/dist/origins.js +90 -0
- package/dist/pick.js +32 -0
- package/dist/rank.js +92 -0
- package/dist/scores.js +177 -0
- package/dist/sections.js +93 -0
- package/dist/similarity.js +37 -0
- package/dist/skill.js +45 -0
- package/dist/split.js +399 -0
- package/dist/steerdown.js +7 -0
- package/dist/urls.js +16 -0
- package/package.json +44 -3
- package/skills/steerdown-concepts/SKILL.md +44 -0
package/dist/concepts.js
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { ReportedError } from "./errors.js";
|
|
4
|
+
import { readGraph } from "./graph.js";
|
|
5
|
+
import { conceptHash, coveredCounts, mentionedScore, overBroadConcepts, uncoveredSections, } from "./scores.js";
|
|
6
|
+
const conceptFields = ["id", "label", "definition"];
|
|
7
|
+
/** Lowercase letters and digits, in words joined by single hyphens. */
|
|
8
|
+
const conceptIdPattern = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
9
|
+
const sentenceSegmenter = new Intl.Segmenter("en", { granularity: "sentence" });
|
|
10
|
+
/** The hand-edited concept file, next to the graph file. */
|
|
11
|
+
export function conceptFilePath(graphPath) {
|
|
12
|
+
return join(dirname(graphPath), "steerdown.concepts.json");
|
|
13
|
+
}
|
|
14
|
+
/** The file proposed concepts wait in until a person moves them into the concept file. */
|
|
15
|
+
export function proposalFilePath(graphPath) {
|
|
16
|
+
return join(dirname(graphPath), "steerdown.proposals.json");
|
|
17
|
+
}
|
|
18
|
+
function readJson(path) {
|
|
19
|
+
try {
|
|
20
|
+
return JSON.parse(readFileSync(path, "utf8"));
|
|
21
|
+
}
|
|
22
|
+
catch (error) {
|
|
23
|
+
throw new ReportedError(`cannot read ${path}: ${error.message}`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function sentenceCount(text) {
|
|
27
|
+
const sentences = [...sentenceSegmenter.segment(text)];
|
|
28
|
+
return sentences.filter(({ segment }) => segment.trim() !== "").length;
|
|
29
|
+
}
|
|
30
|
+
/** Missing, empty and unknown fields of one concept. */
|
|
31
|
+
function fieldProblems(fields) {
|
|
32
|
+
const problems = [];
|
|
33
|
+
for (const field of conceptFields) {
|
|
34
|
+
const value = fields[field];
|
|
35
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
36
|
+
problems.push(`needs a non-empty string "${field}"`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
for (const field of Object.keys(fields)) {
|
|
40
|
+
if (!conceptFields.includes(field)) {
|
|
41
|
+
problems.push(`has an unknown field "${field}"`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return problems;
|
|
45
|
+
}
|
|
46
|
+
/** What is wrong with one concept on its own, ignoring the other concepts. */
|
|
47
|
+
function itemProblems(item) {
|
|
48
|
+
const isObject = typeof item === "object" && item !== null && !Array.isArray(item);
|
|
49
|
+
if (!isObject) {
|
|
50
|
+
return ["is not an object"];
|
|
51
|
+
}
|
|
52
|
+
const problems = fieldProblems(item);
|
|
53
|
+
if (problems.length > 0) {
|
|
54
|
+
return problems;
|
|
55
|
+
}
|
|
56
|
+
const concept = item;
|
|
57
|
+
if (!conceptIdPattern.test(concept.id)) {
|
|
58
|
+
problems.push(`has the id "${concept.id}", which is not lowercase words joined by hyphens`);
|
|
59
|
+
}
|
|
60
|
+
const sentences = sentenceCount(concept.definition);
|
|
61
|
+
if (sentences !== 1) {
|
|
62
|
+
problems.push(`has a definition of ${sentences} sentences instead of one`);
|
|
63
|
+
}
|
|
64
|
+
return problems;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Everything wrong with a list of concepts. `takenIds` maps ids the list must not reuse to where
|
|
68
|
+
* they are already used.
|
|
69
|
+
*/
|
|
70
|
+
function conceptProblems(value, takenIds, itemName) {
|
|
71
|
+
if (!Array.isArray(value)) {
|
|
72
|
+
return ["is not a JSON array of concepts"];
|
|
73
|
+
}
|
|
74
|
+
const problems = [];
|
|
75
|
+
const usedIds = new Map(takenIds);
|
|
76
|
+
value.forEach((item, index) => {
|
|
77
|
+
const name = `${itemName} ${index + 1}`;
|
|
78
|
+
for (const problem of itemProblems(item)) {
|
|
79
|
+
problems.push(`${name} ${problem}`);
|
|
80
|
+
}
|
|
81
|
+
const id = item?.id;
|
|
82
|
+
if (typeof id !== "string") {
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
const usedBy = usedIds.get(id);
|
|
86
|
+
if (usedBy === undefined) {
|
|
87
|
+
usedIds.set(id, `${itemName} ${index + 1}`);
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
problems.push(`${name} has the id "${id}", already used by ${usedBy}`);
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
return problems;
|
|
94
|
+
}
|
|
95
|
+
/** Reads a list of concepts, or throws an error naming every problem in it. */
|
|
96
|
+
function readConceptList(path, takenIds, itemName) {
|
|
97
|
+
const value = readJson(path);
|
|
98
|
+
const problems = conceptProblems(value, takenIds, itemName);
|
|
99
|
+
if (problems.length > 0) {
|
|
100
|
+
throw new ReportedError(`${path} has problems:\n ${problems.join("\n ")}`);
|
|
101
|
+
}
|
|
102
|
+
return value;
|
|
103
|
+
}
|
|
104
|
+
/** Reads the concept file, or returns undefined when there is none. */
|
|
105
|
+
export function readConceptFile(path) {
|
|
106
|
+
return existsSync(path) ? readConceptList(path, new Map(), "concept") : undefined;
|
|
107
|
+
}
|
|
108
|
+
/** The concept file's concepts and the proposals waiting for review, empty when a file is absent. */
|
|
109
|
+
function readConceptsAndProposals(graphPath) {
|
|
110
|
+
const conceptPath = conceptFilePath(graphPath);
|
|
111
|
+
const proposalPath = proposalFilePath(graphPath);
|
|
112
|
+
const concepts = readConceptFile(conceptPath) ?? [];
|
|
113
|
+
const proposed = existsSync(proposalPath)
|
|
114
|
+
? readConceptList(proposalPath, new Map(), "proposal")
|
|
115
|
+
: [];
|
|
116
|
+
return { conceptPath, proposalPath, concepts, proposed };
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Checks proposed concepts and adds them to the proposal file. Nothing is written when any
|
|
120
|
+
* proposal has a problem.
|
|
121
|
+
*/
|
|
122
|
+
export function proposeConcepts(graphPath, proposalsPath) {
|
|
123
|
+
const { conceptPath, proposalPath, concepts, proposed } = readConceptsAndProposals(graphPath);
|
|
124
|
+
const takenIds = new Map();
|
|
125
|
+
for (const concept of concepts) {
|
|
126
|
+
takenIds.set(concept.id, conceptPath);
|
|
127
|
+
}
|
|
128
|
+
for (const concept of proposed) {
|
|
129
|
+
takenIds.set(concept.id, proposalPath);
|
|
130
|
+
}
|
|
131
|
+
const added = readConceptList(proposalsPath, takenIds, "proposal");
|
|
132
|
+
writeFileSync(proposalPath, `${JSON.stringify([...proposed, ...added], null, 2)}\n`);
|
|
133
|
+
return { added: added.length, proposalPath };
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* What an agent needs to propose concepts: the graph's sections with the scores that at least
|
|
137
|
+
* mention a concept, the concepts and proposals so far, the sections no concept covers well and
|
|
138
|
+
* the over-broad concepts.
|
|
139
|
+
*/
|
|
140
|
+
export function conceptInput(graphPath) {
|
|
141
|
+
const graph = readGraph(graphPath);
|
|
142
|
+
if (graph === undefined) {
|
|
143
|
+
throw new ReportedError(`no graph file at ${graphPath}. Run steerdown build first.`);
|
|
144
|
+
}
|
|
145
|
+
const { conceptPath, proposalPath, concepts, proposed } = readConceptsAndProposals(graphPath);
|
|
146
|
+
const scoredConcepts = graph.concepts ?? [];
|
|
147
|
+
const scores = graph.scores ?? [];
|
|
148
|
+
const sections = graph.sections.map((section) => ({
|
|
149
|
+
id: section.id,
|
|
150
|
+
page: section.page,
|
|
151
|
+
pageTitle: section.pageTitle,
|
|
152
|
+
heading: section.heading,
|
|
153
|
+
scores: {},
|
|
154
|
+
}));
|
|
155
|
+
for (const [section, concept, score] of scores) {
|
|
156
|
+
if (score >= mentionedScore) {
|
|
157
|
+
sections[section].scores[scoredConcepts[concept].id] = score;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
const counts = coveredCounts(scoredConcepts, scores);
|
|
161
|
+
const countsByHash = new Map(scoredConcepts.map((concept, index) => [concept.hash, counts[index]]));
|
|
162
|
+
return {
|
|
163
|
+
conceptFile: conceptPath,
|
|
164
|
+
proposalFile: proposalPath,
|
|
165
|
+
sections,
|
|
166
|
+
concepts: concepts.map((concept) => ({
|
|
167
|
+
...concept,
|
|
168
|
+
coveredSections: countsByHash.get(conceptHash(concept)) ?? null,
|
|
169
|
+
})),
|
|
170
|
+
proposed,
|
|
171
|
+
uncovered: uncoveredSections(sections.length, scores).map((index) => sections[index].id),
|
|
172
|
+
overBroad: overBroadConcepts(sections.length, scoredConcepts, scores),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { inBatches, postJson, runPooled } from "./http.js";
|
|
4
|
+
import { hash, requestsInFlight } from "./judge.js";
|
|
5
|
+
const textsPerRequest = 64;
|
|
6
|
+
/** The vector cache, next to the graph file. */
|
|
7
|
+
export function embeddingCachePath(graphPath) {
|
|
8
|
+
return join(dirname(graphPath), "steerdown.embeddings.json");
|
|
9
|
+
}
|
|
10
|
+
export function textHash(text) {
|
|
11
|
+
return hash(text);
|
|
12
|
+
}
|
|
13
|
+
function normalized(values) {
|
|
14
|
+
const vector = Float32Array.from(values);
|
|
15
|
+
let squaredLength = 0;
|
|
16
|
+
for (const value of vector) {
|
|
17
|
+
squaredLength += value * value;
|
|
18
|
+
}
|
|
19
|
+
const length = Math.sqrt(squaredLength) || 1;
|
|
20
|
+
return vector.map((value) => value / length);
|
|
21
|
+
}
|
|
22
|
+
function decodeVector(encoded) {
|
|
23
|
+
const bytes = Buffer.from(encoded, "base64");
|
|
24
|
+
const copy = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
25
|
+
return new Float32Array(copy);
|
|
26
|
+
}
|
|
27
|
+
function encodeVector(vector) {
|
|
28
|
+
return Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength).toString("base64");
|
|
29
|
+
}
|
|
30
|
+
/** Reads the cached vectors of a model. Vectors of another model are not comparable, so none are kept. */
|
|
31
|
+
function readCache(path, model) {
|
|
32
|
+
const vectors = new Map();
|
|
33
|
+
if (!existsSync(path)) {
|
|
34
|
+
return vectors;
|
|
35
|
+
}
|
|
36
|
+
const cache = JSON.parse(readFileSync(path, "utf8"));
|
|
37
|
+
if (cache.model !== model) {
|
|
38
|
+
return vectors;
|
|
39
|
+
}
|
|
40
|
+
for (const [key, encoded] of Object.entries(cache.vectors)) {
|
|
41
|
+
vectors.set(key, decodeVector(encoded));
|
|
42
|
+
}
|
|
43
|
+
return vectors;
|
|
44
|
+
}
|
|
45
|
+
/** Writes the vectors of the given text hashes that are known, dropping any the build no longer uses. */
|
|
46
|
+
function writeCache(path, client, known) {
|
|
47
|
+
const cache = { model: client.model, vectors: {} };
|
|
48
|
+
for (const key of known.keys) {
|
|
49
|
+
const vector = known.vectors.get(key);
|
|
50
|
+
if (vector !== undefined) {
|
|
51
|
+
cache.vectors[key] = encodeVector(vector);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
writeFileSync(path, `${JSON.stringify(cache)}\n`);
|
|
55
|
+
}
|
|
56
|
+
/** Asks the endpoint for one batch of texts and stores their vectors. */
|
|
57
|
+
async function embedBatch(client, batch, vectors) {
|
|
58
|
+
const response = await postJson({
|
|
59
|
+
url: `${client.url.replace(/\/+$/, "")}/embeddings`,
|
|
60
|
+
apiKey: client.apiKey,
|
|
61
|
+
body: { model: client.model, input: batch },
|
|
62
|
+
service: "The embedding endpoint",
|
|
63
|
+
});
|
|
64
|
+
if (response.data?.length !== batch.length) {
|
|
65
|
+
throw new Error(`The embedding endpoint returned ${response.data?.length ?? 0} vectors for ${batch.length} texts.`);
|
|
66
|
+
}
|
|
67
|
+
for (const { index, embedding } of response.data) {
|
|
68
|
+
vectors.set(textHash(batch[index]), normalized(embedding));
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Returns a vector for every text, embedding only texts the cache lacks, and rewrites the cache.
|
|
73
|
+
* When a request fails, the vectors so far are cached and the failure is returned.
|
|
74
|
+
*/
|
|
75
|
+
export async function embedTexts({ client, texts, cachePath, }) {
|
|
76
|
+
const vectors = readCache(cachePath, client.model);
|
|
77
|
+
const unique = [...new Map(texts.map((text) => [textHash(text), text])).values()];
|
|
78
|
+
const missing = unique.filter((text) => !vectors.has(textHash(text)));
|
|
79
|
+
const progress = { requests: 0, embedded: 0, failure: undefined };
|
|
80
|
+
try {
|
|
81
|
+
await runPooled(inBatches(missing, textsPerRequest), requestsInFlight, async (batch) => {
|
|
82
|
+
await embedBatch(client, batch, vectors);
|
|
83
|
+
progress.requests++;
|
|
84
|
+
progress.embedded += batch.length;
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
progress.failure = error;
|
|
89
|
+
}
|
|
90
|
+
writeCache(cachePath, client, { vectors, keys: unique.map(textHash) });
|
|
91
|
+
return { vectors, progress };
|
|
92
|
+
}
|
package/dist/emit.js
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { getHtmlIdAttrs } from "./anchors.js";
|
|
4
|
+
import { ReportedError } from "./errors.js";
|
|
5
|
+
import { readGraph } from "./graph.js";
|
|
6
|
+
import { candidatesBySection } from "./rank.js";
|
|
7
|
+
import { coveredScore } from "./scores.js";
|
|
8
|
+
import { dataFilePath, pageUrl } from "./urls.js";
|
|
9
|
+
/** Reads a graph page's built HTML, or returns undefined when the site no longer has it. */
|
|
10
|
+
function readBuiltPage(htmlDirectory, page) {
|
|
11
|
+
const htmlPath = join(htmlDirectory, page);
|
|
12
|
+
if (!existsSync(htmlPath)) {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
return {
|
|
16
|
+
url: pageUrl(page),
|
|
17
|
+
ids: getHtmlIdAttrs(readFileSync(htmlPath, "utf8")),
|
|
18
|
+
dataFile: dataFilePath(page),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
/** The heading id after the `#` in a section id, or undefined for a page's intro. */
|
|
22
|
+
function headingId(section) {
|
|
23
|
+
return section.id === section.page ? undefined : section.id.slice(`${section.page}#`.length);
|
|
24
|
+
}
|
|
25
|
+
/** Each section's scores from "Covers part of it" up, or undefined when the graph has no concepts. */
|
|
26
|
+
function coveredScoresBySection(graph) {
|
|
27
|
+
if (graph.concepts === undefined || graph.scores === undefined) {
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
const bySection = graph.sections.map(() => []);
|
|
31
|
+
for (const [section, concept, score] of graph.scores) {
|
|
32
|
+
if (score >= coveredScore) {
|
|
33
|
+
bySection[section].push([concept, score]);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return bySection;
|
|
37
|
+
}
|
|
38
|
+
/** Builds one page's file, adding targets and concepts as its links and scores refer to them. */
|
|
39
|
+
class PageFileBuilder {
|
|
40
|
+
context;
|
|
41
|
+
file = { sections: [], targets: [] };
|
|
42
|
+
targetIndexes = new Map();
|
|
43
|
+
conceptIndexes = new Map();
|
|
44
|
+
constructor(context) {
|
|
45
|
+
this.context = context;
|
|
46
|
+
if (context.scores !== undefined) {
|
|
47
|
+
this.file.concepts = [];
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** The section's scores with concept indexes into this file, or undefined without concepts. */
|
|
51
|
+
scoresOf(section) {
|
|
52
|
+
const scores = this.context.scores?.[section];
|
|
53
|
+
if (scores === undefined) {
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
return scores.map(([concept, score]) => [this.conceptIndex(concept), score]);
|
|
57
|
+
}
|
|
58
|
+
conceptIndex(concept) {
|
|
59
|
+
let index = this.conceptIndexes.get(concept);
|
|
60
|
+
if (index === undefined) {
|
|
61
|
+
const { id, label } = this.context.graph.concepts[concept];
|
|
62
|
+
index = this.file.concepts.push([id, label]) - 1;
|
|
63
|
+
this.conceptIndexes.set(concept, index);
|
|
64
|
+
}
|
|
65
|
+
return index;
|
|
66
|
+
}
|
|
67
|
+
/** The target's index in this file, or undefined when its built HTML gives it no anchor. */
|
|
68
|
+
targetIndex(to) {
|
|
69
|
+
const known = this.targetIndexes.get(to);
|
|
70
|
+
if (known !== undefined) {
|
|
71
|
+
return known;
|
|
72
|
+
}
|
|
73
|
+
const section = this.context.graph.sections[to];
|
|
74
|
+
const built = this.context.builtPages.get(section.page);
|
|
75
|
+
const anchor = headingId(section);
|
|
76
|
+
if (built === undefined) {
|
|
77
|
+
this.context.report.linksWithoutPage++;
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
if (anchor !== undefined && !built.ids.has(anchor)) {
|
|
81
|
+
this.context.report.linksWithoutId++;
|
|
82
|
+
return undefined;
|
|
83
|
+
}
|
|
84
|
+
const url = anchor === undefined ? built.url : `${built.url}#${anchor}`;
|
|
85
|
+
const row = [url, section.pageTitle, section.heading];
|
|
86
|
+
const scores = this.scoresOf(to);
|
|
87
|
+
const index = this.file.targets.push(scores === undefined ? row : [...row, scores]) - 1;
|
|
88
|
+
this.targetIndexes.set(to, index);
|
|
89
|
+
return index;
|
|
90
|
+
}
|
|
91
|
+
/** Adds a section with its showable links whose targets the built HTML has. */
|
|
92
|
+
addSection(from) {
|
|
93
|
+
const links = [];
|
|
94
|
+
const linkIndexByTarget = new Map();
|
|
95
|
+
for (const { to, relation, probability } of this.context.candidates[from]) {
|
|
96
|
+
const target = this.targetIndex(to);
|
|
97
|
+
if (target === undefined) {
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
const origin = this.context.origins.get(`${from}>${to}`);
|
|
101
|
+
linkIndexByTarget.set(to, links.length);
|
|
102
|
+
links.push(origin === undefined
|
|
103
|
+
? [target, relation, probability]
|
|
104
|
+
: [target, relation, probability, ...origin]);
|
|
105
|
+
}
|
|
106
|
+
const defaults = (this.context.graph.links[from] ?? [])
|
|
107
|
+
.map(([to]) => linkIndexByTarget.get(to))
|
|
108
|
+
.filter((index) => index !== undefined);
|
|
109
|
+
const section = { id: this.context.graph.sections[from].id, links, defaults };
|
|
110
|
+
const scores = this.scoresOf(from);
|
|
111
|
+
if (scores !== undefined) {
|
|
112
|
+
section.scores = scores;
|
|
113
|
+
}
|
|
114
|
+
this.file.sections.push(section);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/** The graph's sections grouped by page, each in graph order. */
|
|
118
|
+
function sectionsByPage(sections) {
|
|
119
|
+
const pages = new Map();
|
|
120
|
+
sections.forEach((section, index) => {
|
|
121
|
+
const list = pages.get(section.page) ?? [];
|
|
122
|
+
list.push(index);
|
|
123
|
+
pages.set(section.page, list);
|
|
124
|
+
});
|
|
125
|
+
return pages;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Writes one data file per graph page that has built HTML, under the `steerdown` folder of the
|
|
129
|
+
* built site. Each holds the page's sections with their showable links, the build's default
|
|
130
|
+
* links, every target's URL, title and heading and, with concepts, the scores that pin links.
|
|
131
|
+
*/
|
|
132
|
+
export function emit({ graphPath, htmlDirectory, }) {
|
|
133
|
+
const graph = readGraph(graphPath);
|
|
134
|
+
if (graph === undefined) {
|
|
135
|
+
throw new ReportedError(`no graph file at ${graphPath}. Run steerdown build first.`);
|
|
136
|
+
}
|
|
137
|
+
const pages = sectionsByPage(graph.sections);
|
|
138
|
+
const report = {
|
|
139
|
+
written: 0,
|
|
140
|
+
bytes: 0,
|
|
141
|
+
leftOut: [],
|
|
142
|
+
linksWithoutPage: 0,
|
|
143
|
+
linksWithoutId: 0,
|
|
144
|
+
};
|
|
145
|
+
const context = {
|
|
146
|
+
graph,
|
|
147
|
+
builtPages: new Map([...pages.keys()].map((page) => [page, readBuiltPage(htmlDirectory, page)])),
|
|
148
|
+
candidates: candidatesBySection(graph.sections, graph.edges, graph.scopes ?? []),
|
|
149
|
+
origins: new Map((graph.origins ?? []).map(([from, to, sentence, sentenceHash]) => [
|
|
150
|
+
`${from}>${to}`,
|
|
151
|
+
[sentence, sentenceHash],
|
|
152
|
+
])),
|
|
153
|
+
scores: coveredScoresBySection(graph),
|
|
154
|
+
report,
|
|
155
|
+
};
|
|
156
|
+
for (const [page, sections] of pages) {
|
|
157
|
+
const built = context.builtPages.get(page);
|
|
158
|
+
if (built === undefined) {
|
|
159
|
+
report.leftOut.push({ page, reason: "no built HTML page" });
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
const builder = new PageFileBuilder(context);
|
|
163
|
+
for (const section of sections) {
|
|
164
|
+
builder.addSection(section);
|
|
165
|
+
}
|
|
166
|
+
const text = JSON.stringify(builder.file);
|
|
167
|
+
const path = join(htmlDirectory, built.dataFile);
|
|
168
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
169
|
+
writeFileSync(path, text);
|
|
170
|
+
report.written++;
|
|
171
|
+
report.bytes += Buffer.byteLength(text);
|
|
172
|
+
}
|
|
173
|
+
return report;
|
|
174
|
+
}
|
package/dist/errors.js
ADDED
package/dist/graph.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
export const graphVersion = 2;
|
|
3
|
+
export const relations = [
|
|
4
|
+
"prerequisite",
|
|
5
|
+
"next_step",
|
|
6
|
+
"detail",
|
|
7
|
+
"alternative",
|
|
8
|
+
"unrelated",
|
|
9
|
+
];
|
|
10
|
+
/** Reads a graph file written by this version, or returns undefined when there is none to reuse. */
|
|
11
|
+
export function readGraph(path) {
|
|
12
|
+
if (!existsSync(path)) {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
const graph = JSON.parse(readFileSync(path, "utf8"));
|
|
16
|
+
return graph.version === graphVersion ? graph : undefined;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Writes one section, skipped section, edge, section's links, concept,
|
|
20
|
+
* score or origin per line, so changes show up as small diffs.
|
|
21
|
+
*/
|
|
22
|
+
export function writeGraph(path, graph) {
|
|
23
|
+
const list = (items) => items.length === 0
|
|
24
|
+
? "[]"
|
|
25
|
+
: `[\n${items.map((item) => ` ${JSON.stringify(item)}`).join(",\n")}\n ]`;
|
|
26
|
+
const hasScopes = graph.scopes !== undefined && graph.scopes.length > 0;
|
|
27
|
+
const fields = [
|
|
28
|
+
` "version": ${graph.version}`,
|
|
29
|
+
` "model": ${JSON.stringify(graph.model)}`,
|
|
30
|
+
` "questions": ${JSON.stringify(graph.questions)}`,
|
|
31
|
+
` "relations": ${JSON.stringify(graph.relations)}`,
|
|
32
|
+
...(hasScopes ? [` "scopes": ${JSON.stringify(graph.scopes)}`] : []),
|
|
33
|
+
` "sections": ${list(graph.sections)}`,
|
|
34
|
+
` "skipped": ${list(graph.skipped)}`,
|
|
35
|
+
` "edges": ${list(graph.edges)}`,
|
|
36
|
+
` "links": ${list(graph.links)}`,
|
|
37
|
+
];
|
|
38
|
+
if (graph.concepts !== undefined && graph.scores !== undefined) {
|
|
39
|
+
fields.push(` "scoreQuestion": ${JSON.stringify(graph.scoreQuestion)}`, ` "concepts": ${list(graph.concepts)}`, ` "scores": ${list(graph.scores)}`);
|
|
40
|
+
}
|
|
41
|
+
if (graph.embeddingModel !== undefined && graph.origins !== undefined) {
|
|
42
|
+
fields.push(` "embeddingModel": ${JSON.stringify(graph.embeddingModel)}`, ` "origins": ${list(graph.origins)}`);
|
|
43
|
+
}
|
|
44
|
+
writeFileSync(path, `{\n${fields.join(",\n")}\n}\n`);
|
|
45
|
+
}
|
package/dist/http.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
export class RequestError extends Error {
|
|
2
|
+
}
|
|
3
|
+
const retriedStatuses = new Set([429, 529]);
|
|
4
|
+
const maximumRetries = 7;
|
|
5
|
+
const firstRetryDelayMs = 1000;
|
|
6
|
+
const longestRetryDelayMs = 60_000;
|
|
7
|
+
function sleep(milliseconds) {
|
|
8
|
+
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
9
|
+
}
|
|
10
|
+
function retryDelay(attempt, retryAfter) {
|
|
11
|
+
const retryAfterSeconds = Number(retryAfter);
|
|
12
|
+
const serverGaveDelay = retryAfter !== null && retryAfter !== "" && Number.isFinite(retryAfterSeconds);
|
|
13
|
+
if (serverGaveDelay) {
|
|
14
|
+
return Math.max(0, retryAfterSeconds * 1000);
|
|
15
|
+
}
|
|
16
|
+
return Math.min(longestRetryDelayMs, firstRetryDelayMs * 2 ** attempt) + Math.random() * 500;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Posts a JSON body and returns the JSON answer, retrying rate-limited (429) and overloaded (529)
|
|
20
|
+
* responses. `service` names the server in the error a failed request throws.
|
|
21
|
+
*/
|
|
22
|
+
export async function postJson({ url, apiKey, body, service, }) {
|
|
23
|
+
const headers = { "content-type": "application/json" };
|
|
24
|
+
if (apiKey !== undefined) {
|
|
25
|
+
headers.authorization = `Bearer ${apiKey}`;
|
|
26
|
+
}
|
|
27
|
+
const send = () => fetch(url, { method: "POST", headers, body: JSON.stringify(body) });
|
|
28
|
+
let response = await send();
|
|
29
|
+
for (let retry = 0; retriedStatuses.has(response.status) && retry < maximumRetries; retry++) {
|
|
30
|
+
await response.body?.cancel();
|
|
31
|
+
await sleep(retryDelay(retry, response.headers.get("retry-after")));
|
|
32
|
+
response = await send();
|
|
33
|
+
}
|
|
34
|
+
if (!response.ok) {
|
|
35
|
+
const detail = (await response.text()).slice(0, 300);
|
|
36
|
+
throw new RequestError(`${service} returned status ${response.status}: ${detail}`);
|
|
37
|
+
}
|
|
38
|
+
return (await response.json());
|
|
39
|
+
}
|
|
40
|
+
/** Splits a list into consecutive batches of at most `size` items. */
|
|
41
|
+
export function inBatches(items, size) {
|
|
42
|
+
const batches = [];
|
|
43
|
+
for (let start = 0; start < items.length; start += size) {
|
|
44
|
+
batches.push(items.slice(start, start + size));
|
|
45
|
+
}
|
|
46
|
+
return batches;
|
|
47
|
+
}
|
|
48
|
+
/** Runs `work` over every item with at most `limit` running at once, stopping at the first error. */
|
|
49
|
+
export async function runPooled(items, limit, work) {
|
|
50
|
+
let nextIndex = 0;
|
|
51
|
+
const errors = [];
|
|
52
|
+
const hasWorkLeft = () => errors.length === 0 && nextIndex < items.length;
|
|
53
|
+
async function worker() {
|
|
54
|
+
while (hasWorkLeft()) {
|
|
55
|
+
const item = items[nextIndex];
|
|
56
|
+
nextIndex++;
|
|
57
|
+
try {
|
|
58
|
+
await work(item);
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
errors.push(error);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const workerCount = Math.min(limit, items.length);
|
|
66
|
+
await Promise.all(Array.from({ length: workerCount }, worker));
|
|
67
|
+
if (errors.length > 0) {
|
|
68
|
+
throw errors[0];
|
|
69
|
+
}
|
|
70
|
+
}
|
package/dist/jev.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { postJson } from "./http.js";
|
|
2
|
+
export const officialEndpoint = "https://api.typesafe.ai/v1/systemone";
|
|
3
|
+
/** Sends one System One request. */
|
|
4
|
+
export function askJev(client, body) {
|
|
5
|
+
return postJson({ url: client.endpoint, apiKey: client.apiKey, body, service: "Jev" });
|
|
6
|
+
}
|
package/dist/judge.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export const model = "jev-latest";
|
|
3
|
+
export const requestsInFlight = 4;
|
|
4
|
+
export const judgedWordLimit = 600;
|
|
5
|
+
/** Probabilities, scores and confidences are stored to three decimals. */
|
|
6
|
+
const storedPrecision = 1000;
|
|
7
|
+
export function hash(value) {
|
|
8
|
+
return createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16);
|
|
9
|
+
}
|
|
10
|
+
export function round(value) {
|
|
11
|
+
return Math.round(value * storedPrecision) / storedPrecision;
|
|
12
|
+
}
|
|
13
|
+
/** The text up to its word limit, keeping its line breaks. */
|
|
14
|
+
function truncated(text) {
|
|
15
|
+
const words = [...text.matchAll(/\S+/g)];
|
|
16
|
+
const lastWord = words[judgedWordLimit - 1];
|
|
17
|
+
if (words.length <= judgedWordLimit || lastWord === undefined) {
|
|
18
|
+
return text;
|
|
19
|
+
}
|
|
20
|
+
return text.slice(0, lastWord.index + lastWord[0].length);
|
|
21
|
+
}
|
|
22
|
+
/** The part of a section Jev reads. Its hash tells a rebuild whether the section changed. */
|
|
23
|
+
export function judgedState(section) {
|
|
24
|
+
return {
|
|
25
|
+
page_title: section.pageTitle,
|
|
26
|
+
heading: section.heading,
|
|
27
|
+
text: truncated(section.text),
|
|
28
|
+
};
|
|
29
|
+
}
|