polite-crawler 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +533 -8
- package/build/cli-daemon--cTmq0Z5.mjs +38 -0
- package/build/cli-daemon--cTmq0Z5.mjs.map +1 -0
- package/build/cli.mjs +288 -20
- package/build/cli.mjs.map +1 -1
- package/build/{open-Ca1lcWR3.d.mts → daemon-DP4EqjFL.d.mts} +913 -16
- package/build/daemon-fZTE24P5.mjs +515 -0
- package/build/daemon-fZTE24P5.mjs.map +1 -0
- package/build/{diff-Dk2eSvi-.mjs → diff-NQcQqKe9.mjs} +2 -2
- package/build/{diff-Dk2eSvi-.mjs.map → diff-NQcQqKe9.mjs.map} +1 -1
- package/build/{extract-DcwNrTK7.mjs → extract-ByFfc8Zk.mjs} +2 -2
- package/build/{extract-DcwNrTK7.mjs.map → extract-ByFfc8Zk.mjs.map} +1 -1
- package/build/hours.d.mts +26 -0
- package/build/hours.mjs +48 -1
- package/build/hours.mjs.map +1 -1
- package/build/index.d.mts +491 -11
- package/build/index.mjs +898 -9
- package/build/index.mjs.map +1 -0
- package/build/loopback-C_QYowSf.mjs +22 -0
- package/build/loopback-C_QYowSf.mjs.map +1 -0
- package/build/mcp.d.mts +158 -2
- package/build/mcp.mjs +567 -18
- package/build/mcp.mjs.map +1 -1
- package/build/ollama-BJxkjny4.mjs +160 -0
- package/build/ollama-BJxkjny4.mjs.map +1 -0
- package/build/{open-DnrWgkC0.mjs → open-C4fhXJgD.mjs} +1404 -21
- package/build/open-C4fhXJgD.mjs.map +1 -0
- package/build/{runtime-k8mo1_7U.mjs → runtime-JToRO63o.mjs} +3 -3
- package/build/{runtime-k8mo1_7U.mjs.map → runtime-JToRO63o.mjs.map} +1 -1
- package/build/transformers-BQMlaGj4.mjs +133 -0
- package/build/transformers-BQMlaGj4.mjs.map +1 -0
- package/build/{duration-iIuMMofi.mjs → version-ntbY-6wh.mjs} +10 -10
- package/build/version-ntbY-6wh.mjs.map +1 -0
- package/build/{wacz-sfOLT8e9.mjs → wacz-BFNG42Kt.mjs} +3 -3
- package/build/{wacz-sfOLT8e9.mjs.map → wacz-BFNG42Kt.mjs.map} +1 -1
- package/build/{warc-VIPscWxY.mjs → warc-CVS0t1fW.mjs} +2 -2
- package/build/{warc-VIPscWxY.mjs.map → warc-CVS0t1fW.mjs.map} +1 -1
- package/package.json +10 -2
- package/build/duration-iIuMMofi.mjs.map +0 -1
- package/build/open-DnrWgkC0.mjs.map +0 -1
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
//#region src/ollama.ts
|
|
4
|
+
var ollama_exports = /* @__PURE__ */ __exportAll({
|
|
5
|
+
NO_OLLAMA: () => NO_OLLAMA,
|
|
6
|
+
OLLAMA_MODELS: () => OLLAMA_MODELS,
|
|
7
|
+
isLoopback: () => isLoopback,
|
|
8
|
+
ollamaEmbedder: () => ollamaEmbedder
|
|
9
|
+
});
|
|
10
|
+
/** These are the facts of three embedding models of Ollama. The name has no tag (`nomic-embed-text`, not `nomic-embed-text:latest`). */
|
|
11
|
+
const OLLAMA_MODELS = {
|
|
12
|
+
"nomic-embed-text": {
|
|
13
|
+
dimensions: 768,
|
|
14
|
+
maxInputTokens: 8192,
|
|
15
|
+
queryPrefix: "search_query: ",
|
|
16
|
+
documentPrefix: "search_document: "
|
|
17
|
+
},
|
|
18
|
+
"all-minilm": {
|
|
19
|
+
dimensions: 384,
|
|
20
|
+
maxInputTokens: 256
|
|
21
|
+
},
|
|
22
|
+
"mxbai-embed-large": {
|
|
23
|
+
dimensions: 1024,
|
|
24
|
+
maxInputTokens: 512,
|
|
25
|
+
queryPrefix: "Represent this sentence for searching relevant passages: "
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
/** The text of the error when no Ollama server answers. */
|
|
29
|
+
const NO_OLLAMA = "Ollama does not answer: install it from https://ollama.com, then start it with ollama serve";
|
|
30
|
+
/** This function tells if a host name is a loopback address: `localhost`, `127.0.0.0/8` or `::1`. */
|
|
31
|
+
const isLoopback = (host) => {
|
|
32
|
+
const name = host.replace(/^\[|\]$/g, "").toLowerCase();
|
|
33
|
+
return name === "localhost" || name.endsWith(".localhost") || name === "::1" || /^127(?:\.\d{1,3}){3}$/.test(name) || /^::ffff:127(?:\.\d{1,3}){3}$/.test(name);
|
|
34
|
+
};
|
|
35
|
+
/** This function gives the name of a model with no tag, for `OLLAMA_MODELS`. */
|
|
36
|
+
const baseName = (model) => {
|
|
37
|
+
const slash = model.lastIndexOf("/");
|
|
38
|
+
const colon = model.indexOf(":", slash + 1);
|
|
39
|
+
return colon === -1 ? model : model.slice(0, colon);
|
|
40
|
+
};
|
|
41
|
+
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
42
|
+
/** This function gives the code of a network error, also when it is the cause of the error of `fetch`. */
|
|
43
|
+
const codeOf = (error) => {
|
|
44
|
+
for (let at = error, depth = 0; at !== void 0 && at !== null && depth < 4; depth += 1) {
|
|
45
|
+
const code = at.code;
|
|
46
|
+
if (typeof code === "string") return code;
|
|
47
|
+
at = at.cause;
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
const NETWORK = /* @__PURE__ */ new Set([
|
|
51
|
+
"ECONNREFUSED",
|
|
52
|
+
"ECONNRESET",
|
|
53
|
+
"ENOTFOUND",
|
|
54
|
+
"EHOSTUNREACH",
|
|
55
|
+
"ENETUNREACH",
|
|
56
|
+
"EAI_AGAIN",
|
|
57
|
+
"UND_ERR_SOCKET",
|
|
58
|
+
"UND_ERR_CONNECT_TIMEOUT"
|
|
59
|
+
]);
|
|
60
|
+
/**
|
|
61
|
+
* This function makes an embedder that sends texts to Ollama (`POST {baseUrl}/api/embed`). It
|
|
62
|
+
* sends `batch` texts in each request, and it stops when the signal aborts. The errors tell what
|
|
63
|
+
* to do: `ollama serve` when no server answers, and `ollama pull <model>` when the model is not
|
|
64
|
+
* there. A `baseUrl` on a different computer throws a `RangeError`, unless `allowRemote` is true.
|
|
65
|
+
*/
|
|
66
|
+
const ollamaEmbedder = (options = {}) => {
|
|
67
|
+
const model = options.model ?? "nomic-embed-text";
|
|
68
|
+
if (model.trim() === "") throw new RangeError("the Ollama model must have a name");
|
|
69
|
+
const base = options.baseUrl ?? "http://127.0.0.1:11434";
|
|
70
|
+
let url;
|
|
71
|
+
try {
|
|
72
|
+
url = new URL(base);
|
|
73
|
+
} catch (cause) {
|
|
74
|
+
throw new RangeError(`the Ollama URL ${base} is not a URL`, { cause });
|
|
75
|
+
}
|
|
76
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") throw new RangeError(`the Ollama URL ${base} must be an http or https URL`);
|
|
77
|
+
if (url.username !== "" || url.password !== "") throw new RangeError("the Ollama URL must not hold a user name or a password");
|
|
78
|
+
if (!isLoopback(url.hostname) && options.allowRemote !== true) throw new RangeError(`the Ollama URL ${url.origin} is not on this computer, and the embedder sends the text of the stored pages to it: give allowRemote: true (--allow-remote-ollama) to permit it`);
|
|
79
|
+
const endpoint = new URL("api/embed", url.href.endsWith("/") ? url.href : `${url.href}/`).href;
|
|
80
|
+
const known = OLLAMA_MODELS[baseName(model)];
|
|
81
|
+
const batch = Math.max(1, Math.floor(options.batch ?? 32));
|
|
82
|
+
const timeoutMs = Math.max(1, Math.floor(options.timeoutMs ?? 12e4));
|
|
83
|
+
const queryPrefix = options.queryPrefix ?? known?.queryPrefix ?? "";
|
|
84
|
+
const documentPrefix = options.documentPrefix ?? known?.documentPrefix ?? "";
|
|
85
|
+
const custom = options.documentPrefix !== void 0 && options.documentPrefix !== (known?.documentPrefix ?? "") ? `#${createHash("sha256").update(options.documentPrefix).digest("hex").slice(0, 8)}` : "";
|
|
86
|
+
let dimensions = options.dimensions ?? known?.dimensions ?? 0;
|
|
87
|
+
const maxInputTokens = options.maxInputTokens ?? known?.maxInputTokens ?? 256;
|
|
88
|
+
const request = async (input, signal) => {
|
|
89
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
90
|
+
const both = signal === void 0 ? timeout : AbortSignal.any([signal, timeout]);
|
|
91
|
+
let response;
|
|
92
|
+
try {
|
|
93
|
+
response = await fetch(endpoint, {
|
|
94
|
+
method: "POST",
|
|
95
|
+
headers: { "content-type": "application/json" },
|
|
96
|
+
body: JSON.stringify({
|
|
97
|
+
model,
|
|
98
|
+
input,
|
|
99
|
+
truncate: true
|
|
100
|
+
}),
|
|
101
|
+
signal: both
|
|
102
|
+
});
|
|
103
|
+
} catch (error) {
|
|
104
|
+
if (signal?.aborted === true) throw signal.reason;
|
|
105
|
+
if (timeout.aborted) throw new Error(`Ollama at ${url.origin} did not answer in ${String(timeoutMs)} ms`, { cause: error });
|
|
106
|
+
const code = codeOf(error);
|
|
107
|
+
if (code !== void 0 && NETWORK.has(code)) throw new Error(`${NO_OLLAMA} (no answer at ${url.origin}: ${code})`, { cause: error });
|
|
108
|
+
const cause = error.cause;
|
|
109
|
+
throw new Error(`the request to Ollama at ${url.origin} failed: ${messageOf(error)}${cause === void 0 ? "" : ` (${messageOf(cause)})`}`, { cause: error });
|
|
110
|
+
}
|
|
111
|
+
let text;
|
|
112
|
+
try {
|
|
113
|
+
text = await response.text();
|
|
114
|
+
} catch (error) {
|
|
115
|
+
if (signal?.aborted === true) throw signal.reason;
|
|
116
|
+
throw new Error(`the answer of Ollama at ${url.origin} stopped: ${messageOf(error)}`, { cause: error });
|
|
117
|
+
}
|
|
118
|
+
let body;
|
|
119
|
+
try {
|
|
120
|
+
body = JSON.parse(text);
|
|
121
|
+
} catch {
|
|
122
|
+
body = void 0;
|
|
123
|
+
}
|
|
124
|
+
const said = typeof body?.error === "string" ? body.error : void 0;
|
|
125
|
+
if (!response.ok) {
|
|
126
|
+
if (said !== void 0 && /not found|pull/i.test(said)) throw new Error(`the model ${model} is not in Ollama: ollama pull ${model} (Ollama said: ${said})`);
|
|
127
|
+
if (response.status === 404) throw new Error(`the server at ${url.origin} has no /api/embed: update Ollama to version 0.3.0 or later`);
|
|
128
|
+
if (said !== void 0 && /embed/i.test(said)) throw new Error(`the model ${model} does not make embeddings: use an embedding model, such as nomic-embed-text (Ollama said: ${said})`);
|
|
129
|
+
throw new Error(`Ollama answered ${String(response.status)}${said === void 0 ? "" : `: ${said}`}`);
|
|
130
|
+
}
|
|
131
|
+
const embeddings = body?.embeddings;
|
|
132
|
+
if (!Array.isArray(embeddings) || embeddings.length !== input.length || !embeddings.every((vector) => Array.isArray(vector))) throw new Error(`Ollama gave no embedding for each text (${String(input.length)} texts): its answer is not the answer of /api/embed`);
|
|
133
|
+
return embeddings;
|
|
134
|
+
};
|
|
135
|
+
return {
|
|
136
|
+
id: `ollama:${model}${custom}`,
|
|
137
|
+
get dimensions() {
|
|
138
|
+
return dimensions;
|
|
139
|
+
},
|
|
140
|
+
maxInputTokens,
|
|
141
|
+
async embed(texts, call = {}) {
|
|
142
|
+
const prefix = call.purpose === "query" ? queryPrefix : documentPrefix;
|
|
143
|
+
const out = [];
|
|
144
|
+
for (let at = 0; at < texts.length; at += batch) {
|
|
145
|
+
call.signal?.throwIfAborted();
|
|
146
|
+
const part = texts.slice(at, at + batch).map((text) => `${prefix}${text}`);
|
|
147
|
+
for (const vector of await request(part, call.signal)) {
|
|
148
|
+
if (dimensions === 0) dimensions = vector.length;
|
|
149
|
+
if (vector.length !== dimensions) throw new Error(`the model ${model} gave a vector of ${String(vector.length)} dimensions, not ${String(dimensions)}`);
|
|
150
|
+
out.push(Float32Array.from(vector));
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return out;
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
};
|
|
157
|
+
//#endregion
|
|
158
|
+
export { ollama_exports as a, ollamaEmbedder as i, OLLAMA_MODELS as n, isLoopback as r, NO_OLLAMA as t };
|
|
159
|
+
|
|
160
|
+
//# sourceMappingURL=ollama-BJxkjny4.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama-BJxkjny4.mjs","names":[],"sources":["../src/ollama.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { EmbedCallOptions, Embedder } from \"./embed.ts\";\n\n/**\n * The Ollama embedder sends texts to the local HTTP API of Ollama (`POST /api/embed`). The package\n * has no dependency for it: you install Ollama, start it (`ollama serve`) and pull a model\n * (`ollama pull nomic-embed-text`).\n *\n * The page text goes to the Ollama server. Thus the embedder accepts only a server on this\n * computer (a loopback address), unless `allowRemote` is true.\n */\n\n/** These are the facts of the embedding models that the Ollama library has. */\nexport interface EmbeddingModel {\n readonly dimensions: number;\n readonly maxInputTokens: number;\n /** This prefix goes before the text of a question. */\n readonly queryPrefix?: string | undefined;\n /** This prefix goes before each chunk of a stored text. */\n readonly documentPrefix?: string | undefined;\n}\n\n/** These are the facts of three embedding models of Ollama. The name has no tag (`nomic-embed-text`, not `nomic-embed-text:latest`). */\nexport const OLLAMA_MODELS: Readonly<Record<string, EmbeddingModel>> = {\n \"nomic-embed-text\": {\n dimensions: 768,\n maxInputTokens: 8192,\n queryPrefix: \"search_query: \",\n documentPrefix: \"search_document: \",\n },\n \"all-minilm\": { dimensions: 384, maxInputTokens: 256 },\n \"mxbai-embed-large\": {\n dimensions: 1024,\n maxInputTokens: 512,\n queryPrefix: \"Represent this sentence for searching relevant passages: \",\n },\n};\n\n/** These options make an Ollama embedder. */\nexport interface OllamaEmbedderOptions {\n /** This value is the model of Ollama, with a tag when you want one. The default is `nomic-embed-text`. */\n readonly model?: string | undefined;\n /** This URL is the Ollama server. The default is `http://127.0.0.1:11434`. */\n readonly baseUrl?: string | undefined;\n /** This value is the number of texts in one request. The default is 32. */\n readonly batch?: number | undefined;\n /**\n * The value true lets `baseUrl` be a server on a different computer. The default is false,\n * because the embedder sends the text of the stored pages to the server.\n */\n readonly allowRemote?: boolean | undefined;\n /** This value is the length of each vector. The default comes from `OLLAMA_MODELS`, or from the first answer. */\n readonly dimensions?: number | undefined;\n /** This value is the input limit of the model in tokens. The default comes from `OLLAMA_MODELS`, or it is 256. */\n readonly maxInputTokens?: number | undefined;\n /** This prefix goes before the text of a question. The default comes from `OLLAMA_MODELS`. */\n readonly queryPrefix?: string | undefined;\n /** This prefix goes before each chunk of a stored text. The default comes from `OLLAMA_MODELS`. */\n readonly documentPrefix?: string | undefined;\n /** This value is the longest time of one request in milliseconds. The default is 2 minutes, because the first request loads the model. */\n readonly timeoutMs?: number | undefined;\n}\n\n/** The text of the error when no Ollama server answers. */\nexport const NO_OLLAMA =\n \"Ollama does not answer: install it from https://ollama.com, then start it with ollama serve\";\n\n/** This function tells if a host name is a loopback address: `localhost`, `127.0.0.0/8` or `::1`. */\nexport const isLoopback = (host: string): boolean => {\n const name = host.replace(/^\\[|\\]$/g, \"\").toLowerCase();\n return (\n name === \"localhost\" ||\n name.endsWith(\".localhost\") ||\n name === \"::1\" ||\n /^127(?:\\.\\d{1,3}){3}$/.test(name) ||\n /^::ffff:127(?:\\.\\d{1,3}){3}$/.test(name)\n );\n};\n\n/** This function gives the name of a model with no tag, for `OLLAMA_MODELS`. */\nconst baseName = (model: string): string => {\n const slash = model.lastIndexOf(\"/\");\n const colon = model.indexOf(\":\", slash + 1);\n return colon === -1 ? model : model.slice(0, colon);\n};\n\nconst messageOf = (error: unknown): string =>\n error instanceof Error ? error.message : String(error);\n\n/** This function gives the code of a network error, also when it is the cause of the error of `fetch`. */\nconst codeOf = (error: unknown): string | undefined => {\n for (\n let at: unknown = error, depth = 0;\n at !== undefined && at !== null && depth < 4;\n depth += 1\n ) {\n const code = (at as { code?: unknown }).code;\n if (typeof code === \"string\") return code;\n at = (at as { cause?: unknown }).cause;\n }\n return undefined;\n};\n\nconst NETWORK = new Set([\n \"ECONNREFUSED\",\n \"ECONNRESET\",\n \"ENOTFOUND\",\n \"EHOSTUNREACH\",\n \"ENETUNREACH\",\n \"EAI_AGAIN\",\n \"UND_ERR_SOCKET\",\n \"UND_ERR_CONNECT_TIMEOUT\",\n]);\n\n/**\n * This function makes an embedder that sends texts to Ollama (`POST {baseUrl}/api/embed`). It\n * sends `batch` texts in each request, and it stops when the signal aborts. The errors tell what\n * to do: `ollama serve` when no server answers, and `ollama pull <model>` when the model is not\n * there. A `baseUrl` on a different computer throws a `RangeError`, unless `allowRemote` is true.\n */\nexport const ollamaEmbedder = (options: OllamaEmbedderOptions = {}): Embedder => {\n const model = options.model ?? \"nomic-embed-text\";\n if (model.trim() === \"\") throw new RangeError(\"the Ollama model must have a name\");\n const base = options.baseUrl ?? \"http://127.0.0.1:11434\";\n let url: URL;\n try {\n url = new URL(base);\n } catch (cause) {\n throw new RangeError(`the Ollama URL ${base} is not a URL`, { cause });\n }\n if (url.protocol !== \"http:\" && url.protocol !== \"https:\")\n throw new RangeError(`the Ollama URL ${base} must be an http or https URL`);\n if (url.username !== \"\" || url.password !== \"\")\n throw new RangeError(\"the Ollama URL must not hold a user name or a password\");\n if (!isLoopback(url.hostname) && options.allowRemote !== true) {\n throw new RangeError(\n `the Ollama URL ${url.origin} is not on this computer, and the embedder sends the text of the stored pages to it: give allowRemote: true (--allow-remote-ollama) to permit it`,\n );\n }\n const endpoint = new URL(\"api/embed\", url.href.endsWith(\"/\") ? url.href : `${url.href}/`).href;\n const known = OLLAMA_MODELS[baseName(model)];\n const batch = Math.max(1, Math.floor(options.batch ?? 32));\n const timeoutMs = Math.max(1, Math.floor(options.timeoutMs ?? 120_000));\n const queryPrefix = options.queryPrefix ?? known?.queryPrefix ?? \"\";\n const documentPrefix = options.documentPrefix ?? known?.documentPrefix ?? \"\";\n // a document prefix of your own changes each stored vector, so it changes the id\n const custom =\n options.documentPrefix !== undefined && options.documentPrefix !== (known?.documentPrefix ?? \"\")\n ? `#${createHash(\"sha256\").update(options.documentPrefix).digest(\"hex\").slice(0, 8)}`\n : \"\";\n let dimensions = options.dimensions ?? known?.dimensions ?? 0;\n const maxInputTokens = options.maxInputTokens ?? known?.maxInputTokens ?? 256;\n\n const request = async (\n input: readonly string[],\n signal: AbortSignal | undefined,\n ): Promise<number[][]> => {\n const timeout = AbortSignal.timeout(timeoutMs);\n const both = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);\n let response: Response;\n try {\n response = await fetch(endpoint, {\n method: \"POST\",\n headers: { \"content-type\": \"application/json\" },\n body: JSON.stringify({ model, input, truncate: true }),\n signal: both,\n });\n } catch (error) {\n if (signal?.aborted === true) throw signal.reason;\n if (timeout.aborted)\n throw new Error(`Ollama at ${url.origin} did not answer in ${String(timeoutMs)} ms`, {\n cause: error,\n });\n const code = codeOf(error);\n if (code !== undefined && NETWORK.has(code))\n throw new Error(`${NO_OLLAMA} (no answer at ${url.origin}: ${code})`, { cause: error });\n // fetch says only \"fetch failed\": the cause tells why\n const cause = (error as { cause?: unknown }).cause;\n throw new Error(\n `the request to Ollama at ${url.origin} failed: ${messageOf(error)}${cause === undefined ? \"\" : ` (${messageOf(cause)})`}`,\n { cause: error },\n );\n }\n let text: string;\n try {\n text = await response.text();\n } catch (error) {\n if (signal?.aborted === true) throw signal.reason;\n throw new Error(`the answer of Ollama at ${url.origin} stopped: ${messageOf(error)}`, {\n cause: error,\n });\n }\n let body: { embeddings?: unknown; error?: unknown } | undefined;\n try {\n body = JSON.parse(text) as { embeddings?: unknown; error?: unknown };\n } catch {\n body = undefined;\n }\n const said = typeof body?.error === \"string\" ? body.error : undefined;\n if (!response.ok) {\n if (said !== undefined && /not found|pull/i.test(said))\n throw new Error(\n `the model ${model} is not in Ollama: ollama pull ${model} (Ollama said: ${said})`,\n );\n if (response.status === 404)\n throw new Error(\n `the server at ${url.origin} has no /api/embed: update Ollama to version 0.3.0 or later`,\n );\n if (said !== undefined && /embed/i.test(said))\n throw new Error(\n `the model ${model} does not make embeddings: use an embedding model, such as nomic-embed-text (Ollama said: ${said})`,\n );\n throw new Error(\n `Ollama answered ${String(response.status)}${said === undefined ? \"\" : `: ${said}`}`,\n );\n }\n const embeddings = body?.embeddings;\n if (\n !Array.isArray(embeddings) ||\n embeddings.length !== input.length ||\n !embeddings.every((vector) => Array.isArray(vector))\n ) {\n throw new Error(\n `Ollama gave no embedding for each text (${String(input.length)} texts): its answer is not the answer of /api/embed`,\n );\n }\n return embeddings as number[][];\n };\n\n return {\n id: `ollama:${model}${custom}`,\n get dimensions() {\n return dimensions;\n },\n maxInputTokens,\n async embed(texts: readonly string[], call: EmbedCallOptions = {}): Promise<Float32Array[]> {\n const prefix = call.purpose === \"query\" ? queryPrefix : documentPrefix;\n const out: Float32Array[] = [];\n for (let at = 0; at < texts.length; at += batch) {\n call.signal?.throwIfAborted();\n const part = texts.slice(at, at + batch).map((text) => `${prefix}${text}`);\n for (const vector of await request(part, call.signal)) {\n if (dimensions === 0) dimensions = vector.length;\n if (vector.length !== dimensions) {\n throw new Error(\n `the model ${model} gave a vector of ${String(vector.length)} dimensions, not ${String(dimensions)}`,\n );\n }\n out.push(Float32Array.from(vector));\n }\n }\n return out;\n },\n };\n};\n"],"mappings":";;;;;;;;;;AAuBA,MAAa,gBAA0D;CACrE,oBAAoB;EAClB,YAAY;EACZ,gBAAgB;EAChB,aAAa;EACb,gBAAgB;CAClB;CACA,cAAc;EAAE,YAAY;EAAK,gBAAgB;CAAI;CACrD,qBAAqB;EACnB,YAAY;EACZ,gBAAgB;EAChB,aAAa;CACf;AACF;;AA4BA,MAAa,YACX;;AAGF,MAAa,cAAc,SAA0B;CACnD,MAAM,OAAO,KAAK,QAAQ,YAAY,EAAE,CAAC,CAAC,YAAY;CACtD,OACE,SAAS,eACT,KAAK,SAAS,YAAY,KAC1B,SAAS,SACT,wBAAwB,KAAK,IAAI,KACjC,+BAA+B,KAAK,IAAI;AAE5C;;AAGA,MAAM,YAAY,UAA0B;CAC1C,MAAM,QAAQ,MAAM,YAAY,GAAG;CACnC,MAAM,QAAQ,MAAM,QAAQ,KAAK,QAAQ,CAAC;CAC1C,OAAO,UAAU,KAAK,QAAQ,MAAM,MAAM,GAAG,KAAK;AACpD;AAEA,MAAM,aAAa,UACjB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;;AAGvD,MAAM,UAAU,UAAuC;CACrD,KACE,IAAI,KAAc,OAAO,QAAQ,GACjC,OAAO,KAAA,KAAa,OAAO,QAAQ,QAAQ,GAC3C,SAAS,GACT;EACA,MAAM,OAAQ,GAA0B;EACxC,IAAI,OAAO,SAAS,UAAU,OAAO;EACrC,KAAM,GAA2B;CACnC;AAEF;AAEA,MAAM,0BAAU,IAAI,IAAI;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;AAQD,MAAa,kBAAkB,UAAiC,CAAC,MAAgB;CAC/E,MAAM,QAAQ,QAAQ,SAAS;CAC/B,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,IAAI,WAAW,mCAAmC;CACjF,MAAM,OAAO,QAAQ,WAAW;CAChC,IAAI;CACJ,IAAI;EACF,MAAM,IAAI,IAAI,IAAI;CACpB,SAAS,OAAO;EACd,MAAM,IAAI,WAAW,kBAAkB,KAAK,gBAAgB,EAAE,MAAM,CAAC;CACvE;CACA,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAC/C,MAAM,IAAI,WAAW,kBAAkB,KAAK,8BAA8B;CAC5E,IAAI,IAAI,aAAa,MAAM,IAAI,aAAa,IAC1C,MAAM,IAAI,WAAW,wDAAwD;CAC/E,IAAI,CAAC,WAAW,IAAI,QAAQ,KAAK,QAAQ,gBAAgB,MACvD,MAAM,IAAI,WACR,kBAAkB,IAAI,OAAO,iJAC/B;CAEF,MAAM,WAAW,IAAI,IAAI,aAAa,IAAI,KAAK,SAAS,GAAG,IAAI,IAAI,OAAO,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;CAC1F,MAAM,QAAQ,cAAc,SAAS,KAAK;CAC1C,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,EAAE,CAAC;CACzD,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,aAAa,IAAO,CAAC;CACtE,MAAM,cAAc,QAAQ,eAAe,OAAO,eAAe;CACjE,MAAM,iBAAiB,QAAQ,kBAAkB,OAAO,kBAAkB;CAE1E,MAAM,SACJ,QAAQ,mBAAmB,KAAA,KAAa,QAAQ,oBAAoB,OAAO,kBAAkB,MACzF,IAAI,WAAW,QAAQ,CAAC,CAAC,OAAO,QAAQ,cAAc,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,MAChF;CACN,IAAI,aAAa,QAAQ,cAAc,OAAO,cAAc;CAC5D,MAAM,iBAAiB,QAAQ,kBAAkB,OAAO,kBAAkB;CAE1E,MAAM,UAAU,OACd,OACA,WACwB;EACxB,MAAM,UAAU,YAAY,QAAQ,SAAS;EAC7C,MAAM,OAAO,WAAW,KAAA,IAAY,UAAU,YAAY,IAAI,CAAC,QAAQ,OAAO,CAAC;EAC/E,IAAI;EACJ,IAAI;GACF,WAAW,MAAM,MAAM,UAAU;IAC/B,QAAQ;IACR,SAAS,EAAE,gBAAgB,mBAAmB;IAC9C,MAAM,KAAK,UAAU;KAAE;KAAO;KAAO,UAAU;IAAK,CAAC;IACrD,QAAQ;GACV,CAAC;EACH,SAAS,OAAO;GACd,IAAI,QAAQ,YAAY,MAAM,MAAM,OAAO;GAC3C,IAAI,QAAQ,SACV,MAAM,IAAI,MAAM,aAAa,IAAI,OAAO,qBAAqB,OAAO,SAAS,EAAE,MAAM,EACnF,OAAO,MACT,CAAC;GACH,MAAM,OAAO,OAAO,KAAK;GACzB,IAAI,SAAS,KAAA,KAAa,QAAQ,IAAI,IAAI,GACxC,MAAM,IAAI,MAAM,GAAG,UAAU,iBAAiB,IAAI,OAAO,IAAI,KAAK,IAAI,EAAE,OAAO,MAAM,CAAC;GAExF,MAAM,QAAS,MAA8B;GAC7C,MAAM,IAAI,MACR,4BAA4B,IAAI,OAAO,WAAW,UAAU,KAAK,IAAI,UAAU,KAAA,IAAY,KAAK,KAAK,UAAU,KAAK,EAAE,MACtH,EAAE,OAAO,MAAM,CACjB;EACF;EACA,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,SAAS,KAAK;EAC7B,SAAS,OAAO;GACd,IAAI,QAAQ,YAAY,MAAM,MAAM,OAAO;GAC3C,MAAM,IAAI,MAAM,2BAA2B,IAAI,OAAO,YAAY,UAAU,KAAK,KAAK,EACpF,OAAO,MACT,CAAC;EACH;EACA,IAAI;EACJ,IAAI;GACF,OAAO,KAAK,MAAM,IAAI;EACxB,QAAQ;GACN,OAAO,KAAA;EACT;EACA,MAAM,OAAO,OAAO,MAAM,UAAU,WAAW,KAAK,QAAQ,KAAA;EAC5D,IAAI,CAAC,SAAS,IAAI;GAChB,IAAI,SAAS,KAAA,KAAa,kBAAkB,KAAK,IAAI,GACnD,MAAM,IAAI,MACR,aAAa,MAAM,iCAAiC,MAAM,iBAAiB,KAAK,EAClF;GACF,IAAI,SAAS,WAAW,KACtB,MAAM,IAAI,MACR,iBAAiB,IAAI,OAAO,4DAC9B;GACF,IAAI,SAAS,KAAA,KAAa,SAAS,KAAK,IAAI,GAC1C,MAAM,IAAI,MACR,aAAa,MAAM,4FAA4F,KAAK,EACtH;GACF,MAAM,IAAI,MACR,mBAAmB,OAAO,SAAS,MAAM,IAAI,SAAS,KAAA,IAAY,KAAK,KAAK,QAC9E;EACF;EACA,MAAM,aAAa,MAAM;EACzB,IACE,CAAC,MAAM,QAAQ,UAAU,KACzB,WAAW,WAAW,MAAM,UAC5B,CAAC,WAAW,OAAO,WAAW,MAAM,QAAQ,MAAM,CAAC,GAEnD,MAAM,IAAI,MACR,2CAA2C,OAAO,MAAM,MAAM,EAAE,oDAClE;EAEF,OAAO;CACT;CAEA,OAAO;EACL,IAAI,UAAU,QAAQ;EACtB,IAAI,aAAa;GACf,OAAO;EACT;EACA;EACA,MAAM,MAAM,OAA0B,OAAyB,CAAC,GAA4B;GAC1F,MAAM,SAAS,KAAK,YAAY,UAAU,cAAc;GACxD,MAAM,MAAsB,CAAC;GAC7B,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,QAAQ,MAAM,OAAO;IAC/C,KAAK,QAAQ,eAAe;IAC5B,MAAM,OAAO,MAAM,MAAM,IAAI,KAAK,KAAK,CAAC,CAAC,KAAK,SAAS,GAAG,SAAS,MAAM;IACzE,KAAK,MAAM,UAAU,MAAM,QAAQ,MAAM,KAAK,MAAM,GAAG;KACrD,IAAI,eAAe,GAAG,aAAa,OAAO;KAC1C,IAAI,OAAO,WAAW,YACpB,MAAM,IAAI,MACR,aAAa,MAAM,oBAAoB,OAAO,OAAO,MAAM,EAAE,mBAAmB,OAAO,UAAU,GACnG;KAEF,IAAI,KAAK,aAAa,KAAK,MAAM,CAAC;IACpC;GACF;GACA,OAAO;EACT;CACF;AACF"}
|