sensemaking 0.15.4 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -5
- package/dist/cjs/cli/download.js +12 -11
- package/dist/cjs/cli/download.js.map +1 -1
- package/dist/cjs/cli/index.d.cts +1 -1
- package/dist/cjs/cli/index.d.ts +1 -1
- package/dist/cjs/cli/index.js +1 -1
- package/dist/cjs/cli/index.js.map +1 -1
- package/dist/cjs/cli/init.js +31 -5
- package/dist/cjs/cli/init.js.map +1 -1
- package/dist/cjs/cli/status.js +343 -99
- package/dist/cjs/cli/status.js.map +1 -1
- package/dist/cjs/commands/related.js +17 -13
- package/dist/cjs/commands/related.js.map +1 -1
- package/dist/cjs/commands/search.js +25 -96
- package/dist/cjs/commands/search.js.map +1 -1
- package/dist/cjs/commands/signals.d.cts +18 -0
- package/dist/cjs/commands/signals.d.ts +18 -0
- package/dist/cjs/commands/signals.js +299 -0
- package/dist/cjs/commands/signals.js.map +1 -0
- package/dist/cjs/commands/status.d.cts +2 -2
- package/dist/cjs/commands/status.d.ts +2 -2
- package/dist/cjs/commands/status.js +1 -1
- package/dist/cjs/commands/status.js.map +1 -1
- package/dist/cjs/config/access.d.cts +5 -2
- package/dist/cjs/config/access.d.ts +5 -2
- package/dist/cjs/config/access.js +30 -12
- package/dist/cjs/config/access.js.map +1 -1
- package/dist/cjs/config/index.d.cts +4 -1
- package/dist/cjs/config/index.d.ts +4 -1
- package/dist/cjs/config/index.js +12 -2
- package/dist/cjs/config/index.js.map +1 -1
- package/dist/cjs/config/load.d.cts +6 -1
- package/dist/cjs/config/load.d.ts +6 -1
- package/dist/cjs/config/load.js +83 -8
- package/dist/cjs/config/load.js.map +1 -1
- package/dist/cjs/config/resolve.js +1 -1
- package/dist/cjs/config/resolve.js.map +1 -1
- package/dist/cjs/config/signals.d.cts +4 -0
- package/dist/cjs/config/signals.d.ts +4 -0
- package/dist/cjs/config/signals.js +30 -0
- package/dist/cjs/config/signals.js.map +1 -0
- package/dist/cjs/config/types.d.cts +6 -4
- package/dist/cjs/config/types.d.ts +6 -4
- package/dist/cjs/config/types.js +1 -1
- package/dist/cjs/config/types.js.map +1 -1
- package/dist/cjs/config/validate.js +120 -11
- package/dist/cjs/config/validate.js.map +1 -1
- package/dist/cjs/db/open.d.cts +1 -1
- package/dist/cjs/db/open.d.ts +1 -1
- package/dist/cjs/db/open.js +6 -1
- package/dist/cjs/db/open.js.map +1 -1
- package/dist/cjs/db/reconcile.d.cts +1 -0
- package/dist/cjs/db/reconcile.d.ts +1 -0
- package/dist/cjs/db/reconcile.js +15 -0
- package/dist/cjs/db/reconcile.js.map +1 -1
- package/dist/cjs/embed/cohere.d.cts +2 -0
- package/dist/cjs/embed/cohere.d.ts +2 -0
- package/dist/cjs/embed/cohere.js +238 -0
- package/dist/cjs/embed/cohere.js.map +1 -0
- package/dist/cjs/embed/http.d.cts +2 -0
- package/dist/cjs/embed/http.d.ts +2 -0
- package/dist/cjs/embed/http.js +258 -0
- package/dist/cjs/embed/http.js.map +1 -0
- package/dist/cjs/embed/identity.d.cts +13 -0
- package/dist/cjs/embed/identity.d.ts +13 -0
- package/dist/cjs/embed/identity.js +194 -0
- package/dist/cjs/embed/identity.js.map +1 -0
- package/dist/cjs/embed/langfit.d.cts +6 -0
- package/dist/cjs/embed/langfit.d.ts +6 -0
- package/dist/cjs/embed/langfit.js +326 -0
- package/dist/cjs/embed/langfit.js.map +1 -0
- package/dist/cjs/embed/languages.d.cts +3 -0
- package/dist/cjs/embed/languages.d.ts +3 -0
- package/dist/cjs/embed/languages.js +98 -0
- package/dist/cjs/embed/languages.js.map +1 -0
- package/dist/cjs/embed/openai.d.cts +2 -0
- package/dist/cjs/embed/openai.d.ts +2 -0
- package/dist/cjs/embed/openai.js +268 -0
- package/dist/cjs/embed/openai.js.map +1 -0
- package/dist/cjs/embed/query.d.cts +21 -0
- package/dist/cjs/embed/query.d.ts +21 -0
- package/dist/cjs/embed/query.js +610 -0
- package/dist/cjs/embed/query.js.map +1 -0
- package/dist/cjs/embed/registry.d.cts +3 -0
- package/dist/cjs/embed/registry.d.ts +3 -0
- package/dist/cjs/embed/registry.js +114 -0
- package/dist/cjs/embed/registry.js.map +1 -0
- package/dist/cjs/embed/static.d.cts +2 -0
- package/dist/cjs/embed/static.d.ts +2 -0
- package/dist/cjs/embed/static.js +367 -0
- package/dist/cjs/embed/static.js.map +1 -0
- package/dist/cjs/embed/store.d.cts +8 -0
- package/dist/cjs/embed/store.d.ts +8 -0
- package/dist/cjs/embed/store.js +498 -0
- package/dist/cjs/embed/store.js.map +1 -0
- package/dist/cjs/embed/types.d.cts +8 -0
- package/dist/cjs/embed/types.d.ts +8 -0
- package/dist/cjs/embed/types.js +7 -0
- package/dist/cjs/embed/types.js.map +1 -0
- package/dist/cjs/errors.d.cts +1 -1
- package/dist/cjs/errors.d.ts +1 -1
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/features/embed.d.cts +4 -27
- package/dist/cjs/features/embed.d.ts +4 -27
- package/dist/cjs/features/embed.js +3 -969
- package/dist/cjs/features/embed.js.map +1 -1
- package/dist/cjs/index.d.cts +1 -1
- package/dist/cjs/index.d.ts +1 -1
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/output.d.cts +2 -2
- package/dist/cjs/output.d.ts +2 -2
- package/dist/cjs/output.js +9 -2
- package/dist/cjs/output.js.map +1 -1
- package/dist/cjs/scan.js +8 -5
- package/dist/cjs/scan.js.map +1 -1
- package/dist/cjs/segment.js +67 -14
- package/dist/cjs/segment.js.map +1 -1
- package/dist/esm/cli/download.js +9 -8
- package/dist/esm/cli/download.js.map +1 -1
- package/dist/esm/cli/index.d.ts +1 -1
- package/dist/esm/cli/index.js +1 -1
- package/dist/esm/cli/index.js.map +1 -1
- package/dist/esm/cli/init.js +32 -6
- package/dist/esm/cli/init.js.map +1 -1
- package/dist/esm/cli/status.js +36 -8
- package/dist/esm/cli/status.js.map +1 -1
- package/dist/esm/commands/related.js +14 -10
- package/dist/esm/commands/related.js.map +1 -1
- package/dist/esm/commands/search.js +27 -78
- package/dist/esm/commands/search.js.map +1 -1
- package/dist/esm/commands/signals.d.ts +18 -0
- package/dist/esm/commands/signals.js +78 -0
- package/dist/esm/commands/signals.js.map +1 -0
- package/dist/esm/commands/status.d.ts +2 -2
- package/dist/esm/commands/status.js +2 -2
- package/dist/esm/commands/status.js.map +1 -1
- package/dist/esm/config/access.d.ts +5 -2
- package/dist/esm/config/access.js +31 -12
- package/dist/esm/config/access.js.map +1 -1
- package/dist/esm/config/index.d.ts +4 -1
- package/dist/esm/config/index.js +2 -1
- package/dist/esm/config/index.js.map +1 -1
- package/dist/esm/config/load.d.ts +6 -1
- package/dist/esm/config/load.js +67 -9
- package/dist/esm/config/load.js.map +1 -1
- package/dist/esm/config/resolve.js +2 -2
- package/dist/esm/config/resolve.js.map +1 -1
- package/dist/esm/config/signals.d.ts +4 -0
- package/dist/esm/config/signals.js +14 -0
- package/dist/esm/config/signals.js.map +1 -0
- package/dist/esm/config/types.d.ts +6 -4
- package/dist/esm/config/types.js +1 -1
- package/dist/esm/config/types.js.map +1 -1
- package/dist/esm/config/validate.js +58 -11
- package/dist/esm/config/validate.js.map +1 -1
- package/dist/esm/db/open.d.ts +1 -1
- package/dist/esm/db/open.js +7 -2
- package/dist/esm/db/open.js.map +1 -1
- package/dist/esm/db/reconcile.d.ts +1 -0
- package/dist/esm/db/reconcile.js +10 -0
- package/dist/esm/db/reconcile.js.map +1 -1
- package/dist/esm/embed/cohere.d.ts +2 -0
- package/dist/esm/embed/cohere.js +46 -0
- package/dist/esm/embed/cohere.js.map +1 -0
- package/dist/esm/embed/http.d.ts +2 -0
- package/dist/esm/embed/http.js +33 -0
- package/dist/esm/embed/http.js.map +1 -0
- package/dist/esm/embed/identity.d.ts +13 -0
- package/dist/esm/embed/identity.js +92 -0
- package/dist/esm/embed/identity.js.map +1 -0
- package/dist/esm/embed/langfit.d.ts +6 -0
- package/dist/esm/embed/langfit.js +44 -0
- package/dist/esm/embed/langfit.js.map +1 -0
- package/dist/esm/embed/languages.d.ts +3 -0
- package/dist/esm/embed/languages.js +80 -0
- package/dist/esm/embed/languages.js.map +1 -0
- package/dist/esm/embed/openai.d.ts +2 -0
- package/dist/esm/embed/openai.js +49 -0
- package/dist/esm/embed/openai.js.map +1 -0
- package/dist/esm/embed/query.d.ts +21 -0
- package/dist/esm/embed/query.js +181 -0
- package/dist/esm/embed/query.js.map +1 -0
- package/dist/esm/embed/registry.d.ts +3 -0
- package/dist/esm/embed/registry.js +45 -0
- package/dist/esm/embed/registry.js.map +1 -0
- package/dist/esm/embed/static.d.ts +2 -0
- package/dist/esm/embed/static.js +72 -0
- package/dist/esm/embed/static.js.map +1 -0
- package/dist/esm/embed/store.d.ts +8 -0
- package/dist/esm/embed/store.js +123 -0
- package/dist/esm/embed/store.js.map +1 -0
- package/dist/esm/embed/types.d.ts +8 -0
- package/dist/esm/embed/types.js +3 -0
- package/dist/esm/embed/types.js.map +1 -0
- package/dist/esm/errors.d.ts +1 -1
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/features/embed.d.ts +4 -27
- package/dist/esm/features/embed.js +0 -346
- package/dist/esm/features/embed.js.map +1 -1
- package/dist/esm/index.d.ts +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/output.d.ts +2 -2
- package/dist/esm/output.js +6 -2
- package/dist/esm/output.js.map +1 -1
- package/dist/esm/scan.js +7 -4
- package/dist/esm/scan.js.map +1 -1
- package/dist/esm/segment.js +49 -19
- package/dist/esm/segment.js.map +1 -1
- package/package.json +9 -1
- package/schema.json +14 -7
- package/skills/sense/SKILL.md +21 -5
- package/skills/sense-setup/EXAMPLES.md +6 -6
- package/skills/sense-setup/SKILL.md +6 -4
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", {
|
|
3
|
+
value: true
|
|
4
|
+
});
|
|
5
|
+
Object.defineProperty(exports, "cohereProvider", {
|
|
6
|
+
enumerable: true,
|
|
7
|
+
get: function() {
|
|
8
|
+
return cohereProvider;
|
|
9
|
+
}
|
|
10
|
+
});
|
|
11
|
+
var _errorsts = require("../errors.js");
|
|
12
|
+
var _httpts = require("./http.js");
|
|
13
|
+
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
|
|
14
|
+
try {
|
|
15
|
+
var info = gen[key](arg);
|
|
16
|
+
var value = info.value;
|
|
17
|
+
} catch (error) {
|
|
18
|
+
reject(error);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
if (info.done) resolve(value);
|
|
22
|
+
else Promise.resolve(value).then(_next, _throw);
|
|
23
|
+
}
|
|
24
|
+
function _async_to_generator(fn) {
|
|
25
|
+
return function() {
|
|
26
|
+
var self = this, args = arguments;
|
|
27
|
+
return new Promise(function(resolve, reject) {
|
|
28
|
+
var gen = fn.apply(self, args);
|
|
29
|
+
function _next(value) {
|
|
30
|
+
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
|
|
31
|
+
}
|
|
32
|
+
function _throw(err) {
|
|
33
|
+
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
|
|
34
|
+
}
|
|
35
|
+
_next(undefined);
|
|
36
|
+
});
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function _ts_generator(thisArg, body) {
|
|
40
|
+
var f, y, t, _ = {
|
|
41
|
+
label: 0,
|
|
42
|
+
sent: function() {
|
|
43
|
+
if (t[0] & 1) throw t[1];
|
|
44
|
+
return t[1];
|
|
45
|
+
},
|
|
46
|
+
trys: [],
|
|
47
|
+
ops: []
|
|
48
|
+
}, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
|
|
49
|
+
return d(g, "next", {
|
|
50
|
+
value: verb(0)
|
|
51
|
+
}), d(g, "throw", {
|
|
52
|
+
value: verb(1)
|
|
53
|
+
}), d(g, "return", {
|
|
54
|
+
value: verb(2)
|
|
55
|
+
}), typeof Symbol === "function" && d(g, Symbol.iterator, {
|
|
56
|
+
value: function() {
|
|
57
|
+
return this;
|
|
58
|
+
}
|
|
59
|
+
}), g;
|
|
60
|
+
function verb(n) {
|
|
61
|
+
return function(v) {
|
|
62
|
+
return step([
|
|
63
|
+
n,
|
|
64
|
+
v
|
|
65
|
+
]);
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
function step(op) {
|
|
69
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
70
|
+
while(g && (g = 0, op[0] && (_ = 0)), _)try {
|
|
71
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
72
|
+
if (y = 0, t) op = [
|
|
73
|
+
op[0] & 2,
|
|
74
|
+
t.value
|
|
75
|
+
];
|
|
76
|
+
switch(op[0]){
|
|
77
|
+
case 0:
|
|
78
|
+
case 1:
|
|
79
|
+
t = op;
|
|
80
|
+
break;
|
|
81
|
+
case 4:
|
|
82
|
+
_.label++;
|
|
83
|
+
return {
|
|
84
|
+
value: op[1],
|
|
85
|
+
done: false
|
|
86
|
+
};
|
|
87
|
+
case 5:
|
|
88
|
+
_.label++;
|
|
89
|
+
y = op[1];
|
|
90
|
+
op = [
|
|
91
|
+
0
|
|
92
|
+
];
|
|
93
|
+
continue;
|
|
94
|
+
case 7:
|
|
95
|
+
op = _.ops.pop();
|
|
96
|
+
_.trys.pop();
|
|
97
|
+
continue;
|
|
98
|
+
default:
|
|
99
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
|
|
100
|
+
_ = 0;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
|
|
104
|
+
_.label = op[1];
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
107
|
+
if (op[0] === 6 && _.label < t[1]) {
|
|
108
|
+
_.label = t[1];
|
|
109
|
+
t = op;
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
if (t && _.label < t[2]) {
|
|
113
|
+
_.label = t[2];
|
|
114
|
+
_.ops.push(op);
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
if (t[2]) _.ops.pop();
|
|
118
|
+
_.trys.pop();
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
op = body.call(thisArg, _);
|
|
122
|
+
} catch (e) {
|
|
123
|
+
op = [
|
|
124
|
+
6,
|
|
125
|
+
e
|
|
126
|
+
];
|
|
127
|
+
y = 0;
|
|
128
|
+
} finally{
|
|
129
|
+
f = t = 0;
|
|
130
|
+
}
|
|
131
|
+
if (op[0] & 5) throw op[1];
|
|
132
|
+
return {
|
|
133
|
+
value: op[0] ? op[1] : void 0,
|
|
134
|
+
done: true
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
var BATCH_CAP = 96;
|
|
139
|
+
function cohereProvider(model, url, keyEnv) {
|
|
140
|
+
return _async_to_generator(function() {
|
|
141
|
+
var base, headers, key, dims;
|
|
142
|
+
function post(texts, inputType) {
|
|
143
|
+
return _async_to_generator(function() {
|
|
144
|
+
var res, body, vectors;
|
|
145
|
+
return _ts_generator(this, function(_state) {
|
|
146
|
+
switch(_state.label){
|
|
147
|
+
case 0:
|
|
148
|
+
return [
|
|
149
|
+
4,
|
|
150
|
+
(0, _httpts.fetchWithRetry)("".concat(base, "/v2/embed"), {
|
|
151
|
+
method: 'POST',
|
|
152
|
+
headers: headers,
|
|
153
|
+
body: JSON.stringify({
|
|
154
|
+
model: model,
|
|
155
|
+
input_type: inputType,
|
|
156
|
+
texts: texts,
|
|
157
|
+
embedding_types: [
|
|
158
|
+
'float'
|
|
159
|
+
]
|
|
160
|
+
})
|
|
161
|
+
})
|
|
162
|
+
];
|
|
163
|
+
case 1:
|
|
164
|
+
res = _state.sent();
|
|
165
|
+
if (!res.ok) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(base, "/v2/embed -> HTTP ").concat(res.status));
|
|
166
|
+
return [
|
|
167
|
+
4,
|
|
168
|
+
res.json()
|
|
169
|
+
];
|
|
170
|
+
case 2:
|
|
171
|
+
body = _state.sent();
|
|
172
|
+
// Cohere returns one vector per text in order, with no index field to re-sort by; the
|
|
173
|
+
// count is what a caller mapping positionally still has to be able to trust.
|
|
174
|
+
vectors = body.embeddings.float;
|
|
175
|
+
if (vectors.length !== texts.length) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(base, "/v2/embed returned ").concat(vectors.length, " embeddings for ").concat(texts.length, " inputs"));
|
|
176
|
+
return [
|
|
177
|
+
2,
|
|
178
|
+
vectors.map(function(v) {
|
|
179
|
+
return Float32Array.from(v);
|
|
180
|
+
})
|
|
181
|
+
];
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
})();
|
|
185
|
+
}
|
|
186
|
+
return _ts_generator(this, function(_state) {
|
|
187
|
+
switch(_state.label){
|
|
188
|
+
case 0:
|
|
189
|
+
base = (url !== null && url !== void 0 ? url : 'https://api.cohere.com').replace(/\/+$/, '');
|
|
190
|
+
headers = {
|
|
191
|
+
'content-type': 'application/json'
|
|
192
|
+
};
|
|
193
|
+
key = keyEnv ? process.env[keyEnv] : undefined;
|
|
194
|
+
if (key) headers.authorization = "Bearer ".concat(key);
|
|
195
|
+
return [
|
|
196
|
+
4,
|
|
197
|
+
post([
|
|
198
|
+
'dimension probe'
|
|
199
|
+
], 'search_query')
|
|
200
|
+
];
|
|
201
|
+
case 1:
|
|
202
|
+
dims = _state.sent()[0].length;
|
|
203
|
+
return [
|
|
204
|
+
2,
|
|
205
|
+
{
|
|
206
|
+
id: "cohere:".concat(base, ":").concat(model),
|
|
207
|
+
dims: dims,
|
|
208
|
+
batchCap: BATCH_CAP,
|
|
209
|
+
embedDocuments: function embedDocuments(texts) {
|
|
210
|
+
return post(texts, 'search_document');
|
|
211
|
+
},
|
|
212
|
+
embedQuery: function embedQuery(text) {
|
|
213
|
+
return _async_to_generator(function() {
|
|
214
|
+
return _ts_generator(this, function(_state) {
|
|
215
|
+
switch(_state.label){
|
|
216
|
+
case 0:
|
|
217
|
+
return [
|
|
218
|
+
4,
|
|
219
|
+
post([
|
|
220
|
+
text
|
|
221
|
+
], 'search_query')
|
|
222
|
+
];
|
|
223
|
+
case 1:
|
|
224
|
+
return [
|
|
225
|
+
2,
|
|
226
|
+
_state.sent()[0]
|
|
227
|
+
];
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
})();
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
];
|
|
234
|
+
}
|
|
235
|
+
});
|
|
236
|
+
})();
|
|
237
|
+
}
|
|
238
|
+
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/cohere.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 96;\n\n// Cohere's native /v2/embed: input_type distinguishes doc vs query embeddings, which the\n// OpenAI-compatible shape cannot express.\nexport async function cohereProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n const base = (url ?? 'https://api.cohere.com').replace(/\\/+$/, '');\n const headers: Record<string, string> = { 'content-type': 'application/json' };\n const key = keyEnv ? process.env[keyEnv] : undefined;\n if (key) headers.authorization = `Bearer ${key}`;\n\n async function post(texts: string[], inputType: 'search_document' | 'search_query'): Promise<Float32Array[]> {\n const res = await fetchWithRetry(`${base}/v2/embed`, { method: 'POST', headers, body: JSON.stringify({ model, input_type: inputType, texts, embedding_types: ['float'] }) });\n if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/v2/embed -> HTTP ${res.status}`);\n const body = (await res.json()) as { embeddings: { float: number[][] } };\n // Cohere returns one vector per text in order, with no index field to re-sort by; the\n // count is what a caller mapping positionally still has to be able to trust.\n const vectors = body.embeddings.float;\n if (vectors.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/v2/embed returned ${vectors.length} embeddings for ${texts.length} inputs`);\n return vectors.map((v) => Float32Array.from(v));\n }\n\n const dims = (await post(['dimension probe'], 'search_query'))[0].length;\n return { id: `cohere:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: (texts) => post(texts, 'search_document'), embedQuery: async (text) => (await post([text], 'search_query'))[0] };\n}\n"],"names":["cohereProvider","BATCH_CAP","model","url","keyEnv","base","headers","key","dims","post","texts","inputType","res","body","vectors","fetchWithRetry","method","JSON","stringify","input_type","embedding_types","ok","SenseError","status","json","embeddings","float","length","map","v","Float32Array","from","replace","process","env","undefined","authorization","id","batchCap","embedDocuments","embedQuery","text"],"mappings":";;;;+BAQsBA;;;eAAAA;;;wBARK;sBACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/B,IAAMC,YAAY;AAIX,SAAeD,eAAeE,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;;YAC/FC,MACAC,SACAC,KAcAC;QAXN,SAAeC,KAAKC,KAAe,EAAEC,SAA6C;;oBAC1EC,KAEAC,MAGAC;;;;4BALM;;gCAAMC,IAAAA,sBAAc,EAAC,AAAC,GAAO,OAALV,MAAK,cAAY;oCAAEW,QAAQ;oCAAQV,SAAAA;oCAASO,MAAMI,KAAKC,SAAS,CAAC;wCAAEhB,OAAAA;wCAAOiB,YAAYR;wCAAWD,OAAAA;wCAAOU,eAAe;4CAAG;;oCAAS;gCAAG;;;4BAApKR,MAAM;4BACZ,IAAI,CAACA,IAAIS,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA2BV,OAAzBP,MAAK,sBAA+B,OAAXO,IAAIW,MAAM;4BACzE;;gCAAMX,IAAIY,IAAI;;;4BAAtBX,OAAQ;4BACd,sFAAsF;4BACtF,6EAA6E;4BACvEC,UAAUD,KAAKY,UAAU,CAACC,KAAK;4BACrC,IAAIZ,QAAQa,MAAM,KAAKjB,MAAMiB,MAAM,EAAE,MAAM,IAAIL,oBAAU,CAAC,eAAe,AAAC,GAA4BR,OAA1BT,MAAK,uBAAsDK,OAAjCI,QAAQa,MAAM,EAAC,oBAA+B,OAAbjB,MAAMiB,MAAM,EAAC;4BACpJ;;gCAAOb,QAAQc,GAAG,CAAC,SAACC;2CAAMC,aAAaC,IAAI,CAACF;;;;;YAC9C;;;;;oBAdMxB,OAAO,CAACF,gBAAAA,iBAAAA,MAAO,0BAA0B6B,OAAO,CAAC,QAAQ;oBACzD1B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAAS6B,QAAQC,GAAG,CAAC9B,OAAO,GAAG+B;oBAC3C,IAAI5B,KAAKD,QAAQ8B,aAAa,GAAG,AAAC,UAAa,OAAJ7B;oBAa7B;;wBAAME;4BAAM;2BAAoB;;;oBAAxCD,OAAO,AAAC,aAAgD,CAAC,EAAE,CAACmB,MAAM;oBACxE;;wBAAO;4BAAEU,IAAI,AAAC,UAAiBnC,OAARG,MAAK,KAAS,OAANH;4BAASM,MAAAA;4BAAM8B,UAAUrC;4BAAWsC,gBAAgB,SAAhBA,eAAiB7B;uCAAUD,KAAKC,OAAO;;4BAAoB8B,YAAY,SAAZA,WAAmBC;;;;;gDAAU;;oDAAMhC;wDAAMgC;uDAAO;;;;;oDAAnB,aAAmC,CAAC,EAAE;;;;;;wBAAC;;;;IACpM"}
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", {
|
|
3
|
+
value: true
|
|
4
|
+
});
|
|
5
|
+
function _export(target, all) {
|
|
6
|
+
for(var name in all)Object.defineProperty(target, name, {
|
|
7
|
+
enumerable: true,
|
|
8
|
+
get: Object.getOwnPropertyDescriptor(all, name).get
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
_export(exports, {
|
|
12
|
+
get fetchWithRetry () {
|
|
13
|
+
return fetchWithRetry;
|
|
14
|
+
},
|
|
15
|
+
get probeReachable () {
|
|
16
|
+
return probeReachable;
|
|
17
|
+
}
|
|
18
|
+
});
|
|
19
|
+
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
|
|
20
|
+
try {
|
|
21
|
+
var info = gen[key](arg);
|
|
22
|
+
var value = info.value;
|
|
23
|
+
} catch (error) {
|
|
24
|
+
reject(error);
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
if (info.done) resolve(value);
|
|
28
|
+
else Promise.resolve(value).then(_next, _throw);
|
|
29
|
+
}
|
|
30
|
+
function _async_to_generator(fn) {
|
|
31
|
+
return function() {
|
|
32
|
+
var self = this, args = arguments;
|
|
33
|
+
return new Promise(function(resolve, reject) {
|
|
34
|
+
var gen = fn.apply(self, args);
|
|
35
|
+
function _next(value) {
|
|
36
|
+
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
|
|
37
|
+
}
|
|
38
|
+
function _throw(err) {
|
|
39
|
+
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
|
|
40
|
+
}
|
|
41
|
+
_next(undefined);
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function _ts_generator(thisArg, body) {
|
|
46
|
+
var f, y, t, _ = {
|
|
47
|
+
label: 0,
|
|
48
|
+
sent: function() {
|
|
49
|
+
if (t[0] & 1) throw t[1];
|
|
50
|
+
return t[1];
|
|
51
|
+
},
|
|
52
|
+
trys: [],
|
|
53
|
+
ops: []
|
|
54
|
+
}, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
|
|
55
|
+
return d(g, "next", {
|
|
56
|
+
value: verb(0)
|
|
57
|
+
}), d(g, "throw", {
|
|
58
|
+
value: verb(1)
|
|
59
|
+
}), d(g, "return", {
|
|
60
|
+
value: verb(2)
|
|
61
|
+
}), typeof Symbol === "function" && d(g, Symbol.iterator, {
|
|
62
|
+
value: function() {
|
|
63
|
+
return this;
|
|
64
|
+
}
|
|
65
|
+
}), g;
|
|
66
|
+
function verb(n) {
|
|
67
|
+
return function(v) {
|
|
68
|
+
return step([
|
|
69
|
+
n,
|
|
70
|
+
v
|
|
71
|
+
]);
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function step(op) {
|
|
75
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
76
|
+
while(g && (g = 0, op[0] && (_ = 0)), _)try {
|
|
77
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
78
|
+
if (y = 0, t) op = [
|
|
79
|
+
op[0] & 2,
|
|
80
|
+
t.value
|
|
81
|
+
];
|
|
82
|
+
switch(op[0]){
|
|
83
|
+
case 0:
|
|
84
|
+
case 1:
|
|
85
|
+
t = op;
|
|
86
|
+
break;
|
|
87
|
+
case 4:
|
|
88
|
+
_.label++;
|
|
89
|
+
return {
|
|
90
|
+
value: op[1],
|
|
91
|
+
done: false
|
|
92
|
+
};
|
|
93
|
+
case 5:
|
|
94
|
+
_.label++;
|
|
95
|
+
y = op[1];
|
|
96
|
+
op = [
|
|
97
|
+
0
|
|
98
|
+
];
|
|
99
|
+
continue;
|
|
100
|
+
case 7:
|
|
101
|
+
op = _.ops.pop();
|
|
102
|
+
_.trys.pop();
|
|
103
|
+
continue;
|
|
104
|
+
default:
|
|
105
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
|
|
106
|
+
_ = 0;
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
|
|
110
|
+
_.label = op[1];
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
if (op[0] === 6 && _.label < t[1]) {
|
|
114
|
+
_.label = t[1];
|
|
115
|
+
t = op;
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
if (t && _.label < t[2]) {
|
|
119
|
+
_.label = t[2];
|
|
120
|
+
_.ops.push(op);
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
if (t[2]) _.ops.pop();
|
|
124
|
+
_.trys.pop();
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
op = body.call(thisArg, _);
|
|
128
|
+
} catch (e) {
|
|
129
|
+
op = [
|
|
130
|
+
6,
|
|
131
|
+
e
|
|
132
|
+
];
|
|
133
|
+
y = 0;
|
|
134
|
+
} finally{
|
|
135
|
+
f = t = 0;
|
|
136
|
+
}
|
|
137
|
+
if (op[0] & 5) throw op[1];
|
|
138
|
+
return {
|
|
139
|
+
value: op[0] ? op[1] : void 0,
|
|
140
|
+
done: true
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
var MAX_ATTEMPTS = 3;
|
|
145
|
+
function sleep(ms) {
|
|
146
|
+
return new Promise(function(resolve) {
|
|
147
|
+
return setTimeout(resolve, ms);
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
function fetchWithRetry(url, init) {
|
|
151
|
+
var baseDelayMs = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : 200;
|
|
152
|
+
return _async_to_generator(function() {
|
|
153
|
+
var attempt, res, retryAfter, err;
|
|
154
|
+
return _ts_generator(this, function(_state) {
|
|
155
|
+
switch(_state.label){
|
|
156
|
+
case 0:
|
|
157
|
+
attempt = 0;
|
|
158
|
+
_state.label = 1;
|
|
159
|
+
case 1:
|
|
160
|
+
if (!(attempt < MAX_ATTEMPTS)) return [
|
|
161
|
+
3,
|
|
162
|
+
8
|
|
163
|
+
];
|
|
164
|
+
_state.label = 2;
|
|
165
|
+
case 2:
|
|
166
|
+
_state.trys.push([
|
|
167
|
+
2,
|
|
168
|
+
5,
|
|
169
|
+
,
|
|
170
|
+
7
|
|
171
|
+
]);
|
|
172
|
+
return [
|
|
173
|
+
4,
|
|
174
|
+
fetch(url, init)
|
|
175
|
+
];
|
|
176
|
+
case 3:
|
|
177
|
+
res = _state.sent();
|
|
178
|
+
if (res.ok || res.status !== 429 && res.status < 500 || attempt === MAX_ATTEMPTS - 1) return [
|
|
179
|
+
2,
|
|
180
|
+
res
|
|
181
|
+
];
|
|
182
|
+
retryAfter = Number(res.headers.get('retry-after'));
|
|
183
|
+
// Capped: a server naming a huge Retry-After must not stall an embed run for it.
|
|
184
|
+
return [
|
|
185
|
+
4,
|
|
186
|
+
sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10000) : baseDelayMs * Math.pow(2, attempt))
|
|
187
|
+
];
|
|
188
|
+
case 4:
|
|
189
|
+
_state.sent();
|
|
190
|
+
return [
|
|
191
|
+
3,
|
|
192
|
+
7
|
|
193
|
+
];
|
|
194
|
+
case 5:
|
|
195
|
+
err = _state.sent();
|
|
196
|
+
if (attempt === MAX_ATTEMPTS - 1) throw err;
|
|
197
|
+
return [
|
|
198
|
+
4,
|
|
199
|
+
sleep(baseDelayMs * Math.pow(2, attempt))
|
|
200
|
+
];
|
|
201
|
+
case 6:
|
|
202
|
+
_state.sent();
|
|
203
|
+
return [
|
|
204
|
+
3,
|
|
205
|
+
7
|
|
206
|
+
];
|
|
207
|
+
case 7:
|
|
208
|
+
attempt++;
|
|
209
|
+
return [
|
|
210
|
+
3,
|
|
211
|
+
1
|
|
212
|
+
];
|
|
213
|
+
case 8:
|
|
214
|
+
throw new Error('unreachable');
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
})();
|
|
218
|
+
}
|
|
219
|
+
function probeReachable(url) {
|
|
220
|
+
var timeoutMs = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : 1500;
|
|
221
|
+
return _async_to_generator(function() {
|
|
222
|
+
var unused;
|
|
223
|
+
return _ts_generator(this, function(_state) {
|
|
224
|
+
switch(_state.label){
|
|
225
|
+
case 0:
|
|
226
|
+
_state.trys.push([
|
|
227
|
+
0,
|
|
228
|
+
2,
|
|
229
|
+
,
|
|
230
|
+
3
|
|
231
|
+
]);
|
|
232
|
+
return [
|
|
233
|
+
4,
|
|
234
|
+
fetch(url, {
|
|
235
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
236
|
+
})
|
|
237
|
+
];
|
|
238
|
+
case 1:
|
|
239
|
+
_state.sent();
|
|
240
|
+
return [
|
|
241
|
+
2,
|
|
242
|
+
true
|
|
243
|
+
];
|
|
244
|
+
case 2:
|
|
245
|
+
unused = _state.sent();
|
|
246
|
+
return [
|
|
247
|
+
2,
|
|
248
|
+
false
|
|
249
|
+
];
|
|
250
|
+
case 3:
|
|
251
|
+
return [
|
|
252
|
+
2
|
|
253
|
+
];
|
|
254
|
+
}
|
|
255
|
+
});
|
|
256
|
+
})();
|
|
257
|
+
}
|
|
258
|
+
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/http.ts"],"sourcesContent":["const MAX_ATTEMPTS = 3;\n\nfunction sleep(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\n// Shared retry glue for the HTTP providers: 429/5xx/network failure retry with exponential\n// backoff honoring Retry-After; any other status returns immediately for the caller to handle.\nexport async function fetchWithRetry(url: string, init: RequestInit, baseDelayMs = 200): Promise<Response> {\n for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n try {\n const res = await fetch(url, init);\n if (res.ok || (res.status !== 429 && res.status < 500) || attempt === MAX_ATTEMPTS - 1) return res;\n const retryAfter = Number(res.headers.get('retry-after'));\n // Capped: a server naming a huge Retry-After must not stall an embed run for it.\n await sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10_000) : baseDelayMs * 2 ** attempt);\n } catch (err) {\n if (attempt === MAX_ATTEMPTS - 1) throw err;\n await sleep(baseDelayMs * 2 ** attempt);\n }\n }\n throw new Error('unreachable');\n}\n\n// Any HTTP response means the endpoint is up; only a network failure or the timeout counts as\n// unreachable, so an air-gapped machine never waits more than timeoutMs on `sense status`.\nexport async function probeReachable(url: string, timeoutMs = 1500): Promise<boolean> {\n try {\n await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n return true;\n } catch {\n return false;\n }\n}\n"],"names":["fetchWithRetry","probeReachable","MAX_ATTEMPTS","sleep","ms","Promise","resolve","setTimeout","url","init","baseDelayMs","attempt","res","retryAfter","err","fetch","ok","status","Number","headers","get","Math","min","Error","timeoutMs","signal","AbortSignal","timeout"],"mappings":";;;;;;;;;;;QAQsBA;eAAAA;;QAkBAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA1BtB,IAAMC,eAAe;AAErB,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,SAACC;eAAYC,WAAWD,SAASF;;AACtD;AAIO,SAAeJ,eAAeQ,GAAW,EAAEC,IAAiB;QAAEC,cAAAA,iEAAc;;YACxEC,SAECC,KAEAC,YAGCC;;;;oBAPFH,UAAU;;;yBAAGA,CAAAA,UAAUT,YAAW;;;;;;;;;;;;oBAE3B;;wBAAMa,MAAMP,KAAKC;;;oBAAvBG,MAAM;oBACZ,IAAIA,IAAII,EAAE,IAAKJ,IAAIK,MAAM,KAAK,OAAOL,IAAIK,MAAM,GAAG,OAAQN,YAAYT,eAAe,GAAG;;wBAAOU;;oBACzFC,aAAaK,OAAON,IAAIO,OAAO,CAACC,GAAG,CAAC;oBAC1C,iFAAiF;oBACjF;;wBAAMjB,MAAMU,aAAa,IAAIQ,KAAKC,GAAG,CAACT,aAAa,MAAM,SAAUH,uBAAc,GAAKC;;;oBAAtF;;;;;;oBACOG;oBACP,IAAIH,YAAYT,eAAe,GAAG,MAAMY;oBACxC;;wBAAMX,MAAMO,uBAAc,GAAKC;;;oBAA/B;;;;;;oBAT0CA;;;;;;oBAY9C,MAAM,IAAIY,MAAM;;;IAClB;;AAIO,SAAetB,eAAeO,GAAW;QAAEgB,YAAAA,iEAAY;;;;;;;;;;;;oBAE1D;;wBAAMT,MAAMP,KAAK;4BAAEiB,QAAQC,YAAYC,OAAO,CAACH;wBAAW;;;oBAA1D;oBACA;;wBAAO;;;;oBAEP;;wBAAO;;;;;;;;IAEX"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare const MODEL_FILES: string[];
|
|
2
|
+
export declare const MODEL_FILENAMES: string;
|
|
3
|
+
export declare function isDownloadable(model: string): boolean;
|
|
4
|
+
export declare function refsMainPath(model: string): string;
|
|
5
|
+
export declare function readRef(model: string): string | undefined;
|
|
6
|
+
export declare function writeRef(model: string, sha: string): void;
|
|
7
|
+
export declare function snapshotDir(model: string, sha: string): string;
|
|
8
|
+
export declare function languagesPath(model: string): string;
|
|
9
|
+
export declare function readLanguages(model: string): string[] | undefined;
|
|
10
|
+
export declare function writeLanguages(model: string, languages: string[]): void;
|
|
11
|
+
export declare function modelDir(model: string): string;
|
|
12
|
+
export declare function hasModelFiles(model: string): boolean;
|
|
13
|
+
export declare function modelIdentity(model: string): string | undefined;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare const MODEL_FILES: string[];
|
|
2
|
+
export declare const MODEL_FILENAMES: string;
|
|
3
|
+
export declare function isDownloadable(model: string): boolean;
|
|
4
|
+
export declare function refsMainPath(model: string): string;
|
|
5
|
+
export declare function readRef(model: string): string | undefined;
|
|
6
|
+
export declare function writeRef(model: string, sha: string): void;
|
|
7
|
+
export declare function snapshotDir(model: string, sha: string): string;
|
|
8
|
+
export declare function languagesPath(model: string): string;
|
|
9
|
+
export declare function readLanguages(model: string): string[] | undefined;
|
|
10
|
+
export declare function writeLanguages(model: string, languages: string[]): void;
|
|
11
|
+
export declare function modelDir(model: string): string;
|
|
12
|
+
export declare function hasModelFiles(model: string): boolean;
|
|
13
|
+
export declare function modelIdentity(model: string): string | undefined;
|