vocs 2.4.0 → 2.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +18 -1
- package/dist/cli.js.map +1 -1
- package/dist/internal/retriever.d.ts +21 -2
- package/dist/internal/retriever.d.ts.map +1 -1
- package/dist/internal/retriever.js +152 -43
- package/dist/internal/retriever.js.map +1 -1
- package/dist/internal/search.d.ts.map +1 -1
- package/dist/internal/search.js +2 -1
- package/dist/internal/search.js.map +1 -1
- package/dist/internal/vector-store.d.ts +98 -9
- package/dist/internal/vector-store.d.ts.map +1 -1
- package/dist/internal/vector-store.js +235 -6
- package/dist/internal/vector-store.js.map +1 -1
- package/dist/internal/vite-plugins.d.ts.map +1 -1
- package/dist/internal/vite-plugins.js +12 -5
- package/dist/internal/vite-plugins.js.map +1 -1
- package/dist/waku/internal/middleware/ai-search-warmup.d.ts +16 -0
- package/dist/waku/internal/middleware/ai-search-warmup.d.ts.map +1 -0
- package/dist/waku/internal/middleware/ai-search-warmup.js +54 -0
- package/dist/waku/internal/middleware/ai-search-warmup.js.map +1 -0
- package/dist/waku/internal/middleware/index.d.ts +1 -0
- package/dist/waku/internal/middleware/index.d.ts.map +1 -1
- package/dist/waku/internal/middleware/index.js +1 -0
- package/dist/waku/internal/middleware/index.js.map +1 -1
- package/dist/waku/middleware.d.ts +1 -1
- package/package.json +1 -1
- package/src/cli.ts +20 -1
- package/src/internal/retriever.test.ts +177 -10
- package/src/internal/retriever.ts +177 -46
- package/src/internal/search.test.ts +0 -2
- package/src/internal/search.ts +2 -1
- package/src/internal/vector-store.test.ts +202 -1
- package/src/internal/vector-store.ts +369 -9
- package/src/internal/vite-plugins.ts +13 -5
- package/src/waku/internal/middleware/ai-search-warmup.test.ts +102 -0
- package/src/waku/internal/middleware/ai-search-warmup.ts +52 -0
- package/src/waku/internal/middleware/index.ts +1 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect, it } from 'vitest'
|
|
1
|
+
import { afterEach, describe, expect, it, vi } from 'vitest'
|
|
2
2
|
import * as VectorStore from './vector-store.js'
|
|
3
3
|
|
|
4
4
|
describe('normalize', () => {
|
|
@@ -64,3 +64,204 @@ describe('resolveFormat', () => {
|
|
|
64
64
|
expect(VectorStore.resolveFormat('int8', 'server')).toBe('int8')
|
|
65
65
|
})
|
|
66
66
|
})
|
|
67
|
+
|
|
68
|
+
describe('cloudflare (remote adapter)', () => {
|
|
69
|
+
afterEach(() => vi.restoreAllMocks())
|
|
70
|
+
|
|
71
|
+
/** In-memory Vectorize API fake backing the `fetch` mock. */
|
|
72
|
+
function mockVectorize() {
|
|
73
|
+
const state = {
|
|
74
|
+
created: false,
|
|
75
|
+
dimensions: undefined as number | undefined,
|
|
76
|
+
requests: [] as { body: string | undefined; method: string; path: string }[],
|
|
77
|
+
vectors: new Map<string, { metadata: Record<string, unknown>; values: number[] }>(),
|
|
78
|
+
}
|
|
79
|
+
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => {
|
|
80
|
+
const url = new URL(String(input))
|
|
81
|
+
const method = init?.method ?? 'GET'
|
|
82
|
+
const body = typeof init?.body === 'string' ? init.body : undefined
|
|
83
|
+
state.requests.push({ body, method, path: url.pathname + url.search })
|
|
84
|
+
|
|
85
|
+
const base = '/client/v4/accounts/acc/vectorize/v2/indexes'
|
|
86
|
+
if (!url.pathname.startsWith(base)) return new Response('not found', { status: 404 })
|
|
87
|
+
const rest = url.pathname.slice(base.length)
|
|
88
|
+
|
|
89
|
+
// Create index.
|
|
90
|
+
if (rest === '' && method === 'POST') {
|
|
91
|
+
state.created = true
|
|
92
|
+
state.dimensions = JSON.parse(body ?? '{}').config?.dimensions
|
|
93
|
+
return Response.json({ success: true })
|
|
94
|
+
}
|
|
95
|
+
// Get index.
|
|
96
|
+
if (/^\/[^/]+$/.test(rest) && method === 'GET') {
|
|
97
|
+
if (!state.created) return new Response('not found', { status: 404 })
|
|
98
|
+
return Response.json({ result: { config: { dimensions: state.dimensions } } })
|
|
99
|
+
}
|
|
100
|
+
if (rest.endsWith('/list'))
|
|
101
|
+
return Response.json({
|
|
102
|
+
result: { isTruncated: false, vectors: [...state.vectors.keys()].map((id) => ({ id })) },
|
|
103
|
+
})
|
|
104
|
+
if (rest.endsWith('/upsert')) {
|
|
105
|
+
for (const line of (body ?? '').split('\n')) {
|
|
106
|
+
const { id, metadata, values } = JSON.parse(line)
|
|
107
|
+
state.vectors.set(id, { metadata, values })
|
|
108
|
+
}
|
|
109
|
+
return Response.json({ success: true })
|
|
110
|
+
}
|
|
111
|
+
if (rest.endsWith('/delete_by_ids')) {
|
|
112
|
+
const ids = JSON.parse(body ?? '{}').ids ?? []
|
|
113
|
+
// Mirrors Vectorize error 40007.
|
|
114
|
+
if (ids.length > 100)
|
|
115
|
+
return Response.json(
|
|
116
|
+
{ success: false, errors: [{ code: 40007, message: 'too many ids in payload' }] },
|
|
117
|
+
{ status: 400 },
|
|
118
|
+
)
|
|
119
|
+
for (const id of ids) state.vectors.delete(id)
|
|
120
|
+
return Response.json({ success: true })
|
|
121
|
+
}
|
|
122
|
+
if (rest.endsWith('/query')) {
|
|
123
|
+
const { topK, vector } = JSON.parse(body ?? '{}')
|
|
124
|
+
const matches = [...state.vectors.entries()]
|
|
125
|
+
.map(([id, v]) => ({
|
|
126
|
+
id,
|
|
127
|
+
metadata: v.metadata,
|
|
128
|
+
score: v.values.reduce((sum, x, i) => sum + x * (vector[i] ?? 0), 0),
|
|
129
|
+
}))
|
|
130
|
+
.sort((a, b) => b.score - a.score)
|
|
131
|
+
.slice(0, topK)
|
|
132
|
+
return Response.json({ result: { matches } })
|
|
133
|
+
}
|
|
134
|
+
return new Response('bad request', { status: 400 })
|
|
135
|
+
})
|
|
136
|
+
return state
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function entry(id: string, text: string, vector: number[]): VectorStore.RemoteEntry {
|
|
140
|
+
return {
|
|
141
|
+
id,
|
|
142
|
+
metadata: { href: `/${id}`, text, title: id },
|
|
143
|
+
vector: VectorStore.normalize(vector),
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const credentials = { accountId: 'acc', apiToken: 'tok' }
|
|
148
|
+
|
|
149
|
+
it('throws on missing credentials', async () => {
|
|
150
|
+
const store = VectorStore.cloudflare({ accountId: '', apiToken: '' })
|
|
151
|
+
await expect(store.query(new Float32Array([1]), { topK: 5 })).rejects.toThrow(/accountId/)
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
it('creates the index on first sync and upserts all entries', async () => {
|
|
155
|
+
const state = mockVectorize()
|
|
156
|
+
const store = VectorStore.cloudflare(credentials)
|
|
157
|
+
|
|
158
|
+
const result = await store.sync([entry('a', 'alpha', [1, 0]), entry('b', 'beta', [0, 1])], {
|
|
159
|
+
dimensions: 2,
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
expect(state.created).toBe(true)
|
|
163
|
+
expect(state.dimensions).toBe(2)
|
|
164
|
+
expect(result).toEqual({ deleted: 0, skipped: 0, upserted: 2 })
|
|
165
|
+
expect(state.vectors.size).toBe(2)
|
|
166
|
+
// Content-addressed ids: 64 hex chars (Vectorize's id byte cap).
|
|
167
|
+
for (const id of state.vectors.keys()) expect(id).toMatch(/^[0-9a-f]{64}$/)
|
|
168
|
+
})
|
|
169
|
+
|
|
170
|
+
it('re-sync skips unchanged entries and prunes stale ones', async () => {
|
|
171
|
+
const state = mockVectorize()
|
|
172
|
+
const store = VectorStore.cloudflare(credentials)
|
|
173
|
+
|
|
174
|
+
await store.sync([entry('a', 'alpha', [1, 0]), entry('b', 'beta', [0, 1])], { dimensions: 2 })
|
|
175
|
+
const result = await store.sync([entry('a', 'alpha', [1, 0]), entry('c', 'gamma', [1, 1])], {
|
|
176
|
+
dimensions: 2,
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
expect(result).toEqual({ deleted: 1, skipped: 1, upserted: 1 })
|
|
180
|
+
expect(state.vectors.size).toBe(2)
|
|
181
|
+
})
|
|
182
|
+
|
|
183
|
+
it('batches deletes within the 100-id Vectorize limit', async () => {
|
|
184
|
+
const state = mockVectorize()
|
|
185
|
+
const store = VectorStore.cloudflare(credentials)
|
|
186
|
+
|
|
187
|
+
const entries = Array.from({ length: 150 }, (_, i) => entry(`e${i}`, `text ${i}`, [1, i]))
|
|
188
|
+
await store.sync(entries, { dimensions: 2 })
|
|
189
|
+
const result = await store.sync([entry('keep', 'kept', [1, 0])], { dimensions: 2 })
|
|
190
|
+
|
|
191
|
+
expect(result).toEqual({ deleted: 150, skipped: 0, upserted: 1 })
|
|
192
|
+
expect(state.vectors.size).toBe(1)
|
|
193
|
+
})
|
|
194
|
+
|
|
195
|
+
it('rejects a dimension change against an existing index', async () => {
|
|
196
|
+
mockVectorize()
|
|
197
|
+
const store = VectorStore.cloudflare(credentials)
|
|
198
|
+
await store.sync([entry('a', 'alpha', [1, 0])], { dimensions: 2 })
|
|
199
|
+
await expect(store.sync([entry('a', 'alpha', [1, 0, 0])], { dimensions: 3 })).rejects.toThrow(
|
|
200
|
+
/dimensions/,
|
|
201
|
+
)
|
|
202
|
+
})
|
|
203
|
+
|
|
204
|
+
it('bounds oversized metadata under the Vectorize cap', async () => {
|
|
205
|
+
const state = mockVectorize()
|
|
206
|
+
const store = VectorStore.cloudflare(credentials)
|
|
207
|
+
|
|
208
|
+
await store.sync([entry('big', 'x'.repeat(20_000), [1, 0])], { dimensions: 2 })
|
|
209
|
+
|
|
210
|
+
const [stored] = [...state.vectors.values()]
|
|
211
|
+
const size = new TextEncoder().encode(JSON.stringify(stored?.metadata)).length
|
|
212
|
+
expect(size).toBeLessThanOrEqual(9216)
|
|
213
|
+
// Trimmed, not dropped.
|
|
214
|
+
expect(String(stored?.metadata['text']).length).toBeGreaterThan(0)
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
it('sanitizes metadata to Vectorize types (string | number | boolean | string[])', async () => {
|
|
218
|
+
const state = mockVectorize()
|
|
219
|
+
const store = VectorStore.cloudflare(credentials)
|
|
220
|
+
|
|
221
|
+
await store.sync(
|
|
222
|
+
[
|
|
223
|
+
{
|
|
224
|
+
id: 'a',
|
|
225
|
+
// Sparse `titles` (skipped heading levels) serialize as nulls.
|
|
226
|
+
metadata: {
|
|
227
|
+
href: '/a',
|
|
228
|
+
nested: { drop: true },
|
|
229
|
+
none: null,
|
|
230
|
+
title: 'a',
|
|
231
|
+
titles: ['Guide', null, 'Steps'],
|
|
232
|
+
weight: 0.8,
|
|
233
|
+
},
|
|
234
|
+
vector: VectorStore.normalize([1, 0]),
|
|
235
|
+
},
|
|
236
|
+
],
|
|
237
|
+
{ dimensions: 2 },
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
const [stored] = [...state.vectors.values()]
|
|
241
|
+
expect(stored?.metadata).toEqual({
|
|
242
|
+
href: '/a',
|
|
243
|
+
title: 'a',
|
|
244
|
+
titles: ['Guide', 'Steps'],
|
|
245
|
+
weight: 0.8,
|
|
246
|
+
})
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
it('queries nearest vectors and caps topK at 50', async () => {
|
|
250
|
+
const state = mockVectorize()
|
|
251
|
+
const store = VectorStore.cloudflare(credentials)
|
|
252
|
+
await store.sync([entry('a', 'alpha', [1, 0]), entry('b', 'beta', [0, 1])], { dimensions: 2 })
|
|
253
|
+
|
|
254
|
+
const hits = await store.query(VectorStore.normalize([0.9, 0.1]), { topK: 100 })
|
|
255
|
+
|
|
256
|
+
expect(hits[0]?.metadata?.['href']).toBe('/a')
|
|
257
|
+
expect(hits[0]?.score ?? 0).toBeGreaterThan(hits[1]?.score ?? 0)
|
|
258
|
+
const queryRequest = state.requests.find((r) => r.path.endsWith('/query'))
|
|
259
|
+
expect(JSON.parse(queryRequest?.body ?? '{}').topK).toBe(50)
|
|
260
|
+
})
|
|
261
|
+
|
|
262
|
+
it('propagates query failures', async () => {
|
|
263
|
+
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('nope', { status: 401 }))
|
|
264
|
+
const store = VectorStore.cloudflare(credentials)
|
|
265
|
+
await expect(store.query(new Float32Array([1]), { topK: 5 })).rejects.toThrow(/401/)
|
|
266
|
+
})
|
|
267
|
+
})
|
|
@@ -1,16 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
* alternative to a hosted vector DB (Pinecone, etc).
|
|
2
|
+
* Vector stores for AI search.
|
|
4
3
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* `
|
|
8
|
-
*
|
|
4
|
+
* Two storage targets:
|
|
5
|
+
*
|
|
6
|
+
* - `static` (default) — the built-in, open-source alternative to a hosted
|
|
7
|
+
* vector DB. Vectors are packed into a single typed-array matrix and
|
|
8
|
+
* base64-encoded, so the whole index is one compact artifact that works
|
|
9
|
+
* server-side (Node) and, with `int8` quantization, in the browser. Cosine
|
|
10
|
+
* similarity reduces to a dot product because all vectors are normalized
|
|
11
|
+
* before packing.
|
|
12
|
+
* - `remote` — vectors live in a hosted vector database (e.g. Cloudflare
|
|
13
|
+
* Vectorize). The build syncs vectors up; the server queries the database at
|
|
14
|
+
* runtime, so no index artifact ships with the server bundle.
|
|
9
15
|
*/
|
|
10
16
|
|
|
11
17
|
export type Format = 'float32' | 'int8'
|
|
12
18
|
|
|
13
|
-
export type Adapter =
|
|
19
|
+
export type Adapter = StaticAdapter | RemoteAdapter
|
|
20
|
+
|
|
21
|
+
export type StaticAdapter = {
|
|
14
22
|
/** Also emit a public/browser artifact (for `runtime: 'server'` deploys). */
|
|
15
23
|
expose: boolean
|
|
16
24
|
/** Storage format. `'auto'` resolves to `float32` (server) / `int8` (client). */
|
|
@@ -23,6 +31,55 @@ export type Adapter = {
|
|
|
23
31
|
type: string
|
|
24
32
|
}
|
|
25
33
|
|
|
34
|
+
/** Chunk vector + metadata synced into a remote store at build time. */
|
|
35
|
+
export type RemoteEntry = {
|
|
36
|
+
/** Stable chunk identifier (adapters may derive their own storage id). */
|
|
37
|
+
id: string
|
|
38
|
+
/** Chunk metadata returned verbatim by `query` at runtime. */
|
|
39
|
+
metadata: Record<string, unknown>
|
|
40
|
+
/** L2-normalized embedding vector. */
|
|
41
|
+
vector: Float32Array
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Nearest-neighbor match returned by a remote store. */
|
|
45
|
+
export type RemoteHit = {
|
|
46
|
+
/** Storage identifier of the matched vector. */
|
|
47
|
+
id: string
|
|
48
|
+
/** Chunk metadata attached at sync time (if any). */
|
|
49
|
+
metadata: Record<string, unknown> | undefined
|
|
50
|
+
/** Similarity score (higher is more relevant). */
|
|
51
|
+
score: number
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export type RemoteSyncResult = {
|
|
55
|
+
/** Stale vectors removed from the store. */
|
|
56
|
+
deleted: number
|
|
57
|
+
/** Vectors already up to date (not re-uploaded). */
|
|
58
|
+
skipped: number
|
|
59
|
+
/** New or changed vectors uploaded. */
|
|
60
|
+
upserted: number
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export type RemoteAdapter = {
|
|
64
|
+
/** Returns the `topK` nearest vectors to a normalized query vector. */
|
|
65
|
+
query: (
|
|
66
|
+
vector: Float32Array,
|
|
67
|
+
options: { signal?: AbortSignal | undefined; topK: number },
|
|
68
|
+
) => Promise<RemoteHit[]>
|
|
69
|
+
/** Syncs the full chunk set into the store (upserts new, prunes stale). */
|
|
70
|
+
sync: (
|
|
71
|
+
entries: readonly RemoteEntry[],
|
|
72
|
+
options: {
|
|
73
|
+
dimensions: number
|
|
74
|
+
onProgress?: ((done: number, total: number) => void) | undefined
|
|
75
|
+
},
|
|
76
|
+
) => Promise<RemoteSyncResult>
|
|
77
|
+
/** Storage target backing the adapter. */
|
|
78
|
+
target: 'remote'
|
|
79
|
+
/** Adapter type identifier. */
|
|
80
|
+
type: string
|
|
81
|
+
}
|
|
82
|
+
|
|
26
83
|
/**
|
|
27
84
|
* The built-in static vector store.
|
|
28
85
|
*
|
|
@@ -38,7 +95,7 @@ export type Adapter = {
|
|
|
38
95
|
* ```
|
|
39
96
|
*/
|
|
40
97
|
// biome-ignore lint/correctness/noUnusedVariables: re-exported below as `static`
|
|
41
|
-
function staticStore(options: staticStore.Options = {}):
|
|
98
|
+
function staticStore(options: staticStore.Options = {}): StaticAdapter {
|
|
42
99
|
return {
|
|
43
100
|
type: 'static',
|
|
44
101
|
target: 'static',
|
|
@@ -61,11 +118,311 @@ export declare namespace staticStore {
|
|
|
61
118
|
|
|
62
119
|
export { staticStore as static }
|
|
63
120
|
|
|
121
|
+
/**
|
|
122
|
+
* Cloudflare Vectorize vector store (remote). Vocs still chunks and embeds
|
|
123
|
+
* locally (with the configured `embedding` adapter); vectors are synced into a
|
|
124
|
+
* Vectorize index at build time and queried over REST at runtime, so no vector
|
|
125
|
+
* artifact ships with the server bundle.
|
|
126
|
+
*
|
|
127
|
+
* The index is created on first sync (cosine metric, dimensions inferred from
|
|
128
|
+
* the embedding). Sync is incremental: vector ids are content-addressed, so
|
|
129
|
+
* unchanged chunks are skipped and stale vectors are pruned. The API token
|
|
130
|
+
* needs the `Vectorize:Edit` permission.
|
|
131
|
+
*
|
|
132
|
+
* @example
|
|
133
|
+
* ```ts
|
|
134
|
+
* import { defineConfig, Embedding, Retriever, VectorStore } from 'vocs/config'
|
|
135
|
+
*
|
|
136
|
+
* export default defineConfig({
|
|
137
|
+
* ai: {
|
|
138
|
+
* retriever: Retriever.local({
|
|
139
|
+
* embedding: Embedding.openai(),
|
|
140
|
+
* vectorStore: VectorStore.cloudflare({ index: 'my-docs' }),
|
|
141
|
+
* }),
|
|
142
|
+
* },
|
|
143
|
+
* })
|
|
144
|
+
* ```
|
|
145
|
+
*/
|
|
146
|
+
export function cloudflare(options: cloudflare.Options = {}): RemoteAdapter {
|
|
147
|
+
const {
|
|
148
|
+
accountId = process.env['CLOUDFLARE_ACCOUNT_ID'],
|
|
149
|
+
apiToken = process.env['CLOUDFLARE_API_TOKEN'],
|
|
150
|
+
baseUrl = 'https://api.cloudflare.com/client/v4',
|
|
151
|
+
headers,
|
|
152
|
+
index = 'vocs-ai-search',
|
|
153
|
+
} = options
|
|
154
|
+
|
|
155
|
+
async function request(
|
|
156
|
+
path: string,
|
|
157
|
+
init: {
|
|
158
|
+
body?: string | undefined
|
|
159
|
+
contentType?: string | undefined
|
|
160
|
+
method?: string | undefined
|
|
161
|
+
signal?: AbortSignal | undefined
|
|
162
|
+
} = {},
|
|
163
|
+
): Promise<Response> {
|
|
164
|
+
if (!accountId)
|
|
165
|
+
throw new Error(
|
|
166
|
+
'[vocs] VectorStore.cloudflare: missing `accountId` (or CLOUDFLARE_ACCOUNT_ID).',
|
|
167
|
+
)
|
|
168
|
+
if (!apiToken)
|
|
169
|
+
throw new Error(
|
|
170
|
+
'[vocs] VectorStore.cloudflare: missing `apiToken` (or CLOUDFLARE_API_TOKEN).',
|
|
171
|
+
)
|
|
172
|
+
const url = `${baseUrl.replace(/\/$/, '')}/accounts/${accountId}/vectorize/v2/indexes${path}`
|
|
173
|
+
return await fetch(url, {
|
|
174
|
+
method: init.method ?? 'GET',
|
|
175
|
+
headers: {
|
|
176
|
+
Authorization: `Bearer ${apiToken}`,
|
|
177
|
+
...(init.contentType ? { 'Content-Type': init.contentType } : {}),
|
|
178
|
+
...headers,
|
|
179
|
+
},
|
|
180
|
+
...(init.body !== undefined ? { body: init.body } : {}),
|
|
181
|
+
signal: init.signal ?? null,
|
|
182
|
+
})
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Creates the index when missing; validates dimensions when present. */
|
|
186
|
+
async function ensureIndex(dimensions: number): Promise<void> {
|
|
187
|
+
const existing = await request(`/${index}`)
|
|
188
|
+
if (existing.ok) {
|
|
189
|
+
const json = (await existing.json()) as {
|
|
190
|
+
result?: { config?: { dimensions?: number | undefined } | undefined } | undefined
|
|
191
|
+
}
|
|
192
|
+
const remote = json.result?.config?.dimensions
|
|
193
|
+
if (remote !== undefined && remote !== dimensions)
|
|
194
|
+
throw new Error(
|
|
195
|
+
`[vocs] Vectorize index "${index}" has ${remote} dimensions, embedding produces ${dimensions}. Delete the index or configure a different \`index\` name.`,
|
|
196
|
+
)
|
|
197
|
+
return
|
|
198
|
+
}
|
|
199
|
+
if (existing.status !== 404)
|
|
200
|
+
throw new Error(
|
|
201
|
+
`[vocs] Vectorize get index failed (${existing.status}): ${await safeText(existing)}`,
|
|
202
|
+
)
|
|
203
|
+
const created = await request('', {
|
|
204
|
+
method: 'POST',
|
|
205
|
+
contentType: 'application/json',
|
|
206
|
+
body: JSON.stringify({ config: { dimensions, metric: 'cosine' }, name: index }),
|
|
207
|
+
})
|
|
208
|
+
if (!created.ok)
|
|
209
|
+
throw new Error(
|
|
210
|
+
`[vocs] Vectorize create index failed (${created.status}): ${await safeText(created)}`,
|
|
211
|
+
)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Enumerates all vector ids in the index, or `undefined` when unsupported. */
|
|
215
|
+
async function listIds(): Promise<Set<string> | undefined> {
|
|
216
|
+
const ids = new Set<string>()
|
|
217
|
+
let cursor: string | undefined
|
|
218
|
+
do {
|
|
219
|
+
const params = new URLSearchParams({ count: '1000' })
|
|
220
|
+
if (cursor) params.set('cursor', cursor)
|
|
221
|
+
const response = await request(`/${index}/list?${params}`)
|
|
222
|
+
if (!response.ok) {
|
|
223
|
+
console.warn(
|
|
224
|
+
`[vocs] Vectorize list vectors failed (${response.status}); skipping incremental sync (re-upserting all, no pruning).`,
|
|
225
|
+
)
|
|
226
|
+
return undefined
|
|
227
|
+
}
|
|
228
|
+
const json = (await response.json()) as {
|
|
229
|
+
result?:
|
|
230
|
+
| {
|
|
231
|
+
isTruncated?: boolean | undefined
|
|
232
|
+
nextCursor?: string | undefined
|
|
233
|
+
vectors?: { id?: string | undefined }[] | undefined
|
|
234
|
+
}
|
|
235
|
+
| undefined
|
|
236
|
+
}
|
|
237
|
+
for (const vector of json.result?.vectors ?? []) if (vector.id) ids.add(vector.id)
|
|
238
|
+
cursor = json.result?.isTruncated ? json.result?.nextCursor : undefined
|
|
239
|
+
} while (cursor)
|
|
240
|
+
return ids
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
return {
|
|
244
|
+
type: 'cloudflare',
|
|
245
|
+
target: 'remote',
|
|
246
|
+
async query(vector, { signal, topK }) {
|
|
247
|
+
const response = await request(`/${index}/query`, {
|
|
248
|
+
method: 'POST',
|
|
249
|
+
contentType: 'application/json',
|
|
250
|
+
body: JSON.stringify({
|
|
251
|
+
vector: Array.from(vector),
|
|
252
|
+
// Vectorize caps topK at 50 when returning metadata.
|
|
253
|
+
topK: Math.min(topK, 50),
|
|
254
|
+
returnValues: false,
|
|
255
|
+
returnMetadata: 'all',
|
|
256
|
+
}),
|
|
257
|
+
signal,
|
|
258
|
+
})
|
|
259
|
+
if (!response.ok)
|
|
260
|
+
throw new Error(
|
|
261
|
+
`[vocs] Vectorize query failed (${response.status}): ${await safeText(response)}`,
|
|
262
|
+
)
|
|
263
|
+
const json = (await response.json()) as {
|
|
264
|
+
result?:
|
|
265
|
+
| {
|
|
266
|
+
matches?:
|
|
267
|
+
| { id?: string | undefined; metadata?: unknown; score?: number | undefined }[]
|
|
268
|
+
| undefined
|
|
269
|
+
}
|
|
270
|
+
| undefined
|
|
271
|
+
}
|
|
272
|
+
return (json.result?.matches ?? []).map((match) => ({
|
|
273
|
+
id: match.id ?? '',
|
|
274
|
+
metadata:
|
|
275
|
+
typeof match.metadata === 'object' && match.metadata !== null
|
|
276
|
+
? (match.metadata as Record<string, unknown>)
|
|
277
|
+
: undefined,
|
|
278
|
+
score: match.score ?? 0,
|
|
279
|
+
}))
|
|
280
|
+
},
|
|
281
|
+
async sync(entries, { dimensions, onProgress }) {
|
|
282
|
+
await ensureIndex(dimensions)
|
|
283
|
+
|
|
284
|
+
// Content-addressed ids: unchanged chunks keep their id across builds,
|
|
285
|
+
// so sync reduces to "upsert what's new, delete what's gone". Ids hash
|
|
286
|
+
// the bounded metadata so truncation stays stable across builds.
|
|
287
|
+
const local = new Map<string, RemoteEntry>()
|
|
288
|
+
for (const entry of entries) {
|
|
289
|
+
const bounded = {
|
|
290
|
+
...entry,
|
|
291
|
+
metadata: boundMetadata(sanitizeMetadata(entry.metadata)),
|
|
292
|
+
}
|
|
293
|
+
local.set(await contentId(bounded), bounded)
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
const remote = await listIds()
|
|
297
|
+
const toUpsert = remote
|
|
298
|
+
? [...local.keys()].filter((id) => !remote.has(id))
|
|
299
|
+
: [...local.keys()]
|
|
300
|
+
const toDelete = remote ? [...remote].filter((id) => !local.has(id)) : []
|
|
301
|
+
|
|
302
|
+
let upserted = 0
|
|
303
|
+
const batchSize = 500
|
|
304
|
+
for (let i = 0; i < toUpsert.length; i += batchSize) {
|
|
305
|
+
const batch = toUpsert.slice(i, i + batchSize)
|
|
306
|
+
const ndjson = batch
|
|
307
|
+
.map((id) => {
|
|
308
|
+
const entry = local.get(id)
|
|
309
|
+
if (!entry) return undefined
|
|
310
|
+
return JSON.stringify({
|
|
311
|
+
id,
|
|
312
|
+
metadata: entry.metadata,
|
|
313
|
+
values: Array.from(entry.vector),
|
|
314
|
+
})
|
|
315
|
+
})
|
|
316
|
+
.filter((line): line is string => line !== undefined)
|
|
317
|
+
.join('\n')
|
|
318
|
+
const response = await request(`/${index}/upsert`, {
|
|
319
|
+
method: 'POST',
|
|
320
|
+
contentType: 'application/x-ndjson',
|
|
321
|
+
body: ndjson,
|
|
322
|
+
})
|
|
323
|
+
if (!response.ok)
|
|
324
|
+
throw new Error(
|
|
325
|
+
`[vocs] Vectorize upsert failed (${response.status}): ${await safeText(response)}`,
|
|
326
|
+
)
|
|
327
|
+
upserted += batch.length
|
|
328
|
+
onProgress?.(upserted, toUpsert.length)
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// Vectorize caps delete_by_ids at 100 ids per request (error 40007).
|
|
332
|
+
for (let i = 0; i < toDelete.length; i += 100) {
|
|
333
|
+
const batch = toDelete.slice(i, i + 100)
|
|
334
|
+
const response = await request(`/${index}/delete_by_ids`, {
|
|
335
|
+
method: 'POST',
|
|
336
|
+
contentType: 'application/json',
|
|
337
|
+
body: JSON.stringify({ ids: batch }),
|
|
338
|
+
})
|
|
339
|
+
if (!response.ok)
|
|
340
|
+
throw new Error(
|
|
341
|
+
`[vocs] Vectorize delete failed (${response.status}): ${await safeText(response)}`,
|
|
342
|
+
)
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
return {
|
|
346
|
+
deleted: toDelete.length,
|
|
347
|
+
skipped: local.size - toUpsert.length,
|
|
348
|
+
upserted: toUpsert.length,
|
|
349
|
+
}
|
|
350
|
+
},
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
export declare namespace cloudflare {
|
|
355
|
+
type Options = {
|
|
356
|
+
/** Cloudflare account id. @default process.env.CLOUDFLARE_ACCOUNT_ID */
|
|
357
|
+
accountId?: string | undefined
|
|
358
|
+
/** API token with the `Vectorize:Edit` permission. @default process.env.CLOUDFLARE_API_TOKEN */
|
|
359
|
+
apiToken?: string | undefined
|
|
360
|
+
/** API base URL. @default 'https://api.cloudflare.com/client/v4' */
|
|
361
|
+
baseUrl?: string | undefined
|
|
362
|
+
/** Extra request headers. */
|
|
363
|
+
headers?: Record<string, string> | undefined
|
|
364
|
+
/** Vectorize index name (created on first sync). @default 'vocs-ai-search' */
|
|
365
|
+
index?: string | undefined
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
64
369
|
/** Creates a vector-store adapter from a custom definition. */
|
|
65
370
|
export function from(adapter: Adapter): Adapter {
|
|
66
371
|
return adapter
|
|
67
372
|
}
|
|
68
373
|
|
|
374
|
+
/** Vectorize metadata values must be string | number | boolean | string[]; drops the rest. */
|
|
375
|
+
function sanitizeMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
|
|
376
|
+
const out: Record<string, unknown> = {}
|
|
377
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
378
|
+
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean')
|
|
379
|
+
out[key] = value
|
|
380
|
+
else if (Array.isArray(value))
|
|
381
|
+
out[key] = value.filter((item): item is string => typeof item === 'string')
|
|
382
|
+
}
|
|
383
|
+
return out
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/** Vectorize caps metadata at 10KiB/vector; trims the unbounded `text` field to fit. */
|
|
387
|
+
function boundMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
|
|
388
|
+
const limit = 9216
|
|
389
|
+
let text = typeof metadata['text'] === 'string' ? metadata['text'] : undefined
|
|
390
|
+
for (;;) {
|
|
391
|
+
const out = text !== undefined ? { ...metadata, text } : metadata
|
|
392
|
+
const size = new TextEncoder().encode(JSON.stringify(out)).length
|
|
393
|
+
if (size <= limit || !text) return out
|
|
394
|
+
text = text.slice(0, Math.max(0, text.length - (size - limit)))
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Content-addressed vector id: SHA-256 over chunk id, metadata, and vector
|
|
400
|
+
* bytes. 64 hex chars — exactly Vectorize's 64-byte id cap.
|
|
401
|
+
*/
|
|
402
|
+
async function contentId(entry: RemoteEntry): Promise<string> {
|
|
403
|
+
const prefix = new TextEncoder().encode(`${entry.id}\0${JSON.stringify(entry.metadata)}\0`)
|
|
404
|
+
const vector = new Uint8Array(
|
|
405
|
+
entry.vector.buffer,
|
|
406
|
+
entry.vector.byteOffset,
|
|
407
|
+
entry.vector.byteLength,
|
|
408
|
+
)
|
|
409
|
+
const bytes = new Uint8Array(prefix.length + vector.length)
|
|
410
|
+
bytes.set(prefix, 0)
|
|
411
|
+
bytes.set(vector, prefix.length)
|
|
412
|
+
const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))
|
|
413
|
+
let hex = ''
|
|
414
|
+
for (const byte of digest) hex += byte.toString(16).padStart(2, '0')
|
|
415
|
+
return hex
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
async function safeText(response: Response): Promise<string> {
|
|
419
|
+
try {
|
|
420
|
+
return (await response.text()).slice(0, 200)
|
|
421
|
+
} catch {
|
|
422
|
+
return ''
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
69
426
|
/**
|
|
70
427
|
* Serialized vector matrix. `data` is base64 of the packed typed array; for
|
|
71
428
|
* `int8`, `scales` is base64 of one Float32 dequantization scale per vector.
|
|
@@ -194,7 +551,10 @@ export function search(store: Store, query: Float32Array, topK: number): SearchR
|
|
|
194
551
|
}
|
|
195
552
|
|
|
196
553
|
/** Resolves `'auto'` to a concrete format for the given runtime. */
|
|
197
|
-
export function resolveFormat(
|
|
554
|
+
export function resolveFormat(
|
|
555
|
+
format: StaticAdapter['format'],
|
|
556
|
+
runtime: 'server' | 'client',
|
|
557
|
+
): Format {
|
|
198
558
|
if (format !== 'auto') return format
|
|
199
559
|
return runtime === 'client' ? 'int8' : 'float32'
|
|
200
560
|
}
|
|
@@ -822,8 +822,12 @@ export function aiSearch(config: Config.Config): PluginOption {
|
|
|
822
822
|
|
|
823
823
|
const { rootDir, basePath } = config
|
|
824
824
|
const localEnabled = Boolean(config._localRetriever)
|
|
825
|
+
const vectorStore = config._localRetriever?.vectorStore
|
|
826
|
+
// Remote store: build-time work is syncing vectors into the database; no
|
|
827
|
+
// index artifact (server manifest or public browser file) is emitted.
|
|
828
|
+
const remoteStore = vectorStore?.target === 'remote'
|
|
825
829
|
const exposePublic =
|
|
826
|
-
ai?.runtime === 'client' ||
|
|
830
|
+
ai?.runtime === 'client' || (vectorStore?.target === 'static' && vectorStore.expose === true)
|
|
827
831
|
|
|
828
832
|
let mode: 'development' | 'production' = 'development'
|
|
829
833
|
let manifestPromise: Promise<Retriever.IndexManifest> | undefined
|
|
@@ -848,7 +852,9 @@ export function aiSearch(config: Config.Config): PluginOption {
|
|
|
848
852
|
}
|
|
849
853
|
await Retriever.saveIndex(config, manifest)
|
|
850
854
|
logger.info(
|
|
851
|
-
|
|
855
|
+
remoteStore
|
|
856
|
+
? `AI search index synced (${manifest.vectorStore.count} chunks, remote vector store).`
|
|
857
|
+
: `AI search index built (${manifest.vectorStore.count} chunks, ${manifest.vectorStore.format}).`,
|
|
852
858
|
{ timestamp: true },
|
|
853
859
|
)
|
|
854
860
|
return manifest
|
|
@@ -909,8 +915,10 @@ export function aiSearch(config: Config.Config): PluginOption {
|
|
|
909
915
|
async load(id) {
|
|
910
916
|
if (id === resolvedServerManifestId) {
|
|
911
917
|
// Dev never builds; the server loads the manifest written by
|
|
912
|
-
// `vocs embeddings generate` from disk instead.
|
|
913
|
-
|
|
918
|
+
// `vocs embeddings generate` from disk instead. A remote store needs
|
|
919
|
+
// no baked manifest either — the server queries the database directly.
|
|
920
|
+
if (!localEnabled || remoteStore || mode === 'development' || skipSeeding())
|
|
921
|
+
return emptyManifestModule
|
|
914
922
|
const manifest = await buildManifest()
|
|
915
923
|
// Emit as a gzipped asset. Inlining the JSON into the chunk bloats
|
|
916
924
|
// server bundles (manifests can be tens of MB).
|
|
@@ -952,7 +960,7 @@ export const getAiSearchManifest = async () =>
|
|
|
952
960
|
await fs.writeFile(path.join(dir, `ai-search-index-${publicHash}.json`), json, 'utf-8')
|
|
953
961
|
|
|
954
962
|
const bytes = Buffer.byteLength(json)
|
|
955
|
-
const max =
|
|
963
|
+
const max = vectorStore?.target === 'static' ? vectorStore.maxClientBytes : undefined
|
|
956
964
|
if (max && bytes > max)
|
|
957
965
|
logger.warn(
|
|
958
966
|
`AI search client index is ${(bytes / 1e6).toFixed(1)}MB, exceeds maxClientBytes (${(max / 1e6).toFixed(1)}MB). Consider int8 format or fewer chunks.`,
|