veryfront 0.1.994 → 0.1.995
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/deno.js +1 -1
- package/esm/src/embedding/index.d.ts +1 -1
- package/esm/src/embedding/index.d.ts.map +1 -1
- package/esm/src/embedding/rag-store.d.ts.map +1 -1
- package/esm/src/embedding/rag-store.js +37 -0
- package/esm/src/embedding/types.d.ts +7 -0
- package/esm/src/embedding/types.d.ts.map +1 -1
- package/esm/src/embedding/upload-loader.d.ts +5 -4
- package/esm/src/embedding/upload-loader.d.ts.map +1 -1
- package/esm/src/embedding/upload-loader.js +5 -4
- package/esm/src/embedding/veryfront-cloud/rag-store.d.ts.map +1 -1
- package/esm/src/embedding/veryfront-cloud/rag-store.js +45 -5
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
package/esm/deno.js
CHANGED
|
@@ -24,6 +24,6 @@ export { loadUpload } from "./upload-loader.js";
|
|
|
24
24
|
export { useUploads } from "./react/use-uploads.js";
|
|
25
25
|
export type { UseUploadsOptions, UseUploadsResult } from "./react/use-uploads.js";
|
|
26
26
|
export { clearEmbeddingProviders, registerEmbeddingProvider, resolveEmbeddingModel, } from "./resolve.js";
|
|
27
|
-
export type { ChunkOptions, Embedding, EmbeddingConfig, RagChunk, RagDocumentMeta, RagSearchOptions, RagSearchResult, RagStore, RagStoreBackend, RagStoreConfig, RagStoreData, SearchOptions, SearchResult, VectorStore, VectorStoreConfig, } from "./types.js";
|
|
27
|
+
export type { ChunkOptions, Embedding, EmbeddingConfig, RagChunk, RagDocumentMeta, RagRefreshOptions, RagSearchOptions, RagSearchResult, RagStore, RagStoreBackend, RagStoreConfig, RagStoreData, SearchOptions, SearchResult, VectorStore, VectorStoreConfig, } from "./types.js";
|
|
28
28
|
export type { UploadAuthorizationResult, UploadAuthorize, UploadHandlerAuthConfig, UploadHandlerConfig, } from "./upload-handler.js";
|
|
29
29
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/embedding/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,gBAAgB,IAAI,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EACL,uBAAuB,EACvB,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,QAAQ,EACR,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,QAAQ,EACR,eAAe,EACf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,WAAW,EACX,iBAAiB,GAClB,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,yBAAyB,EACzB,eAAe,EACf,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/embedding/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,gBAAgB,IAAI,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EACL,uBAAuB,EACvB,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,QAAQ,EACR,eAAe,EACf,iBAAiB,EACjB,gBAAgB,EAChB,eAAe,EACf,QAAQ,EACR,eAAe,EACf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,WAAW,EACX,iBAAiB,GAClB,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,yBAAyB,EACzB,eAAe,EACf,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rag-store.d.ts","sourceRoot":"","sources":["../../../src/src/embedding/rag-store.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"rag-store.d.ts","sourceRoot":"","sources":["../../../src/src/embedding/rag-store.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAMV,QAAQ,EAER,cAAc,EAEf,MAAM,YAAY,CAAC;AA2HpB;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,QAAQ,CAyCzD"}
|
|
@@ -126,6 +126,13 @@ export function ragStore(config) {
|
|
|
126
126
|
ingest(title, text, meta) {
|
|
127
127
|
return getStore().ingest(title, text, meta);
|
|
128
128
|
},
|
|
129
|
+
refreshDocument(id, text, meta) {
|
|
130
|
+
const store = getStore();
|
|
131
|
+
if (!store.refreshDocument) {
|
|
132
|
+
throw INVALID_ARGUMENT.create({ detail: "RAG store does not support document refresh" });
|
|
133
|
+
}
|
|
134
|
+
return store.refreshDocument(id, text, meta);
|
|
135
|
+
},
|
|
129
136
|
search(query, options) {
|
|
130
137
|
return getStore().search(query, options);
|
|
131
138
|
},
|
|
@@ -404,6 +411,36 @@ function createLocalJsonRagStore(config) {
|
|
|
404
411
|
return documentId;
|
|
405
412
|
});
|
|
406
413
|
},
|
|
414
|
+
async refreshDocument(id, text, meta) {
|
|
415
|
+
return withLock(async () => {
|
|
416
|
+
const data = await load();
|
|
417
|
+
const document = data.documents.find((doc) => doc.id === id);
|
|
418
|
+
if (!document) {
|
|
419
|
+
throw INVALID_ARGUMENT.create({ detail: `RAG document not found: ${id}` });
|
|
420
|
+
}
|
|
421
|
+
if (text.length > MAX_TEXT_LENGTH) {
|
|
422
|
+
throw INVALID_ARGUMENT.create({
|
|
423
|
+
detail: `Upload text exceeds ${MAX_TEXT_LENGTH / 1024 / 1024} MB limit`,
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
const chunks = await chunk(text, chunkOptions);
|
|
427
|
+
if (chunks.length === 0) {
|
|
428
|
+
throw INVALID_ARGUMENT.create({ detail: "Upload contains no extractable text" });
|
|
429
|
+
}
|
|
430
|
+
document.title = meta?.title ?? document.title;
|
|
431
|
+
document.source = meta?.source ?? document.source;
|
|
432
|
+
document.type = meta?.type ?? document.type;
|
|
433
|
+
data.chunks = data.chunks.filter((chunk) => chunk.documentId !== id);
|
|
434
|
+
data.chunks.push(...chunks.map((chunkText, index) => ({
|
|
435
|
+
id: dntShim.crypto.randomUUID(),
|
|
436
|
+
documentId: id,
|
|
437
|
+
text: chunkText,
|
|
438
|
+
embedding: [],
|
|
439
|
+
index,
|
|
440
|
+
})));
|
|
441
|
+
await save(data);
|
|
442
|
+
});
|
|
443
|
+
},
|
|
407
444
|
async search(query, options) {
|
|
408
445
|
if (!query.trim())
|
|
409
446
|
return [];
|
|
@@ -102,12 +102,19 @@ export interface RagSearchOptions {
|
|
|
102
102
|
topK?: number;
|
|
103
103
|
threshold?: number;
|
|
104
104
|
}
|
|
105
|
+
/** Options accepted when refreshing an existing rag document. */
|
|
106
|
+
export interface RagRefreshOptions {
|
|
107
|
+
title?: string;
|
|
108
|
+
source?: string;
|
|
109
|
+
type?: string;
|
|
110
|
+
}
|
|
105
111
|
/** Public API contract for rag store. */
|
|
106
112
|
export interface RagStore {
|
|
107
113
|
ingest(title: string, text: string, meta?: {
|
|
108
114
|
source?: string;
|
|
109
115
|
type?: string;
|
|
110
116
|
}): Promise<string>;
|
|
117
|
+
refreshDocument?(id: string, text: string, meta?: RagRefreshOptions): Promise<void>;
|
|
111
118
|
search(query: string, options?: RagSearchOptions): Promise<RagSearchResult[]>;
|
|
112
119
|
listDocuments(): Promise<RagDocumentMeta[]>;
|
|
113
120
|
removeDocument(id: string): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/embedding/types.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,yCAAyC;AACzC,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,8DAA8D;IAC9D,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,yFAAyF;IACzF,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CACjD;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,0CAA0C;AAC1C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,SAAS,CAAC;CACrB;AAED,kCAAkC;AAClC,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,mCAAmC;AACnC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,4CAA4C;AAC5C,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1E,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACxE,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAMD,iDAAiD;AACjD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,MAAM,EAAE,QAAQ,EAAE,CAAC;CACpB;AAED,iDAAiD;AACjD,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,YAAY,GAAG,iBAAiB,CAAC;AAExE,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,uCAAuC;AACvC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAChG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAC9E,aAAa,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAC5C,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAClC"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/embedding/types.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,yCAAyC;AACzC,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,8DAA8D;IAC9D,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,yFAAyF;IACzF,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CACjD;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,0CAA0C;AAC1C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,SAAS,CAAC;CACrB;AAED,kCAAkC;AAClC,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,mCAAmC;AACnC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,4CAA4C;AAC5C,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1E,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACxE,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAMD,iDAAiD;AACjD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,MAAM,EAAE,QAAQ,EAAE,CAAC;CACpB;AAED,iDAAiD;AACjD,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,YAAY,GAAG,iBAAiB,CAAC;AAExE,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,uCAAuC;AACvC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iEAAiE;AACjE,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAChG,eAAe,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpF,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAC9E,aAAa,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAC5C,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAClC"}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Extracts
|
|
2
|
+
* Extracts embedding-ready text or Markdown from upload formats.
|
|
3
3
|
*
|
|
4
4
|
* Text and CSV are handled inline (CSV uses a RAG-optimized format that
|
|
5
5
|
* denormalizes headers into every row). All other formats (PDF, DOCX, XLSX,
|
|
6
|
-
* PPTX, HTML, RTF, EPUB, etc.) are delegated to the `DocumentExtractor`
|
|
7
|
-
* contract, which owns kreuzberg and
|
|
6
|
+
* PPTX, HTML, RTF, EPUB, etc.) are delegated to the `DocumentExtractor`
|
|
7
|
+
* extension contract, which owns kreuzberg, Markdown extraction, and Worker
|
|
8
|
+
* isolation on Deno.
|
|
8
9
|
*
|
|
9
10
|
* @example
|
|
10
11
|
* ```ts
|
|
11
|
-
* const
|
|
12
|
+
* const markdown = await loadUpload(buffer, "application/pdf");
|
|
12
13
|
* ```
|
|
13
14
|
*/
|
|
14
15
|
export declare function loadUpload(buffer: ArrayBuffer, mimeType: string): Promise<string>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload-loader.d.ts","sourceRoot":"","sources":["../../../src/src/embedding/upload-loader.ts"],"names":[],"mappings":"AAGA
|
|
1
|
+
{"version":3,"file":"upload-loader.d.ts","sourceRoot":"","sources":["../../../src/src/embedding/upload-loader.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;GAaG;AACH,wBAAsB,UAAU,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BvF"}
|
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import { tryResolve } from "../extensions/contracts.js";
|
|
2
2
|
/**
|
|
3
|
-
* Extracts
|
|
3
|
+
* Extracts embedding-ready text or Markdown from upload formats.
|
|
4
4
|
*
|
|
5
5
|
* Text and CSV are handled inline (CSV uses a RAG-optimized format that
|
|
6
6
|
* denormalizes headers into every row). All other formats (PDF, DOCX, XLSX,
|
|
7
|
-
* PPTX, HTML, RTF, EPUB, etc.) are delegated to the `DocumentExtractor`
|
|
8
|
-
* contract, which owns kreuzberg and
|
|
7
|
+
* PPTX, HTML, RTF, EPUB, etc.) are delegated to the `DocumentExtractor`
|
|
8
|
+
* extension contract, which owns kreuzberg, Markdown extraction, and Worker
|
|
9
|
+
* isolation on Deno.
|
|
9
10
|
*
|
|
10
11
|
* @example
|
|
11
12
|
* ```ts
|
|
12
|
-
* const
|
|
13
|
+
* const markdown = await loadUpload(buffer, "application/pdf");
|
|
13
14
|
* ```
|
|
14
15
|
*/
|
|
15
16
|
export async function loadUpload(buffer, mimeType) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rag-store.d.ts","sourceRoot":"","sources":["../../../../src/src/embedding/veryfront-cloud/rag-store.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"rag-store.d.ts","sourceRoot":"","sources":["../../../../src/src/embedding/veryfront-cloud/rag-store.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAKV,QAAQ,EACR,cAAc,EACf,MAAM,aAAa,CAAC;AA4ErB,KAAK,2BAA2B,GAAG,cAAc,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AA8kBtE,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,2BAA2B,GAAG,QAAQ,CAuI1F"}
|
|
@@ -56,12 +56,26 @@ function buildDocumentFilePath(documentId, type) {
|
|
|
56
56
|
const extension = type?.trim().toLowerCase().replace(/[^a-z0-9]+/g, "");
|
|
57
57
|
return `${DOCUMENTS_DIR}/${documentId}.${extension || "txt"}`;
|
|
58
58
|
}
|
|
59
|
+
function buildRefreshDocumentFilePath(documentId, type) {
|
|
60
|
+
const extension = type?.trim().toLowerCase().replace(/[^a-z0-9]+/g, "");
|
|
61
|
+
return `${DOCUMENTS_DIR}/${documentId}.refresh-${dntShim.crypto.randomUUID()}.${extension || "txt"}`;
|
|
62
|
+
}
|
|
59
63
|
function isRecord(value) {
|
|
60
64
|
return typeof value === "object" && value !== null;
|
|
61
65
|
}
|
|
62
66
|
function toStringValue(value, fallback = "") {
|
|
63
67
|
return typeof value === "string" ? value : fallback;
|
|
64
68
|
}
|
|
69
|
+
function toPublicRagDocumentMeta(document) {
|
|
70
|
+
return {
|
|
71
|
+
id: document.id,
|
|
72
|
+
title: document.title,
|
|
73
|
+
source: document.source,
|
|
74
|
+
type: document.type,
|
|
75
|
+
createdAt: document.createdAt,
|
|
76
|
+
url: document.url,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
65
79
|
function buildDocumentChunks(sourceText, chunkTexts, metadata) {
|
|
66
80
|
let searchStart = 0;
|
|
67
81
|
return chunkTexts.map((content, index) => {
|
|
@@ -176,6 +190,7 @@ async function listRagDocuments(context) {
|
|
|
176
190
|
source: doc.source,
|
|
177
191
|
type: doc.type,
|
|
178
192
|
createdAt: new Date(doc.created_at).getTime(),
|
|
193
|
+
filePath: typeof doc.metadata?.filePath === "string" ? doc.metadata.filePath : undefined,
|
|
179
194
|
}));
|
|
180
195
|
}
|
|
181
196
|
async function upsertRagDocument(context, document) {
|
|
@@ -197,6 +212,28 @@ async function deleteRagDocument(context, documentId) {
|
|
|
197
212
|
// Document ingestion (chunks + embeddings + server-side record)
|
|
198
213
|
// ---------------------------------------------------------------------------
|
|
199
214
|
async function ingestDocument(context, config, title, text, meta) {
|
|
215
|
+
const documentId = dntShim.crypto.randomUUID();
|
|
216
|
+
await writeDocumentContent(context, config, documentId, title, text, meta);
|
|
217
|
+
return documentId;
|
|
218
|
+
}
|
|
219
|
+
async function refreshCloudDocument(context, config, documentId, text, meta) {
|
|
220
|
+
const documents = await listRagDocuments(context);
|
|
221
|
+
const existing = documents.find((doc) => doc.id === documentId);
|
|
222
|
+
if (!existing) {
|
|
223
|
+
throw INVALID_ARGUMENT.create({ detail: `RAG document not found: ${documentId}` });
|
|
224
|
+
}
|
|
225
|
+
const type = meta?.type ?? existing.type;
|
|
226
|
+
const previousFilePath = existing.filePath ?? buildDocumentFilePath(documentId, existing.type);
|
|
227
|
+
const filePath = buildRefreshDocumentFilePath(documentId, type);
|
|
228
|
+
await writeDocumentContent(context, config, documentId, meta?.title ?? existing.title, text, {
|
|
229
|
+
source: meta?.source ?? existing.source,
|
|
230
|
+
type,
|
|
231
|
+
}, { filePath });
|
|
232
|
+
if (previousFilePath !== filePath) {
|
|
233
|
+
await deleteFileChunks(context, previousFilePath);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
async function writeDocumentContent(context, config, documentId, title, text, meta, options) {
|
|
200
237
|
if (text.length > MAX_TEXT_LENGTH) {
|
|
201
238
|
throw INVALID_ARGUMENT.create({
|
|
202
239
|
detail: `Upload text exceeds ${MAX_TEXT_LENGTH / 1024 / 1024} MB limit`,
|
|
@@ -206,8 +243,7 @@ async function ingestDocument(context, config, title, text, meta) {
|
|
|
206
243
|
if (chunkTexts.length === 0) {
|
|
207
244
|
throw INVALID_ARGUMENT.create({ detail: "Upload contains no extractable text" });
|
|
208
245
|
}
|
|
209
|
-
const
|
|
210
|
-
const filePath = buildDocumentFilePath(documentId, meta?.type);
|
|
246
|
+
const filePath = options?.filePath ?? buildDocumentFilePath(documentId, meta?.type);
|
|
211
247
|
const embedder = createEmbedder(config);
|
|
212
248
|
const vectors = await embedder.embedMany(chunkTexts);
|
|
213
249
|
const dimension = toSupportedDimension(vectors[0]?.length ?? 0);
|
|
@@ -243,7 +279,6 @@ async function ingestDocument(context, config, title, text, meta) {
|
|
|
243
279
|
type: meta?.type ?? "",
|
|
244
280
|
metadata: { filePath },
|
|
245
281
|
});
|
|
246
|
-
return documentId;
|
|
247
282
|
}
|
|
248
283
|
function createEmbedder(config) {
|
|
249
284
|
return embedding({
|
|
@@ -340,6 +375,10 @@ export function createVeryfrontCloudRagStore(config) {
|
|
|
340
375
|
const context = getCloudStoreContext(config);
|
|
341
376
|
return ingestDocument(context, config, title, text, meta);
|
|
342
377
|
},
|
|
378
|
+
async refreshDocument(id, text, meta) {
|
|
379
|
+
const context = getCloudStoreContext(config);
|
|
380
|
+
await refreshCloudDocument(context, config, id, text, meta);
|
|
381
|
+
},
|
|
343
382
|
async search(query, options) {
|
|
344
383
|
if (!query.trim())
|
|
345
384
|
return [];
|
|
@@ -376,7 +415,8 @@ export function createVeryfrontCloudRagStore(config) {
|
|
|
376
415
|
},
|
|
377
416
|
async listDocuments() {
|
|
378
417
|
const context = getCloudStoreContext(config);
|
|
379
|
-
|
|
418
|
+
const documents = await listRagDocuments(context);
|
|
419
|
+
return documents.map(toPublicRagDocumentMeta);
|
|
380
420
|
},
|
|
381
421
|
async removeDocument(id) {
|
|
382
422
|
const context = getCloudStoreContext(config);
|
|
@@ -387,7 +427,7 @@ export function createVeryfrontCloudRagStore(config) {
|
|
|
387
427
|
await deleteRagDocument(context, id);
|
|
388
428
|
// Best-effort cleanup of file chunks
|
|
389
429
|
if (target) {
|
|
390
|
-
const filePath = buildDocumentFilePath(id, target.type);
|
|
430
|
+
const filePath = target.filePath ?? buildDocumentFilePath(id, target.type);
|
|
391
431
|
try {
|
|
392
432
|
await deleteFileChunks(context, filePath);
|
|
393
433
|
}
|