ruvector 0.2.30 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +540 -32
- package/bin/mcp-server.js +4035 -3854
- package/dist/core/embedding-provenance.d.ts +145 -0
- package/dist/core/embedding-provenance.d.ts.map +1 -0
- package/dist/core/embedding-provenance.js +258 -0
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -0
- package/dist/core/intelligence-engine.d.ts +65 -4
- package/dist/core/intelligence-engine.d.ts.map +1 -1
- package/dist/core/intelligence-engine.js +149 -12
- package/dist/core/onnx/bundled-parallel.mjs +24 -19
- package/dist/core/onnx/loader.js +31 -4
- package/dist/core/onnx-embedder.d.ts +42 -1
- package/dist/core/onnx-embedder.d.ts.map +1 -1
- package/dist/core/onnx-embedder.js +116 -11
- package/dist/core/onnx-optimized.d.ts +8 -1
- package/dist/core/onnx-optimized.d.ts.map +1 -1
- package/dist/core/onnx-optimized.js +41 -6
- package/package.json +5 -4
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding provenance — ADR-210 D0 cross-cutting invariant.
|
|
3
|
+
*
|
|
4
|
+
* Every persisted vector store written through the embedding path records
|
|
5
|
+
* `{ embedderKind, modelId, dimension, normalize, prefixPolicy }`. Inserts
|
|
6
|
+
* whose provenance does not match the store's recorded provenance are
|
|
7
|
+
* REFUSED (clear error naming both sides), never coerced. Stores that
|
|
8
|
+
* predate provenance metadata are treated as legacy hash stores and open
|
|
9
|
+
* read-only for vector writes until re-embedded (`ruvector hooks reembed`).
|
|
10
|
+
*
|
|
11
|
+
* This module is the single source of truth for:
|
|
12
|
+
* - the provenance record type + compare/refuse logic (D0),
|
|
13
|
+
* - legacy-default derivation for pre-ADR-210 stores (D0),
|
|
14
|
+
* - per-model query/passage prefix policies (D4),
|
|
15
|
+
* - rollout flag resolution: RUVECTOR_EMBEDDER / RUVECTOR_ONNX /
|
|
16
|
+
* RUVECTOR_REEMBED (D5),
|
|
17
|
+
* - the once-per-process loud hash-fallback warning (D1).
|
|
18
|
+
*/
|
|
19
|
+
export type PrefixPolicy = 'none' | 'required' | 'query-recommended';
|
|
20
|
+
export type EmbedTextKind = 'query' | 'passage';
|
|
21
|
+
/** Embedder identity classes. `modelId` carries the exact model. */
|
|
22
|
+
export type EmbedderKind = 'onnx-minilm' | 'onnx' | 'hash';
|
|
23
|
+
export interface EmbeddingProvenance {
|
|
24
|
+
/** Embedder family that produced the vectors. */
|
|
25
|
+
embedderKind: EmbedderKind | string;
|
|
26
|
+
/** Exact model id (e.g. 'all-MiniLM-L6-v2'); null for the hash embedder. */
|
|
27
|
+
modelId: string | null;
|
|
28
|
+
/** Vector dimension. */
|
|
29
|
+
dimension: number;
|
|
30
|
+
/** Whether vectors were L2-normalized at embed time. */
|
|
31
|
+
normalize: boolean;
|
|
32
|
+
/** Prefix convention the texts were embedded under (D4). */
|
|
33
|
+
prefixPolicy: PrefixPolicy;
|
|
34
|
+
}
|
|
35
|
+
export type EmbedderSelection = 'auto' | 'minilm' | 'hash';
|
|
36
|
+
export type ReembedPolicy = 'refuse' | 'warn' | 'auto';
|
|
37
|
+
export interface ModelPrefixSpec {
|
|
38
|
+
prefixPolicy: PrefixPolicy;
|
|
39
|
+
queryPrefix: string;
|
|
40
|
+
passagePrefix: string;
|
|
41
|
+
}
|
|
42
|
+
/** BGE en v1.5 documented query instruction (short query → long passage). */
|
|
43
|
+
export declare const BGE_QUERY_INSTRUCTION = "Represent this sentence for searching relevant passages: ";
|
|
44
|
+
/**
|
|
45
|
+
* Prefix conventions per model card:
|
|
46
|
+
* - all-MiniLM-L6-v2 / L12: general semantic search, NO prefixes.
|
|
47
|
+
* - e5-small-v2: REQUIRES 'query: ' / 'passage: ' (quality degrades without).
|
|
48
|
+
* - bge-small/base-en-v1.5: query instruction recommended for retrieval;
|
|
49
|
+
* passages need no instruction.
|
|
50
|
+
* - gte-small: no prefixes documented.
|
|
51
|
+
*/
|
|
52
|
+
export declare const MODEL_PREFIXES: Record<string, ModelPrefixSpec>;
|
|
53
|
+
/**
|
|
54
|
+
* Prefix spec for a model; unknown models get the no-prefix policy.
|
|
55
|
+
* Own-property lookup only: a hostile model id like '__proto__' or
|
|
56
|
+
* 'constructor' must resolve to NO_PREFIX, not to a prototype member
|
|
57
|
+
* (ADR-210 security pass).
|
|
58
|
+
*/
|
|
59
|
+
export declare function getModelPrefixSpec(modelId: string | null | undefined): ModelPrefixSpec;
|
|
60
|
+
/**
|
|
61
|
+
* Pure prefix application (D4): the exact text handed to the tokenizer for a
|
|
62
|
+
* query/passage embed of `text` under `modelId`'s registered policy.
|
|
63
|
+
* MiniLM applies NO prefix on either entry point (acceptance gates 6–7).
|
|
64
|
+
*/
|
|
65
|
+
export declare function prefixText(modelId: string | null | undefined, kind: EmbedTextKind, text: string): string;
|
|
66
|
+
/** Embedder family for an ONNX model id. */
|
|
67
|
+
export declare function embedderKindForModel(modelId: string | null | undefined): EmbedderKind;
|
|
68
|
+
/**
|
|
69
|
+
* Legacy default for stores that predate provenance metadata: hash-embedded,
|
|
70
|
+
* un-normalized as far as we can prove, no prefixes. Such stores open
|
|
71
|
+
* READ-ONLY for vector writes until re-embedded.
|
|
72
|
+
*/
|
|
73
|
+
export declare function legacyHashProvenance(dimension?: number): EmbeddingProvenance;
|
|
74
|
+
/** Human-readable one-liner for error messages. */
|
|
75
|
+
export declare function describeProvenance(p: EmbeddingProvenance): string;
|
|
76
|
+
/** Field names on which two provenance records disagree (empty = match). */
|
|
77
|
+
export declare function compareProvenance(a: EmbeddingProvenance, b: EmbeddingProvenance): string[];
|
|
78
|
+
/** Upper bound accepted for a provenance dimension read from disk. */
|
|
79
|
+
export declare const MAX_PROVENANCE_DIMENSION = 65536;
|
|
80
|
+
/**
|
|
81
|
+
* Sanitize a provenance record read from DISK (a `.meta.json` sidecar or
|
|
82
|
+
* `intelligence.json`). On-disk JSON is untrusted input: a malformed or
|
|
83
|
+
* adversarial record must never crash the caller. Anything that is not a
|
|
84
|
+
* plausibly-valid record is treated as ABSENT (returns null), which callers
|
|
85
|
+
* already handle as the no-provenance / legacy path — conservative for a
|
|
86
|
+
* corrupted stamp (the store degrades to read-only for vector writes rather
|
|
87
|
+
* than accepting writes under a fabricated identity).
|
|
88
|
+
*/
|
|
89
|
+
export declare function sanitizeProvenance(value: unknown): EmbeddingProvenance | null;
|
|
90
|
+
/** Thrown when an insert's provenance does not match the store's (D0). */
|
|
91
|
+
export declare class ProvenanceMismatchError extends Error {
|
|
92
|
+
code: string;
|
|
93
|
+
store: EmbeddingProvenance;
|
|
94
|
+
active: EmbeddingProvenance;
|
|
95
|
+
mismatches: string[];
|
|
96
|
+
constructor(store: EmbeddingProvenance, active: EmbeddingProvenance, mismatches: string[], storeName: string);
|
|
97
|
+
}
|
|
98
|
+
/** Refuse mismatched inserts with an error naming both sides (D0). */
|
|
99
|
+
export declare function assertProvenanceMatch(store: EmbeddingProvenance, active: EmbeddingProvenance, storeName?: string): void;
|
|
100
|
+
/**
|
|
101
|
+
* Resolve RUVECTOR_EMBEDDER / RUVECTOR_ONNX.
|
|
102
|
+
* Precedence: RUVECTOR_EMBEDDER wins when both are set; RUVECTOR_ONNX=0 is
|
|
103
|
+
* shorthand for `hash`, =1 for `minilm`. Unrecognized values fall back to
|
|
104
|
+
* 'auto' (MiniLM when loadable, loud hash fallback otherwise).
|
|
105
|
+
*/
|
|
106
|
+
export declare function resolveEmbedderSelection(env?: NodeJS.ProcessEnv): EmbedderSelection;
|
|
107
|
+
/**
|
|
108
|
+
* Resolve RUVECTOR_REEMBED: what happens when opening a store whose
|
|
109
|
+
* provenance mismatches the active embedder.
|
|
110
|
+
* refuse (default) — error;
|
|
111
|
+
* warn — open read-only with a single warning;
|
|
112
|
+
* auto — re-embed in place when source text exists, refuse otherwise.
|
|
113
|
+
*/
|
|
114
|
+
export declare function resolveReembedPolicy(env?: NodeJS.ProcessEnv): ReembedPolicy;
|
|
115
|
+
/**
|
|
116
|
+
* Emit exactly ONE stderr warning per process the first time the hash
|
|
117
|
+
* fallback serves an embed that the ONNX embedder was supposed to handle
|
|
118
|
+
* (acceptance gate 2). Returns true when the warning was emitted by this call.
|
|
119
|
+
*/
|
|
120
|
+
export declare function warnHashFallbackOnce(reason?: string): boolean;
|
|
121
|
+
/** Whether the once-per-process fallback warning has fired. */
|
|
122
|
+
export declare function hashFallbackWarned(): boolean;
|
|
123
|
+
/** Test hook: reset the once-per-process warning latch. */
|
|
124
|
+
export declare function resetHashFallbackWarningForTests(): void;
|
|
125
|
+
declare const _default: {
|
|
126
|
+
MODEL_PREFIXES: Record<string, ModelPrefixSpec>;
|
|
127
|
+
BGE_QUERY_INSTRUCTION: string;
|
|
128
|
+
getModelPrefixSpec: typeof getModelPrefixSpec;
|
|
129
|
+
prefixText: typeof prefixText;
|
|
130
|
+
embedderKindForModel: typeof embedderKindForModel;
|
|
131
|
+
legacyHashProvenance: typeof legacyHashProvenance;
|
|
132
|
+
describeProvenance: typeof describeProvenance;
|
|
133
|
+
compareProvenance: typeof compareProvenance;
|
|
134
|
+
sanitizeProvenance: typeof sanitizeProvenance;
|
|
135
|
+
MAX_PROVENANCE_DIMENSION: number;
|
|
136
|
+
ProvenanceMismatchError: typeof ProvenanceMismatchError;
|
|
137
|
+
assertProvenanceMatch: typeof assertProvenanceMatch;
|
|
138
|
+
resolveEmbedderSelection: typeof resolveEmbedderSelection;
|
|
139
|
+
resolveReembedPolicy: typeof resolveReembedPolicy;
|
|
140
|
+
warnHashFallbackOnce: typeof warnHashFallbackOnce;
|
|
141
|
+
hashFallbackWarned: typeof hashFallbackWarned;
|
|
142
|
+
resetHashFallbackWarningForTests: typeof resetHashFallbackWarningForTests;
|
|
143
|
+
};
|
|
144
|
+
export default _default;
|
|
145
|
+
//# sourceMappingURL=embedding-provenance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-provenance.d.ts","sourceRoot":"","sources":["../../src/core/embedding-provenance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAMH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,UAAU,GAAG,mBAAmB,CAAC;AACrE,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,SAAS,CAAC;AAEhD,oEAAoE;AACpE,MAAM,MAAM,YAAY,GAAG,aAAa,GAAG,MAAM,GAAG,MAAM,CAAC;AAE3D,MAAM,WAAW,mBAAmB;IAClC,iDAAiD;IACjD,YAAY,EAAE,YAAY,GAAG,MAAM,CAAC;IACpC,4EAA4E;IAC5E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,SAAS,EAAE,OAAO,CAAC;IACnB,4DAA4D;IAC5D,YAAY,EAAE,YAAY,CAAC;CAC5B;AAED,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAC3D,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;AAMvD,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,YAAY,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;CACvB;AAID,6EAA6E;AAC7E,eAAO,MAAM,qBAAqB,8DAC2B,CAAC;AAE9D;;;;;;;GAOG;AACH,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAO1D,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,eAAe,CAKtF;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAIxG;AAED,4CAA4C;AAC5C,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,YAAY,CAErF;AAMD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,GAAE,MAAY,GAAG,mBAAmB,CAEjF;AAED,mDAAmD;AACnD,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,mBAAmB,GAAG,MAAM,CAGjE;AAED,4EAA4E;AAC5E,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,mBAAmB,GAAG,MAAM,EAAE,CAQ1F;AAED,sEAAsE;AACtE,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAI9C;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,mBAAmB,GAAG,IAAI,CAoC7E;AAED,0EAA0E;AAC1E,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,IAAI,SAA8B;IAClC,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,mBAAmB,CAAC;IAC5B,UAAU,EAAE,MAAM,EAAE,CAAC;gBAET,KAAK,EAAE,mBAAmB,EAAE,MAAM,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM;CAa7G;AAED,sEAAsE;AACtE,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,mBAAmB,EAC1B,MAAM,EAAE,mBAAmB,EAC3B,SAAS,GAAE,MAAuB,GACjC,IAAI,CAKN;AAMD;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,iBAAiB,CAOhG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,aAAa,CAIxF;AAQD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAW7D;AAED,+DAA+D;AAC/D,wBAAgB,kBAAkB,IAAI,OAAO,CAE5C;AAED,2DAA2D;AAC3D,wBAAgB,gCAAgC,IAAI,IAAI,CAEvD;;;;;;;;;;;;;;;;;;;;AAED,wBAkBE"}
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Embedding provenance — ADR-210 D0 cross-cutting invariant.
|
|
4
|
+
*
|
|
5
|
+
* Every persisted vector store written through the embedding path records
|
|
6
|
+
* `{ embedderKind, modelId, dimension, normalize, prefixPolicy }`. Inserts
|
|
7
|
+
* whose provenance does not match the store's recorded provenance are
|
|
8
|
+
* REFUSED (clear error naming both sides), never coerced. Stores that
|
|
9
|
+
* predate provenance metadata are treated as legacy hash stores and open
|
|
10
|
+
* read-only for vector writes until re-embedded (`ruvector hooks reembed`).
|
|
11
|
+
*
|
|
12
|
+
* This module is the single source of truth for:
|
|
13
|
+
* - the provenance record type + compare/refuse logic (D0),
|
|
14
|
+
* - legacy-default derivation for pre-ADR-210 stores (D0),
|
|
15
|
+
* - per-model query/passage prefix policies (D4),
|
|
16
|
+
* - rollout flag resolution: RUVECTOR_EMBEDDER / RUVECTOR_ONNX /
|
|
17
|
+
* RUVECTOR_REEMBED (D5),
|
|
18
|
+
* - the once-per-process loud hash-fallback warning (D1).
|
|
19
|
+
*/
|
|
20
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
exports.ProvenanceMismatchError = exports.MAX_PROVENANCE_DIMENSION = exports.MODEL_PREFIXES = exports.BGE_QUERY_INSTRUCTION = void 0;
|
|
22
|
+
exports.getModelPrefixSpec = getModelPrefixSpec;
|
|
23
|
+
exports.prefixText = prefixText;
|
|
24
|
+
exports.embedderKindForModel = embedderKindForModel;
|
|
25
|
+
exports.legacyHashProvenance = legacyHashProvenance;
|
|
26
|
+
exports.describeProvenance = describeProvenance;
|
|
27
|
+
exports.compareProvenance = compareProvenance;
|
|
28
|
+
exports.sanitizeProvenance = sanitizeProvenance;
|
|
29
|
+
exports.assertProvenanceMatch = assertProvenanceMatch;
|
|
30
|
+
exports.resolveEmbedderSelection = resolveEmbedderSelection;
|
|
31
|
+
exports.resolveReembedPolicy = resolveReembedPolicy;
|
|
32
|
+
exports.warnHashFallbackOnce = warnHashFallbackOnce;
|
|
33
|
+
exports.hashFallbackWarned = hashFallbackWarned;
|
|
34
|
+
exports.resetHashFallbackWarningForTests = resetHashFallbackWarningForTests;
|
|
35
|
+
const NO_PREFIX = { prefixPolicy: 'none', queryPrefix: '', passagePrefix: '' };
|
|
36
|
+
/** BGE en v1.5 documented query instruction (short query → long passage). */
|
|
37
|
+
exports.BGE_QUERY_INSTRUCTION = 'Represent this sentence for searching relevant passages: ';
|
|
38
|
+
/**
|
|
39
|
+
* Prefix conventions per model card:
|
|
40
|
+
* - all-MiniLM-L6-v2 / L12: general semantic search, NO prefixes.
|
|
41
|
+
* - e5-small-v2: REQUIRES 'query: ' / 'passage: ' (quality degrades without).
|
|
42
|
+
* - bge-small/base-en-v1.5: query instruction recommended for retrieval;
|
|
43
|
+
* passages need no instruction.
|
|
44
|
+
* - gte-small: no prefixes documented.
|
|
45
|
+
*/
|
|
46
|
+
exports.MODEL_PREFIXES = {
|
|
47
|
+
'all-MiniLM-L6-v2': { ...NO_PREFIX },
|
|
48
|
+
'all-MiniLM-L12-v2': { ...NO_PREFIX },
|
|
49
|
+
'e5-small-v2': { prefixPolicy: 'required', queryPrefix: 'query: ', passagePrefix: 'passage: ' },
|
|
50
|
+
'bge-small-en-v1.5': { prefixPolicy: 'query-recommended', queryPrefix: exports.BGE_QUERY_INSTRUCTION, passagePrefix: '' },
|
|
51
|
+
'bge-base-en-v1.5': { prefixPolicy: 'query-recommended', queryPrefix: exports.BGE_QUERY_INSTRUCTION, passagePrefix: '' },
|
|
52
|
+
'gte-small': { ...NO_PREFIX },
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Prefix spec for a model; unknown models get the no-prefix policy.
|
|
56
|
+
* Own-property lookup only: a hostile model id like '__proto__' or
|
|
57
|
+
* 'constructor' must resolve to NO_PREFIX, not to a prototype member
|
|
58
|
+
* (ADR-210 security pass).
|
|
59
|
+
*/
|
|
60
|
+
function getModelPrefixSpec(modelId) {
|
|
61
|
+
if (modelId && Object.prototype.hasOwnProperty.call(exports.MODEL_PREFIXES, modelId)) {
|
|
62
|
+
return exports.MODEL_PREFIXES[modelId];
|
|
63
|
+
}
|
|
64
|
+
return NO_PREFIX;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Pure prefix application (D4): the exact text handed to the tokenizer for a
|
|
68
|
+
* query/passage embed of `text` under `modelId`'s registered policy.
|
|
69
|
+
* MiniLM applies NO prefix on either entry point (acceptance gates 6–7).
|
|
70
|
+
*/
|
|
71
|
+
function prefixText(modelId, kind, text) {
|
|
72
|
+
const spec = getModelPrefixSpec(modelId);
|
|
73
|
+
const prefix = kind === 'query' ? spec.queryPrefix : spec.passagePrefix;
|
|
74
|
+
return prefix ? prefix + text : text;
|
|
75
|
+
}
|
|
76
|
+
/** Embedder family for an ONNX model id. */
|
|
77
|
+
function embedderKindForModel(modelId) {
|
|
78
|
+
return modelId && modelId.startsWith('all-MiniLM') ? 'onnx-minilm' : 'onnx';
|
|
79
|
+
}
|
|
80
|
+
// ============================================================================
|
|
81
|
+
// D0 — provenance compare / refuse / legacy derivation
|
|
82
|
+
// ============================================================================
|
|
83
|
+
/**
|
|
84
|
+
* Legacy default for stores that predate provenance metadata: hash-embedded,
|
|
85
|
+
* un-normalized as far as we can prove, no prefixes. Such stores open
|
|
86
|
+
* READ-ONLY for vector writes until re-embedded.
|
|
87
|
+
*/
|
|
88
|
+
function legacyHashProvenance(dimension = 256) {
|
|
89
|
+
return { embedderKind: 'hash', modelId: null, dimension, normalize: false, prefixPolicy: 'none' };
|
|
90
|
+
}
|
|
91
|
+
/** Human-readable one-liner for error messages. */
|
|
92
|
+
function describeProvenance(p) {
|
|
93
|
+
const model = p.modelId ? `, model=${p.modelId}` : '';
|
|
94
|
+
return `{ embedder=${p.embedderKind}${model}, dim=${p.dimension}, normalize=${p.normalize}, prefixPolicy=${p.prefixPolicy} }`;
|
|
95
|
+
}
|
|
96
|
+
/** Field names on which two provenance records disagree (empty = match). */
|
|
97
|
+
function compareProvenance(a, b) {
|
|
98
|
+
const mismatches = [];
|
|
99
|
+
if (a.embedderKind !== b.embedderKind)
|
|
100
|
+
mismatches.push('embedderKind');
|
|
101
|
+
if ((a.modelId ?? null) !== (b.modelId ?? null))
|
|
102
|
+
mismatches.push('modelId');
|
|
103
|
+
if (a.dimension !== b.dimension)
|
|
104
|
+
mismatches.push('dimension');
|
|
105
|
+
if (!!a.normalize !== !!b.normalize)
|
|
106
|
+
mismatches.push('normalize');
|
|
107
|
+
if (a.prefixPolicy !== b.prefixPolicy)
|
|
108
|
+
mismatches.push('prefixPolicy');
|
|
109
|
+
return mismatches;
|
|
110
|
+
}
|
|
111
|
+
/** Upper bound accepted for a provenance dimension read from disk. */
|
|
112
|
+
exports.MAX_PROVENANCE_DIMENSION = 65536;
|
|
113
|
+
const VALID_PREFIX_POLICIES = ['none', 'required', 'query-recommended'];
|
|
114
|
+
/**
|
|
115
|
+
* Sanitize a provenance record read from DISK (a `.meta.json` sidecar or
|
|
116
|
+
* `intelligence.json`). On-disk JSON is untrusted input: a malformed or
|
|
117
|
+
* adversarial record must never crash the caller. Anything that is not a
|
|
118
|
+
* plausibly-valid record is treated as ABSENT (returns null), which callers
|
|
119
|
+
* already handle as the no-provenance / legacy path — conservative for a
|
|
120
|
+
* corrupted stamp (the store degrades to read-only for vector writes rather
|
|
121
|
+
* than accepting writes under a fabricated identity).
|
|
122
|
+
*/
|
|
123
|
+
function sanitizeProvenance(value) {
|
|
124
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
125
|
+
return null;
|
|
126
|
+
const v = value;
|
|
127
|
+
const embedderKind = v.embedderKind;
|
|
128
|
+
if (typeof embedderKind !== 'string' || embedderKind.length === 0 || embedderKind.length > 64) {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
const dimension = v.dimension;
|
|
132
|
+
if (typeof dimension !== 'number' ||
|
|
133
|
+
!Number.isInteger(dimension) ||
|
|
134
|
+
dimension < 1 ||
|
|
135
|
+
dimension > exports.MAX_PROVENANCE_DIMENSION) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
let modelId = null;
|
|
139
|
+
if (typeof v.modelId === 'string') {
|
|
140
|
+
if (v.modelId.length === 0 || v.modelId.length > 256)
|
|
141
|
+
return null;
|
|
142
|
+
modelId = v.modelId;
|
|
143
|
+
}
|
|
144
|
+
else if (v.modelId !== null && v.modelId !== undefined) {
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
let prefixPolicy = 'none';
|
|
148
|
+
if (v.prefixPolicy !== undefined) {
|
|
149
|
+
if (typeof v.prefixPolicy !== 'string' || !VALID_PREFIX_POLICIES.includes(v.prefixPolicy)) {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
prefixPolicy = v.prefixPolicy;
|
|
153
|
+
}
|
|
154
|
+
return { embedderKind, modelId, dimension, normalize: !!v.normalize, prefixPolicy };
|
|
155
|
+
}
|
|
156
|
+
/** Thrown when an insert's provenance does not match the store's (D0). */
|
|
157
|
+
class ProvenanceMismatchError extends Error {
|
|
158
|
+
constructor(store, active, mismatches, storeName) {
|
|
159
|
+
super(`Embedding-provenance mismatch (ADR-210): refusing vector write to ${storeName}. ` +
|
|
160
|
+
`Store records ${describeProvenance(store)} but the active embedder is ` +
|
|
161
|
+
`${describeProvenance(active)} (differs on: ${mismatches.join(', ')}). ` +
|
|
162
|
+
`Mixed stores are never created — re-embed the store ('ruvector hooks reembed') ` +
|
|
163
|
+
`or switch the active embedder (RUVECTOR_EMBEDDER=auto|minilm|hash).`);
|
|
164
|
+
this.code = 'ERR_EMBEDDING_PROVENANCE';
|
|
165
|
+
this.name = 'ProvenanceMismatchError';
|
|
166
|
+
this.store = store;
|
|
167
|
+
this.active = active;
|
|
168
|
+
this.mismatches = mismatches;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
exports.ProvenanceMismatchError = ProvenanceMismatchError;
|
|
172
|
+
/** Refuse mismatched inserts with an error naming both sides (D0). */
|
|
173
|
+
function assertProvenanceMatch(store, active, storeName = 'vector store') {
|
|
174
|
+
const mismatches = compareProvenance(store, active);
|
|
175
|
+
if (mismatches.length > 0) {
|
|
176
|
+
throw new ProvenanceMismatchError(store, active, mismatches, storeName);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
// ============================================================================
|
|
180
|
+
// D5 — rollout flags (env overrides config)
|
|
181
|
+
// ============================================================================
|
|
182
|
+
/**
|
|
183
|
+
* Resolve RUVECTOR_EMBEDDER / RUVECTOR_ONNX.
|
|
184
|
+
* Precedence: RUVECTOR_EMBEDDER wins when both are set; RUVECTOR_ONNX=0 is
|
|
185
|
+
* shorthand for `hash`, =1 for `minilm`. Unrecognized values fall back to
|
|
186
|
+
* 'auto' (MiniLM when loadable, loud hash fallback otherwise).
|
|
187
|
+
*/
|
|
188
|
+
function resolveEmbedderSelection(env = process.env) {
|
|
189
|
+
const embedder = (env.RUVECTOR_EMBEDDER || '').trim().toLowerCase();
|
|
190
|
+
if (embedder === 'auto' || embedder === 'minilm' || embedder === 'hash')
|
|
191
|
+
return embedder;
|
|
192
|
+
const onnx = (env.RUVECTOR_ONNX || '').trim();
|
|
193
|
+
if (onnx === '0')
|
|
194
|
+
return 'hash';
|
|
195
|
+
if (onnx === '1')
|
|
196
|
+
return 'minilm';
|
|
197
|
+
return 'auto';
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Resolve RUVECTOR_REEMBED: what happens when opening a store whose
|
|
201
|
+
* provenance mismatches the active embedder.
|
|
202
|
+
* refuse (default) — error;
|
|
203
|
+
* warn — open read-only with a single warning;
|
|
204
|
+
* auto — re-embed in place when source text exists, refuse otherwise.
|
|
205
|
+
*/
|
|
206
|
+
function resolveReembedPolicy(env = process.env) {
|
|
207
|
+
const v = (env.RUVECTOR_REEMBED || '').trim().toLowerCase();
|
|
208
|
+
if (v === 'refuse' || v === 'warn' || v === 'auto')
|
|
209
|
+
return v;
|
|
210
|
+
return 'refuse';
|
|
211
|
+
}
|
|
212
|
+
// ============================================================================
|
|
213
|
+
// D1 — loud (but once-per-process) hash-fallback warning
|
|
214
|
+
// ============================================================================
|
|
215
|
+
let fallbackWarned = false;
|
|
216
|
+
/**
|
|
217
|
+
* Emit exactly ONE stderr warning per process the first time the hash
|
|
218
|
+
* fallback serves an embed that the ONNX embedder was supposed to handle
|
|
219
|
+
* (acceptance gate 2). Returns true when the warning was emitted by this call.
|
|
220
|
+
*/
|
|
221
|
+
function warnHashFallbackOnce(reason) {
|
|
222
|
+
if (fallbackWarned)
|
|
223
|
+
return false;
|
|
224
|
+
fallbackWarned = true;
|
|
225
|
+
const detail = reason ? ` Reason: ${reason}.` : '';
|
|
226
|
+
process.stderr.write(`ruvector: ONNX semantic embedder unavailable — using deterministic hash-fallback embeddings ` +
|
|
227
|
+
`(no semantic signal, reduced search quality).${detail} ` +
|
|
228
|
+
`Set RUVECTOR_EMBEDDER=hash to silence this or RUVECTOR_EMBEDDER=minilm to hard-require the model. ` +
|
|
229
|
+
`(warned once per process)\n`);
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
/** Whether the once-per-process fallback warning has fired. */
|
|
233
|
+
function hashFallbackWarned() {
|
|
234
|
+
return fallbackWarned;
|
|
235
|
+
}
|
|
236
|
+
/** Test hook: reset the once-per-process warning latch. */
|
|
237
|
+
function resetHashFallbackWarningForTests() {
|
|
238
|
+
fallbackWarned = false;
|
|
239
|
+
}
|
|
240
|
+
exports.default = {
|
|
241
|
+
MODEL_PREFIXES: exports.MODEL_PREFIXES,
|
|
242
|
+
BGE_QUERY_INSTRUCTION: exports.BGE_QUERY_INSTRUCTION,
|
|
243
|
+
getModelPrefixSpec,
|
|
244
|
+
prefixText,
|
|
245
|
+
embedderKindForModel,
|
|
246
|
+
legacyHashProvenance,
|
|
247
|
+
describeProvenance,
|
|
248
|
+
compareProvenance,
|
|
249
|
+
sanitizeProvenance,
|
|
250
|
+
MAX_PROVENANCE_DIMENSION: exports.MAX_PROVENANCE_DIMENSION,
|
|
251
|
+
ProvenanceMismatchError,
|
|
252
|
+
assertProvenanceMatch,
|
|
253
|
+
resolveEmbedderSelection,
|
|
254
|
+
resolveReembedPolicy,
|
|
255
|
+
warnHashFallbackOnce,
|
|
256
|
+
hashFallbackWarned,
|
|
257
|
+
resetHashFallbackWarningForTests,
|
|
258
|
+
};
|
package/dist/core/index.d.ts
CHANGED
|
@@ -9,6 +9,7 @@ export * from './attention-fallbacks';
|
|
|
9
9
|
export * from './agentdb-fast';
|
|
10
10
|
export * from './sona-wrapper';
|
|
11
11
|
export * from './intelligence-engine';
|
|
12
|
+
export * from './embedding-provenance';
|
|
12
13
|
export * from './onnx-embedder';
|
|
13
14
|
export * from './onnx-optimized';
|
|
14
15
|
export * from './parallel-intelligence';
|
package/dist/core/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,mBAAmB,CAAC;AAGlC,cAAc,aAAa,CAAC;AAG5B,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,OAAO,IAAI,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,OAAO,IAAI,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,OAAO,IAAI,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,OAAO,IAAI,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,OAAO,IAAI,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EAAE,OAAO,IAAI,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,OAAO,IAAI,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAGrD,OAAO,EAAE,UAAU,IAAI,SAAS,EAAE,MAAM,cAAc,CAAC;AAGvD,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,OAAO,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,mBAAmB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,mBAAmB,CAAC;AAGlC,cAAc,aAAa,CAAC;AAG5B,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,OAAO,IAAI,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,OAAO,IAAI,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,OAAO,IAAI,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,OAAO,IAAI,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,OAAO,IAAI,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EAAE,OAAO,IAAI,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,OAAO,IAAI,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAGrD,OAAO,EAAE,UAAU,IAAI,SAAS,EAAE,MAAM,cAAc,CAAC;AAGvD,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,OAAO,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,mBAAmB,CAAC"}
|
package/dist/core/index.js
CHANGED
|
@@ -29,6 +29,7 @@ __exportStar(require("./attention-fallbacks"), exports);
|
|
|
29
29
|
__exportStar(require("./agentdb-fast"), exports);
|
|
30
30
|
__exportStar(require("./sona-wrapper"), exports);
|
|
31
31
|
__exportStar(require("./intelligence-engine"), exports);
|
|
32
|
+
__exportStar(require("./embedding-provenance"), exports);
|
|
32
33
|
__exportStar(require("./onnx-embedder"), exports);
|
|
33
34
|
__exportStar(require("./onnx-optimized"), exports);
|
|
34
35
|
__exportStar(require("./parallel-intelligence"), exports);
|
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { EpisodeSearchResult } from './agentdb-fast';
|
|
14
14
|
import { SonaConfig, LearnedPattern } from './sona-wrapper';
|
|
15
|
+
import { OnnxEmbedderConfig } from './onnx-embedder';
|
|
16
|
+
import { EmbeddingProvenance } from './embedding-provenance';
|
|
15
17
|
import { ParallelConfig, BatchEpisode } from './parallel-intelligence';
|
|
16
18
|
export interface MemoryEntry {
|
|
17
19
|
id: string;
|
|
@@ -50,6 +52,13 @@ export interface LearningStats {
|
|
|
50
52
|
workerTriggers: number;
|
|
51
53
|
attentionEnabled: boolean;
|
|
52
54
|
onnxEnabled: boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Which embedder actually serves embedAsync() right now (ADR-210 D1):
|
|
57
|
+
* 'onnx-minilm' once the model is loaded, 'hash-fallback' while ONNX is
|
|
58
|
+
* enabled but not (yet) loaded, 'hash' when ONNX is deliberately disabled
|
|
59
|
+
* (config or RUVECTOR_EMBEDDER=hash / RUVECTOR_ONNX=0).
|
|
60
|
+
*/
|
|
61
|
+
embedderKind: 'onnx-minilm' | 'hash-fallback' | 'hash';
|
|
53
62
|
parallelEnabled: boolean;
|
|
54
63
|
parallelWorkers: number;
|
|
55
64
|
parallelBusy: number;
|
|
@@ -66,8 +75,15 @@ export interface IntelligenceConfig {
|
|
|
66
75
|
enableSona?: boolean;
|
|
67
76
|
/** Enable attention mechanisms (default: true if available) */
|
|
68
77
|
enableAttention?: boolean;
|
|
69
|
-
/**
|
|
78
|
+
/**
|
|
79
|
+
* Enable ONNX semantic embeddings (default: TRUE since ADR-210 D1 — the
|
|
80
|
+
* model loads lazily; until ready or when it cannot load, the hash
|
|
81
|
+
* fallback is used and loudly reported). RUVECTOR_EMBEDDER / RUVECTOR_ONNX
|
|
82
|
+
* environment variables override this config (D5).
|
|
83
|
+
*/
|
|
70
84
|
enableOnnx?: boolean;
|
|
85
|
+
/** Options forwarded to the ONNX embedder (model id, cache dir, ...). */
|
|
86
|
+
onnxConfig?: OnnxEmbedderConfig;
|
|
71
87
|
/** SONA configuration */
|
|
72
88
|
sonaConfig?: Partial<SonaConfig>;
|
|
73
89
|
/** Storage path for persistence */
|
|
@@ -91,6 +107,10 @@ export declare class IntelligenceEngine {
|
|
|
91
107
|
private attention;
|
|
92
108
|
private onnxEmbedder;
|
|
93
109
|
private onnxReady;
|
|
110
|
+
private onnxInitPromise;
|
|
111
|
+
private onnxInitError;
|
|
112
|
+
/** RUVECTOR_EMBEDDER=minilm: fail rather than fall back (ADR-210 D5). */
|
|
113
|
+
private onnxHardRequire;
|
|
94
114
|
private parallel;
|
|
95
115
|
private memories;
|
|
96
116
|
private routingPatterns;
|
|
@@ -107,6 +127,14 @@ export declare class IntelligenceEngine {
|
|
|
107
127
|
private episodeBatchQueue;
|
|
108
128
|
constructor(config?: IntelligenceConfig);
|
|
109
129
|
private initOnnx;
|
|
130
|
+
/**
|
|
131
|
+
* Await lazy ONNX initialization. Resolves true once the model is loaded,
|
|
132
|
+
* false when it could not be (offline / restricted CI) — in which case
|
|
133
|
+
* stats().embedderKind reports 'hash-fallback' (ADR-210 D1).
|
|
134
|
+
*/
|
|
135
|
+
awaitOnnx(): Promise<boolean>;
|
|
136
|
+
/** Why ONNX init failed, or null (ADR-210 D1 observability). */
|
|
137
|
+
getOnnxInitError(): Error | null;
|
|
110
138
|
private initVectorDb;
|
|
111
139
|
private initParallel;
|
|
112
140
|
/**
|
|
@@ -114,9 +142,32 @@ export declare class IntelligenceEngine {
|
|
|
114
142
|
*/
|
|
115
143
|
embed(text: string): number[];
|
|
116
144
|
/**
|
|
117
|
-
* Async embedding with ONNX support (recommended for semantic quality)
|
|
145
|
+
* Async embedding with ONNX support (recommended for semantic quality).
|
|
146
|
+
*
|
|
147
|
+
* ADR-210 D1: when ONNX is enabled but the model cannot load, the hash
|
|
148
|
+
* fallback is used and reported (one stderr warning per process, and
|
|
149
|
+
* stats().embedderKind === 'hash-fallback'). Under RUVECTOR_EMBEDDER=minilm
|
|
150
|
+
* the failure is an error instead — no fallback (D5).
|
|
118
151
|
*/
|
|
119
152
|
embedAsync(text: string): Promise<number[]>;
|
|
153
|
+
/**
|
|
154
|
+
* Batch embedding for bulk ingest (ADR-210 D3). When the ONNX model is
|
|
155
|
+
* loaded, batches of 32+ texts route through the bundled parallel worker
|
|
156
|
+
* pool (parallel-fp32 — see embedBulk in onnx-embedder.ts for the int8
|
|
157
|
+
* status note); smaller batches use the single-threaded batch path. On
|
|
158
|
+
* fallback, semantics match embedAsync exactly: hash per-item with the
|
|
159
|
+
* loud once-per-process warning, or a hard error under
|
|
160
|
+
* RUVECTOR_EMBEDDER=minilm (D5). Texts are embedded as passages (D4).
|
|
161
|
+
*
|
|
162
|
+
* Callers that start the pool should call shutdownEmbedderPool() when the
|
|
163
|
+
* bulk work is done so worker threads do not keep the process alive.
|
|
164
|
+
*/
|
|
165
|
+
embedBatchAsync(texts: string[]): Promise<number[][]>;
|
|
166
|
+
/**
|
|
167
|
+
* Shut down the bundled bulk-embed worker pool, releasing its threads
|
|
168
|
+
* (ADR-210 D3). Safe to call when the pool was never started.
|
|
169
|
+
*/
|
|
170
|
+
shutdownEmbedderPool(): Promise<void>;
|
|
120
171
|
/**
|
|
121
172
|
* Attention-based embedding using Flash or Multi-head attention
|
|
122
173
|
*/
|
|
@@ -236,6 +287,12 @@ export declare class IntelligenceEngine {
|
|
|
236
287
|
* Get comprehensive learning statistics
|
|
237
288
|
*/
|
|
238
289
|
getStats(): LearningStats;
|
|
290
|
+
/**
|
|
291
|
+
* Embedding provenance of vectors embedAsync() would produce right now
|
|
292
|
+
* (ADR-210 D0). Hash fallback embeds are 'hash' even while ONNX is enabled
|
|
293
|
+
* but not ready — provenance records what actually happened, not intent.
|
|
294
|
+
*/
|
|
295
|
+
getActiveProvenance(): EmbeddingProvenance;
|
|
239
296
|
/**
|
|
240
297
|
* Export all data for persistence
|
|
241
298
|
*/
|
|
@@ -260,11 +317,15 @@ export declare class IntelligenceEngine {
|
|
|
260
317
|
*/
|
|
261
318
|
export declare function createIntelligenceEngine(config?: IntelligenceConfig): IntelligenceEngine;
|
|
262
319
|
/**
|
|
263
|
-
* Create a high-performance engine with all features enabled
|
|
320
|
+
* Create a high-performance engine with all features enabled.
|
|
321
|
+
* Note (ADR-210): with default-on ONNX the embedding space is 384-dim; the
|
|
322
|
+
* 512-dim setting only applies on the hash path (RUVECTOR_EMBEDDER=hash or
|
|
323
|
+
* ONNX unavailable). SONA dims follow the engine's actual embeddingDim.
|
|
264
324
|
*/
|
|
265
325
|
export declare function createHighPerformanceEngine(): IntelligenceEngine;
|
|
266
326
|
/**
|
|
267
|
-
* Create a lightweight engine for fast startup
|
|
327
|
+
* Create a lightweight engine for fast startup (hash embedder: no model load,
|
|
328
|
+
* no download — the deterministic no-model path stays available, ADR-210).
|
|
268
329
|
*/
|
|
269
330
|
export declare function createLightweightEngine(): IntelligenceEngine;
|
|
270
331
|
export default IntelligenceEngine;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"intelligence-engine.d.ts","sourceRoot":"","sources":["../../src/core/intelligence-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAoC,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvF,OAAO,EAAc,UAAU,EAAE,cAAc,EAA8B,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"intelligence-engine.d.ts","sourceRoot":"","sources":["../../src/core/intelligence-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAoC,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvF,OAAO,EAAc,UAAU,EAAE,cAAc,EAA8B,MAAM,gBAAgB,CAAC;AACpG,OAAO,EAAgB,kBAAkB,EAAiG,MAAM,iBAAiB,CAAC;AAClK,OAAO,EACL,mBAAmB,EAGpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAwB,cAAc,EAAE,YAAY,EAA2B,MAAM,yBAAyB,CAAC;AAMtH,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,cAAc,EAAE,CAAC;IAC5B,UAAU,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3D;AAED,MAAM,WAAW,aAAa;IAE5B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IAGzB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAGlB,WAAW,EAAE,OAAO,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAG1B,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IAGvB,gBAAgB,EAAE,OAAO,CAAC;IAG1B,WAAW,EAAE,OAAO,CAAC;IACrB;;;;;OAKG;IACH,YAAY,EAAE,aAAa,GAAG,eAAe,GAAG,MAAM,CAAC;IAGvD,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,mEAAmE;IACnE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,+DAA+D;IAC/D,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,yEAAyE;IACzE,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,yBAAyB;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,mCAAmC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;CAC1C;AAiDD;;GAEG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,IAAI,CAA2B;IACvC,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,eAAe,CAAiC;IACxD,OAAO,CAAC,aAAa,CAAsB;IAC3C,yEAAyE;IACzE,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAqC;IAGrD,OAAO,CAAC,QAAQ,CAAuC;IACvD,OAAO,CAAC,eAAe,CAA+C;IACtE,OAAO,CAAC,aAAa,CAAoC;IACzD,OAAO,CAAC,cAAc,CAA+C;IACrE,OAAO,CAAC,aAAa,CAAkC;IACvD,OAAO,CAAC,qBAAqB,CAAkE;IAG/F,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,wBAAwB,CAAuB;IACvD,OAAO,CAAC,qBAAqB,CAAiC;IAC9D,OAAO,CAAC,sBAAsB,CAAuB;IACrD,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,eAAe,CAAiB;IACxC,OAAO,CAAC,iBAAiB,CAAsB;gBAEnC,MAAM,GAAE,kBAAuB;YA+E7B,QAAQ;IAetB;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;IAKnC,gEAAgE;IAChE,gBAAgB,IAAI,KAAK,GAAG,IAAI;YAIlB,YAAY;YAYZ,YAAY;IAe1B;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAsB7B;;;;;;;OAOG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAwBjD;;;;;;;;;;;OAWG;IACG,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAsB3D;;;OAGG;IACG,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3C;;OAEG;IACH,OAAO,CAAC,cAAc;IAiDtB;;OAEG;IACH,OAAO,CAAC,SAAS;IA8BjB,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,UAAU;IAWlB,OAAO,CAAC,QAAQ;IAehB;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE,MAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;IAgC/E;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,MAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAoCrE,OAAO,CAAC,gBAAgB;IAgBxB;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAoF7D,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,QAAQ;IAQhB,OAAO,CAAC,aAAa;IAarB;;OAEG;IACH,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI;IAqBrD;;OAEG;IACH,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAW9D;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI;IA4BvD;;OAEG;IACH,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAWvC;;;;;;;;OAQG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAgB/F;;OAEG;IACG,aAAa,CACjB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,OAAO,EACb,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC7B,OAAO,CAAC,IAAI,CAAC;IAwBhB;;OAEG;IACH,YAAY,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAIzC;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC;IAmB1C;;OAEG;IACG,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,GAAE,MAAU,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IASpF;;OAEG;IACH,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAIhF;;OAEG;IACH,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,SAAS;IAIxF;;OAEG;IACG,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAwBxE;;OAEG;IACH,yBAAyB,IAAI,IAAI;IAyBjC;;OAEG;IACH,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAehD;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,MAAU,GAAG,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAc1F;;OAEG;IACH,cAAc,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAUvD;;OAEG;IACH,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE;IA6B1C;;OAEG;IACH,IAAI,IAAI,MAAM,GAAG,IAAI;IAWrB;;OAEG;IACH,UAAU,IAAI,MAAM,GAAG,IAAI;IAe3B;;OAEG;IACH,QAAQ,IAAI,aAAa;IAyDzB;;;;OAIG;IACH,mBAAmB,IAAI,mBAAmB;IA0B1C;;OAEG;IACH,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAmC7B;;OAEG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,KAAK,GAAE,OAAe,GAAG,IAAI;IAmF/D;;OAEG;IACH,KAAK,IAAI,IAAI;IAcb,8BAA8B;IAC9B,IAAI,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAMrD;IAED,mCAAmC;IACnC,IAAI,cAAc,IAAI,MAAM,EAAE,EAAE,CAW/B;IAED,4BAA4B;IAC5B,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAErC;CACF;AAMD;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,CAAC,EAAE,kBAAkB,GAAG,kBAAkB,CAExF;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,IAAI,kBAAkB,CAahE;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,IAAI,kBAAkB,CAS5D;AAED,eAAe,kBAAkB,CAAC"}
|