redis-vl 0.1.0-beta.2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -10
- package/dist/query/text.d.ts +54 -9
- package/dist/query/text.d.ts.map +1 -1
- package/dist/query/text.js +114 -13
- package/dist/query/text.js.map +1 -1
- package/dist/vectorizers/huggingface-vectorizer.d.ts +5 -0
- package/dist/vectorizers/huggingface-vectorizer.d.ts.map +1 -1
- package/dist/vectorizers/huggingface-vectorizer.js +2 -0
- package/dist/vectorizers/huggingface-vectorizer.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -8,17 +8,12 @@
|
|
|
8
8
|
|
|
9
9
|
[](https://opensource.org/licenses/MIT)
|
|
10
10
|

|
|
11
|
-

|
|
11
|
+

|
|
13
12
|
|
|
14
13
|
</div>
|
|
15
14
|
|
|
16
15
|
---
|
|
17
16
|
|
|
18
|
-
> **⚠️** RedisVL TypeScript is in active development. APIs may change before v1.0 release.
|
|
19
|
-
|
|
20
|
-
---
|
|
21
|
-
|
|
22
17
|
## Introduction
|
|
23
18
|
|
|
24
19
|
Redis Vector Library (RedisVL) is the TypeScript/Node.js client for building AI applications on Redis.
|
|
@@ -37,13 +32,10 @@ Redis Vector Library (RedisVL) is the TypeScript/Node.js client for building AI
|
|
|
37
32
|
|
|
38
33
|
## Getting Started
|
|
39
34
|
|
|
40
|
-
> [!WARNING]
|
|
41
|
-
> `RedisVL TypeScript` is in beta. APIs, defaults, and the package itself may change between minor versions.
|
|
42
|
-
|
|
43
35
|
**Option A: Install from npm:**
|
|
44
36
|
|
|
45
37
|
```bash
|
|
46
|
-
npm install redis-vl
|
|
38
|
+
npm install redis-vl
|
|
47
39
|
```
|
|
48
40
|
|
|
49
41
|
**Option B: Install from source** by cloning the repo and linking it locally:
|
package/dist/query/text.d.ts
CHANGED
|
@@ -11,8 +11,12 @@ export type TextScorer = 'BM25' | 'BM25STD' | 'TFIDF' | 'TFIDF.DOCNORM' | 'DISMA
|
|
|
11
11
|
export interface TextQueryConfig {
|
|
12
12
|
/** Free-text query. Tokenised on whitespace, normalized (lowercase, comma + curly-quote strip), stopword-filtered, then OR-joined. */
|
|
13
13
|
text: string;
|
|
14
|
-
/**
|
|
15
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Indexed text field to search against. Pass a string to search a single
|
|
16
|
+
* field, or a `Record<field, weight>` to search multiple fields with
|
|
17
|
+
* per-field weighting. Weights must be finite numbers > 0.
|
|
18
|
+
*/
|
|
19
|
+
textFieldName: string | Record<string, number>;
|
|
16
20
|
/**
|
|
17
21
|
* Scorer to apply when ranking results. Defaults to `BM25STD`.
|
|
18
22
|
* Has no effect unless the index field is configured as TEXT.
|
|
@@ -28,6 +32,17 @@ export interface TextQueryConfig {
|
|
|
28
32
|
offset?: number;
|
|
29
33
|
/** Pagination limit. Defaults to numResults. */
|
|
30
34
|
limit?: number;
|
|
35
|
+
/**
|
|
36
|
+
* Per-token weight map. Keys are individual words (no inner whitespace) and
|
|
37
|
+
* are matched case-insensitively against the escaped query token (Python
|
|
38
|
+
* parity). A key containing Redis special chars (e.g. `wi-fi`, `c++`) will
|
|
39
|
+
* therefore NOT match — the escaped token differs from the un-escaped key —
|
|
40
|
+
* so its weight is silently dropped, mirroring Python's
|
|
41
|
+
* `_tokenize_and_escape_query`. Values must be finite numbers >= 0. A weight
|
|
42
|
+
* of 0 effectively suppresses scoring for that token. When omitted, no
|
|
43
|
+
* per-token weighting is applied.
|
|
44
|
+
*/
|
|
45
|
+
textWeights?: Record<string, number>;
|
|
31
46
|
/**
|
|
32
47
|
* Stopwords to drop before OR-joining tokens.
|
|
33
48
|
*
|
|
@@ -50,24 +65,47 @@ export interface TextQueryConfig {
|
|
|
50
65
|
* survivors inside the target field. Use `filter` to scope the search to
|
|
51
66
|
* a subset of documents (e.g. by tag or numeric range).
|
|
52
67
|
*
|
|
53
|
-
*
|
|
54
|
-
* `
|
|
68
|
+
* Supports per-token weighting via `textWeights` and per-field weighting by
|
|
69
|
+
* passing a `Record<field, weight>` to `textFieldName`. Both render using
|
|
70
|
+
* Redis Search's `=> { $weight: N }` syntax (dialect 2).
|
|
55
71
|
*
|
|
56
|
-
* @example
|
|
72
|
+
* @example Single-field, default weights
|
|
57
73
|
* ```typescript
|
|
58
|
-
* import { TextQuery
|
|
74
|
+
* import { TextQuery } from 'redis-vl';
|
|
59
75
|
*
|
|
60
76
|
* const q = new TextQuery({
|
|
61
77
|
* text: 'machine learning',
|
|
62
78
|
* textFieldName: 'description',
|
|
63
|
-
* filter: Tag('category').eq('tech'),
|
|
64
79
|
* });
|
|
65
|
-
*
|
|
80
|
+
* ```
|
|
81
|
+
*
|
|
82
|
+
* @example Multi-field weighted
|
|
83
|
+
* ```typescript
|
|
84
|
+
* new TextQuery({
|
|
85
|
+
* text: 'machine learning',
|
|
86
|
+
* textFieldName: { title: 5.0, body: 1.0 },
|
|
87
|
+
* });
|
|
88
|
+
* ```
|
|
89
|
+
*
|
|
90
|
+
* @example Per-token weighted
|
|
91
|
+
* ```typescript
|
|
92
|
+
* new TextQuery({
|
|
93
|
+
* text: 'apple orange pear',
|
|
94
|
+
* textFieldName: 'description',
|
|
95
|
+
* textWeights: { apple: 2.0, orange: 0.5 },
|
|
96
|
+
* });
|
|
66
97
|
* ```
|
|
67
98
|
*/
|
|
68
99
|
export declare class TextQuery implements BaseQuery {
|
|
69
100
|
readonly text: string;
|
|
70
|
-
|
|
101
|
+
/**
|
|
102
|
+
* Per-field weights. Iteration follows insertion order, which determines
|
|
103
|
+
* the order of field clauses in the rendered query. A single field with
|
|
104
|
+
* weight 1.0 renders identically to passing a bare string for `textFieldName`.
|
|
105
|
+
*/
|
|
106
|
+
readonly fieldWeights: Readonly<Record<string, number>>;
|
|
107
|
+
/** Per-token weights. Keys are normalised to lowercase, whitespace-trimmed single tokens. */
|
|
108
|
+
readonly textWeights: Readonly<Record<string, number>>;
|
|
71
109
|
readonly textScorer: TextScorer;
|
|
72
110
|
readonly filter?: FilterInput;
|
|
73
111
|
readonly returnFields?: string[];
|
|
@@ -78,5 +116,12 @@ export declare class TextQuery implements BaseQuery {
|
|
|
78
116
|
constructor(config: TextQueryConfig);
|
|
79
117
|
buildQuery(): string;
|
|
80
118
|
buildParams(): Record<string, unknown>;
|
|
119
|
+
/**
|
|
120
|
+
* Returns the configured text field. A bare string is returned when exactly
|
|
121
|
+
* one field is configured with weight 1.0. Otherwise returns a copy of the
|
|
122
|
+
* normalised field-weight record. Mirrors Python's `text_field_name`
|
|
123
|
+
* property for cross-language compatibility.
|
|
124
|
+
*/
|
|
125
|
+
get textFieldName(): string | Readonly<Record<string, number>>;
|
|
81
126
|
}
|
|
82
127
|
//# sourceMappingURL=text.d.ts.map
|
package/dist/query/text.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../src/query/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,SAAS,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAG3E,OAAO,EAAoB,KAAK,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAetF;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,eAAe,GAAG,QAAQ,GAAG,UAAU,CAAC;AAEhG;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B,sIAAsI;IACtI,IAAI,EAAE,MAAM,CAAC;IAEb
|
|
1
|
+
{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../src/query/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,SAAS,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAG3E,OAAO,EAAoB,KAAK,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAetF;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,eAAe,GAAG,QAAQ,GAAG,UAAU,CAAC;AAEhG;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B,sIAAsI;IACtI,IAAI,EAAE,MAAM,CAAC;IAEb;;;;OAIG;IACH,aAAa,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/C;;;OAGG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IAExB,wEAAwE;IACxE,MAAM,CAAC,EAAE,WAAW,CAAC;IAErB,iDAAiD;IACjD,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IAExB,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,gDAAgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAErC;;;;;;;;;;OAUG;IACH,SAAS,CAAC,EAAE,cAAc,CAAC;CAC9B;AA+DD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,SAAU,YAAW,SAAS;IACvC,SAAgB,IAAI,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,SAAgB,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC/D,6FAA6F;IAC7F,SAAgB,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,SAAgB,UAAU,EAAE,UAAU,CAAC;IACvC,SAAgB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrC,SAAgB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChC,SAAgB,KAAK,CAAC,EAAE,MAAM,CAAC;IAC/B,SAAgB,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;gBAE1C,MAAM,EAAE,eAAe;IAanC,UAAU,IAAI,MAAM;IAqCpB,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAItC;;;;;OAKG;IACH,IAAI,aAAa,IAAI,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAS7D;CACJ"}
|
package/dist/query/text.js
CHANGED
|
@@ -12,6 +12,54 @@ function normalizeToken(token) {
|
|
|
12
12
|
.replace(TYPOGRAPHIC_QUOTES, '')
|
|
13
13
|
.toLowerCase();
|
|
14
14
|
}
|
|
15
|
+
function parseFieldWeights(spec) {
|
|
16
|
+
if (spec == null || spec === '') {
|
|
17
|
+
throw new QueryValidationError('textFieldName is required');
|
|
18
|
+
}
|
|
19
|
+
// Defensively copy into a null-prototype record so a caller mutating their
|
|
20
|
+
// config object after construction cannot alter this query, and so a field
|
|
21
|
+
// named `__proto__` cannot pollute Object.prototype.
|
|
22
|
+
const normalized = Object.create(null);
|
|
23
|
+
if (typeof spec === 'string') {
|
|
24
|
+
normalized[spec] = 1.0;
|
|
25
|
+
return Object.freeze(normalized);
|
|
26
|
+
}
|
|
27
|
+
if (Array.isArray(spec)) {
|
|
28
|
+
throw new QueryValidationError('textFieldName must be a string or a record of field:weight mappings');
|
|
29
|
+
}
|
|
30
|
+
const entries = Object.entries(spec);
|
|
31
|
+
if (entries.length === 0) {
|
|
32
|
+
throw new QueryValidationError('textFieldName record must contain at least one field');
|
|
33
|
+
}
|
|
34
|
+
for (const [field, weight] of entries) {
|
|
35
|
+
if (field.length === 0) {
|
|
36
|
+
throw new QueryValidationError('textFieldName keys must be non-empty strings');
|
|
37
|
+
}
|
|
38
|
+
if (!Number.isFinite(weight) || weight <= 0) {
|
|
39
|
+
throw new QueryValidationError(`textFieldName weight for '${field}' must be a finite number > 0, got ${weight}`);
|
|
40
|
+
}
|
|
41
|
+
normalized[field] = weight;
|
|
42
|
+
}
|
|
43
|
+
return Object.freeze(normalized);
|
|
44
|
+
}
|
|
45
|
+
function parseTextWeights(weights) {
|
|
46
|
+
// null-prototype record: assigning a key like `__proto__` stores an own
|
|
47
|
+
// property instead of invoking the prototype setter (prototype pollution).
|
|
48
|
+
const parsed = Object.create(null);
|
|
49
|
+
if (!weights)
|
|
50
|
+
return Object.freeze(parsed);
|
|
51
|
+
for (const [rawKey, weight] of Object.entries(weights)) {
|
|
52
|
+
const key = rawKey.trim().toLowerCase();
|
|
53
|
+
if (!key || /\s/.test(key)) {
|
|
54
|
+
throw new QueryValidationError(`Only individual words may be weighted. Got { ${rawKey}:${weight} }`);
|
|
55
|
+
}
|
|
56
|
+
if (!Number.isFinite(weight) || weight < 0) {
|
|
57
|
+
throw new QueryValidationError(`Weights must be a non-negative number. Got { ${key}:${weight} }`);
|
|
58
|
+
}
|
|
59
|
+
parsed[key] = weight;
|
|
60
|
+
}
|
|
61
|
+
return Object.freeze(parsed);
|
|
62
|
+
}
|
|
15
63
|
/**
|
|
16
64
|
* Full-text search query with optional filter.
|
|
17
65
|
*
|
|
@@ -21,24 +69,47 @@ function normalizeToken(token) {
|
|
|
21
69
|
* survivors inside the target field. Use `filter` to scope the search to
|
|
22
70
|
* a subset of documents (e.g. by tag or numeric range).
|
|
23
71
|
*
|
|
24
|
-
*
|
|
25
|
-
* `
|
|
72
|
+
* Supports per-token weighting via `textWeights` and per-field weighting by
|
|
73
|
+
* passing a `Record<field, weight>` to `textFieldName`. Both render using
|
|
74
|
+
* Redis Search's `=> { $weight: N }` syntax (dialect 2).
|
|
26
75
|
*
|
|
27
|
-
* @example
|
|
76
|
+
* @example Single-field, default weights
|
|
28
77
|
* ```typescript
|
|
29
|
-
* import { TextQuery
|
|
78
|
+
* import { TextQuery } from 'redis-vl';
|
|
30
79
|
*
|
|
31
80
|
* const q = new TextQuery({
|
|
32
81
|
* text: 'machine learning',
|
|
33
82
|
* textFieldName: 'description',
|
|
34
|
-
* filter: Tag('category').eq('tech'),
|
|
35
83
|
* });
|
|
36
|
-
*
|
|
84
|
+
* ```
|
|
85
|
+
*
|
|
86
|
+
* @example Multi-field weighted
|
|
87
|
+
* ```typescript
|
|
88
|
+
* new TextQuery({
|
|
89
|
+
* text: 'machine learning',
|
|
90
|
+
* textFieldName: { title: 5.0, body: 1.0 },
|
|
91
|
+
* });
|
|
92
|
+
* ```
|
|
93
|
+
*
|
|
94
|
+
* @example Per-token weighted
|
|
95
|
+
* ```typescript
|
|
96
|
+
* new TextQuery({
|
|
97
|
+
* text: 'apple orange pear',
|
|
98
|
+
* textFieldName: 'description',
|
|
99
|
+
* textWeights: { apple: 2.0, orange: 0.5 },
|
|
100
|
+
* });
|
|
37
101
|
* ```
|
|
38
102
|
*/
|
|
39
103
|
export class TextQuery {
|
|
40
104
|
text;
|
|
41
|
-
|
|
105
|
+
/**
|
|
106
|
+
* Per-field weights. Iteration follows insertion order, which determines
|
|
107
|
+
* the order of field clauses in the rendered query. A single field with
|
|
108
|
+
* weight 1.0 renders identically to passing a bare string for `textFieldName`.
|
|
109
|
+
*/
|
|
110
|
+
fieldWeights;
|
|
111
|
+
/** Per-token weights. Keys are normalised to lowercase, whitespace-trimmed single tokens. */
|
|
112
|
+
textWeights;
|
|
42
113
|
textScorer;
|
|
43
114
|
filter;
|
|
44
115
|
returnFields;
|
|
@@ -47,11 +118,9 @@ export class TextQuery {
|
|
|
47
118
|
limit;
|
|
48
119
|
stopwords;
|
|
49
120
|
constructor(config) {
|
|
50
|
-
if (!config.textFieldName) {
|
|
51
|
-
throw new QueryValidationError('textFieldName is required');
|
|
52
|
-
}
|
|
53
121
|
this.text = config.text;
|
|
54
|
-
this.
|
|
122
|
+
this.fieldWeights = parseFieldWeights(config.textFieldName);
|
|
123
|
+
this.textWeights = parseTextWeights(config.textWeights);
|
|
55
124
|
this.textScorer = config.textScorer ?? 'BM25STD';
|
|
56
125
|
this.filter = config.filter;
|
|
57
126
|
this.returnFields = config.returnFields;
|
|
@@ -62,6 +131,7 @@ export class TextQuery {
|
|
|
62
131
|
}
|
|
63
132
|
buildQuery() {
|
|
64
133
|
const stopwordSet = this.stopwords;
|
|
134
|
+
const weights = this.textWeights;
|
|
65
135
|
const tokens = [];
|
|
66
136
|
for (const raw of this.text.split(/\s+/)) {
|
|
67
137
|
const norm = normalizeToken(raw);
|
|
@@ -70,9 +140,24 @@ export class TextQuery {
|
|
|
70
140
|
const escaped = escaper.escape(norm);
|
|
71
141
|
if (stopwordSet && stopwordSet.has(escaped))
|
|
72
142
|
continue;
|
|
73
|
-
|
|
143
|
+
if (Object.hasOwn(weights, escaped)) {
|
|
144
|
+
tokens.push(`${escaped}=>{$weight:${weights[escaped]}}`);
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
tokens.push(escaped);
|
|
148
|
+
}
|
|
74
149
|
}
|
|
75
|
-
const
|
|
150
|
+
const orList = tokens.join(' | ');
|
|
151
|
+
const fieldClauses = [];
|
|
152
|
+
for (const [field, weight] of Object.entries(this.fieldWeights)) {
|
|
153
|
+
if (weight === 1.0) {
|
|
154
|
+
fieldClauses.push(`@${field}:(${orList})`);
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
fieldClauses.push(`@${field}:(${orList}) => { $weight: ${weight} }`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
const textClause = fieldClauses.length === 1 ? fieldClauses[0] : `(${fieldClauses.join(' | ')})`;
|
|
76
161
|
const filterStr = renderFilter(this.filter);
|
|
77
162
|
if (filterStr === '*') {
|
|
78
163
|
return textClause;
|
|
@@ -82,5 +167,21 @@ export class TextQuery {
|
|
|
82
167
|
buildParams() {
|
|
83
168
|
return {};
|
|
84
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* Returns the configured text field. A bare string is returned when exactly
|
|
172
|
+
* one field is configured with weight 1.0. Otherwise returns a copy of the
|
|
173
|
+
* normalised field-weight record. Mirrors Python's `text_field_name`
|
|
174
|
+
* property for cross-language compatibility.
|
|
175
|
+
*/
|
|
176
|
+
get textFieldName() {
|
|
177
|
+
const entries = Object.entries(this.fieldWeights);
|
|
178
|
+
if (entries.length === 1) {
|
|
179
|
+
const [field, weight] = entries[0];
|
|
180
|
+
if (weight === 1.0) {
|
|
181
|
+
return field;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return { ...this.fieldWeights };
|
|
185
|
+
}
|
|
85
186
|
}
|
|
86
187
|
//# sourceMappingURL=text.js.map
|
package/dist/query/text.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/query/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAoC,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAuB,MAAM,+BAA+B,CAAC;AAEtF,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;AAEnC,MAAM,6BAA6B,GAAG,UAAU,CAAC;AACjD,MAAM,kBAAkB,GAAG,OAAO,CAAC;AAEnC,SAAS,cAAc,CAAC,KAAa;IACjC,OAAO,KAAK;SACP,IAAI,EAAE;SACN,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;SAC1C,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,WAAW,EAAE,CAAC;AACvB,CAAC;
|
|
1
|
+
{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/query/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAoC,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAuB,MAAM,+BAA+B,CAAC;AAEtF,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;AAEnC,MAAM,6BAA6B,GAAG,UAAU,CAAC;AACjD,MAAM,kBAAkB,GAAG,OAAO,CAAC;AAEnC,SAAS,cAAc,CAAC,KAAa;IACjC,OAAO,KAAK;SACP,IAAI,EAAE;SACN,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;SAC1C,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,WAAW,EAAE,CAAC;AACvB,CAAC;AAqED,SAAS,iBAAiB,CACtB,IAAqC;IAErC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;QAC9B,MAAM,IAAI,oBAAoB,CAAC,2BAA2B,CAAC,CAAC;IAChE,CAAC;IACD,2EAA2E;IAC3E,2EAA2E;IAC3E,qDAAqD;IACrD,MAAM,UAAU,GAA2B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/D,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3B,UAAU,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;QACvB,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,oBAAoB,CAC1B,qEAAqE,CACxE,CAAC;IACN,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,oBAAoB,CAAC,sDAAsD,CAAC,CAAC;IAC3F,CAAC;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,oBAAoB,CAAC,8CAA8C,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,oBAAoB,CAC1B,6BAA6B,KAAK,sCAAsC,MAAM,EAAE,CACnF,CAAC;QACN,CAAC;QACD,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IAC/B,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,gBAAgB,CACrB,OAA2C;IAE3C,wEAAwE;IACxE,2EAA2E;IAC3E,MAAM,MAAM,GAA2B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3C,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,oBAAoB,CAC1B,gDAAgD,MAAM,IAAI,MAAM,IAAI,CACvE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,oBAAoB,CAC1B,gDAAgD,GAAG,IAAI,MAAM,IAAI,CACpE,CAAC;QACN,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAM,OAAO,SAAS;IACF,IAAI,CAAS;IAC7B;;;;OAIG;IACa,YAAY,CAAmC;IAC/D,6FAA6F;IAC7E,WAAW,CAAmC;IAC9C,UAAU,CAAa;IACvB,MAAM,CAAe;IACrB,YAAY,CAAY;IACxB,UAAU,CAAS;IACnB,MAAM,CAAU;IAChB,KAAK,CAAU;IACf,SAAS,CAA6B;IAEtD,YAAY,MAAuB;QAC/B,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACxD,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,SAAS,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED,UAAU;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC;QACjC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAChC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,WAAW,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtD,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,cAAc,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAElC,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;YAC9D,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACjB,YAAY,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,MAAM,GAAG,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACJ,YAAY,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,MAAM,mBAAmB,MAAM,IAAI,CAAC,CAAC;YACzE,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GACZ,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;QAElF,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACpB,OAAO,UAAU,CAAC;QACtB,CAAC;QACD,OAAO,IAAI,SAAS,IAAI,UAAU,GAAG,CAAC;IAC1C,CAAC;IAED,WAAW;QACP,OAAO,EAAE,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,IAAI,aAAa;QACb,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAClD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACjB,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QACD,OAAO,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IACpC,CAAC;CACJ"}
|
|
@@ -36,6 +36,11 @@ export interface HuggingFaceConfig {
|
|
|
36
36
|
* Default: true
|
|
37
37
|
*/
|
|
38
38
|
normalize?: boolean;
|
|
39
|
+
/**
|
|
40
|
+
* Directory for cached model files. Defaults to a transformers.js path
|
|
41
|
+
* inside node_modules, which a reinstall wipes.
|
|
42
|
+
*/
|
|
43
|
+
cacheDir?: string;
|
|
39
44
|
}
|
|
40
45
|
/**
|
|
41
46
|
* HuggingFace vectorizer using Transformers.js for local inference.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"huggingface-vectorizer.d.ts","sourceRoot":"","sources":["../../src/vectorizers/huggingface-vectorizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGtD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B;;;;;;;OAOG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;OAIG;IACH,MAAM,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC;IAE1B;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAE/B;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAEzB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"huggingface-vectorizer.d.ts","sourceRoot":"","sources":["../../src/vectorizers/huggingface-vectorizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGtD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B;;;;;;;OAOG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;OAIG;IACH,MAAM,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC;IAE1B;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAE/B;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAEzB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,qBAAsB,SAAQ,cAAc;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CACiB;IACxC,OAAO,CAAC,gBAAgB,CAAa;IACrC,OAAO,CAAC,SAAS,CAAuB;gBAE5B,MAAM,EAAE,iBAAiB;IAkBrC;;;OAGG;YACW,YAAY;IA2B1B;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAmB5C;;OAEG;IACG,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAsC7E;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAQjB;IAED;;OAEG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;CACJ"}
|
|
@@ -52,6 +52,7 @@ export class HuggingFaceVectorizer extends BaseVectorizer {
|
|
|
52
52
|
dtype: config.dtype ?? 'fp32',
|
|
53
53
|
pooling: config.pooling ?? 'mean',
|
|
54
54
|
normalize: config.normalize ?? true,
|
|
55
|
+
cacheDir: config.cacheDir,
|
|
55
56
|
};
|
|
56
57
|
}
|
|
57
58
|
/**
|
|
@@ -67,6 +68,7 @@ export class HuggingFaceVectorizer extends BaseVectorizer {
|
|
|
67
68
|
this.pipelineInstance = await pipeline('feature-extraction', this.config.model, {
|
|
68
69
|
device: this.config.device,
|
|
69
70
|
dtype: this.config.dtype,
|
|
71
|
+
cache_dir: this.config.cacheDir,
|
|
70
72
|
});
|
|
71
73
|
return this.pipelineInstance;
|
|
72
74
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"huggingface-vectorizer.js","sourceRoot":"","sources":["../../src/vectorizers/huggingface-vectorizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"huggingface-vectorizer.js","sourceRoot":"","sources":["../../src/vectorizers/huggingface-vectorizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAmD/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,MAAM,OAAO,qBAAsB,SAAQ,cAAc;IACpC,MAAM,CACiB;IAChC,gBAAgB,GAAQ,IAAI,CAAC;IAC7B,SAAS,GAAkB,IAAI,CAAC;IAExC,YAAY,MAAyB;QACjC,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9C,MAAM,IAAI,eAAe,CAAC,4BAA4B,CAAC,CAAC;QAC5D,CAAC;QAED,eAAe;QACf,IAAI,CAAC,MAAM,GAAG;YACV,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,KAAK;YAC9B,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,MAAM;YAC7B,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;YACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC5B,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY;QACtB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,gBAAgB,CAAC;QACjC,CAAC;QAED,IAAI,CAAC;YACD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;YAE/D,IAAI,CAAC,gBAAgB,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;gBAC5E,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBACxB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;aAClC,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC,gBAAgB,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBACzE,MAAM,IAAI,eAAe,CACrB,4DAA4D;oBACxD,wDAAwD,EAC5D,EAAE,KAAK,EAAE,KAAK,EAAE,CACnB,CAAC;YACN,CAAC;YACD,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE;YAChC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;SACnC,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAa,CAAC;QAEtD,iCAAiC;QACjC,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC;QACtC,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,KAAe,EAAE,YAAoB,EAAE;QACnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,2CAA2C;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAE5C,gBAAgB;YAChB,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACf,QAAQ,CAAC,IAAI,EAAE;gBACX,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;aACnC,CAAC,CACL,CACJ,CAAC;YAEF,kCAAkC;YAClC,MAAM,eAAe,GAAG,YAAY,CAAC,GAAG,CACpC,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAa,CAClD,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;QACrC,CAAC;QAED,iCAAiC;QACjC,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACvC,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACJ,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,eAAe,CACrB,2CAA2C;gBACvC,4DAA4D,CACnE,CAAC;QACN,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAC7B,CAAC;CACJ"}
|