qwen-embedder 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +48 -37
- package/src/embedder.ts +21 -9
package/package.json
CHANGED
|
@@ -1,37 +1,48 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "qwen-embedder",
|
|
3
|
-
"version": "0.1.
|
|
4
|
-
"description": "Local text embedding with Qwen ONNX model via Transformers.js",
|
|
5
|
-
"type": "module",
|
|
6
|
-
"main": "./dist/index.js",
|
|
7
|
-
"types": "./dist/index.d.ts",
|
|
8
|
-
"exports": {
|
|
9
|
-
".": {
|
|
10
|
-
"types": "./dist/index.d.ts",
|
|
11
|
-
"import": "./dist/index.js"
|
|
12
|
-
}
|
|
13
|
-
},
|
|
14
|
-
"files": [
|
|
15
|
-
"src",
|
|
16
|
-
"dist"
|
|
17
|
-
],
|
|
18
|
-
"scripts": {
|
|
19
|
-
"build": "tsup",
|
|
20
|
-
"test": "vitest run",
|
|
21
|
-
"test:watch": "vitest"
|
|
22
|
-
},
|
|
23
|
-
"
|
|
24
|
-
|
|
25
|
-
"
|
|
26
|
-
"
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
"
|
|
30
|
-
"
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
|
|
34
|
-
"
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
1
|
+
{
|
|
2
|
+
"name": "qwen-embedder",
|
|
3
|
+
"version": "0.1.2",
|
|
4
|
+
"description": "Local text embedding with Qwen ONNX model via Transformers.js",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"src",
|
|
16
|
+
"dist"
|
|
17
|
+
],
|
|
18
|
+
"scripts": {
|
|
19
|
+
"build": "tsup",
|
|
20
|
+
"test": "vitest run",
|
|
21
|
+
"test:watch": "vitest"
|
|
22
|
+
},
|
|
23
|
+
"keywords": [
|
|
24
|
+
"embedding",
|
|
25
|
+
"vector",
|
|
26
|
+
"qwen",
|
|
27
|
+
"onnx",
|
|
28
|
+
"transformers",
|
|
29
|
+
"nlp",
|
|
30
|
+
"text-embedding",
|
|
31
|
+
"local-ai",
|
|
32
|
+
"feature-extraction"
|
|
33
|
+
],
|
|
34
|
+
"license": "MIT",
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"@huggingface/transformers": "^4.2.0",
|
|
37
|
+
"p-queue": "^9.3.0"
|
|
38
|
+
},
|
|
39
|
+
"devDependencies": {
|
|
40
|
+
"@types/node": "^25.9.3",
|
|
41
|
+
"tsup": "^8.5.1",
|
|
42
|
+
"typescript": "^6.0.3",
|
|
43
|
+
"vitest": "^4.1.8"
|
|
44
|
+
},
|
|
45
|
+
"allowScripts": {
|
|
46
|
+
"onnxruntime-node@1.24.3": true
|
|
47
|
+
}
|
|
48
|
+
}
|
package/src/embedder.ts
CHANGED
|
@@ -78,7 +78,8 @@ export class Embedder {
|
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
const results: (number[] | undefined)[] = new Array(texts.length)
|
|
81
|
-
const
|
|
81
|
+
const uncachedByText = new Map<string, number[]>()
|
|
82
|
+
const uncachedTexts: string[] = []
|
|
82
83
|
|
|
83
84
|
for (let i = 0; i < texts.length; i++) {
|
|
84
85
|
const t = texts[i]
|
|
@@ -87,23 +88,33 @@ export class Embedder {
|
|
|
87
88
|
}
|
|
88
89
|
const cached = this.cache.get(t)
|
|
89
90
|
if (cached !== undefined) {
|
|
90
|
-
results[i] = cached
|
|
91
|
+
results[i] = cached.slice()
|
|
91
92
|
} else {
|
|
92
|
-
|
|
93
|
+
const existing = uncachedByText.get(t)
|
|
94
|
+
if (existing !== undefined) {
|
|
95
|
+
existing.push(i)
|
|
96
|
+
} else {
|
|
97
|
+
uncachedByText.set(t, [i])
|
|
98
|
+
uncachedTexts.push(t)
|
|
99
|
+
}
|
|
93
100
|
}
|
|
94
101
|
}
|
|
95
102
|
|
|
96
|
-
if (
|
|
103
|
+
if (uncachedTexts.length === 0) {
|
|
97
104
|
return results as number[][]
|
|
98
105
|
}
|
|
99
106
|
|
|
100
|
-
const uncachedTexts = uncached.map(i => texts[i])
|
|
101
107
|
const inferred = await this.runInference(uncachedTexts)
|
|
102
108
|
|
|
103
|
-
for (let j = 0; j <
|
|
109
|
+
for (let j = 0; j < uncachedTexts.length; j++) {
|
|
110
|
+
const text = uncachedTexts[j]
|
|
104
111
|
const vector = inferred[j]
|
|
105
|
-
|
|
106
|
-
|
|
112
|
+
const stored = vector.slice()
|
|
113
|
+
this.setCache(text, stored)
|
|
114
|
+
const positions = uncachedByText.get(text) ?? []
|
|
115
|
+
for (const position of positions) {
|
|
116
|
+
results[position] = stored.slice()
|
|
117
|
+
}
|
|
107
118
|
}
|
|
108
119
|
|
|
109
120
|
return results as number[][]
|
|
@@ -135,12 +146,13 @@ export class Embedder {
|
|
|
135
146
|
private setCache(key: string, value: number[]): void {
|
|
136
147
|
if (this.maxCacheSize === 0) return
|
|
137
148
|
|
|
149
|
+
const stored = value.slice()
|
|
138
150
|
if (this.cache.size >= this.maxCacheSize) {
|
|
139
151
|
const oldest = this.cache.keys().next()
|
|
140
152
|
if (!oldest.done) {
|
|
141
153
|
this.cache.delete(oldest.value)
|
|
142
154
|
}
|
|
143
155
|
}
|
|
144
|
-
this.cache.set(key,
|
|
156
|
+
this.cache.set(key, stored)
|
|
145
157
|
}
|
|
146
158
|
}
|