pi-wsearch 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -11
- package/package.json +1 -1
- package/src/cache.ts +66 -34
- package/src/config.ts +14 -1
- package/src/dom.ts +13 -12
- package/src/engines.ts +298 -308
- package/src/fetch.ts +1 -5
- package/src/fuse.ts +27 -15
- package/src/keys.ts +2 -22
- package/src/login.ts +1 -1
- package/src/scrape.ts +58 -481
- package/src/text.ts +418 -0
- package/src/tools.ts +55 -66
- package/src/transport.ts +144 -113
- package/src/view.ts +2 -7
- package/src/constants.ts +0 -6
package/src/fuse.ts
CHANGED
|
@@ -11,10 +11,6 @@ export interface FusedHit extends SearchHit {
|
|
|
11
11
|
junk: boolean;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
-
export interface FuseOptions {
|
|
15
|
-
query?: string;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
14
|
const RRF_K = 60;
|
|
19
15
|
// Tiny on purpose: RRF already rewards multi-engine agreement, and the bonus
|
|
20
16
|
// must never scale with how many engines happen to be configured.
|
|
@@ -135,7 +131,13 @@ function queryTerms(query: string): string[] {
|
|
|
135
131
|
}
|
|
136
132
|
|
|
137
133
|
function termIn(term: string, text: string): boolean {
|
|
138
|
-
|
|
134
|
+
let i = text.indexOf(term);
|
|
135
|
+
for (; i >= 0; i = text.indexOf(term, i + 1)) {
|
|
136
|
+
const before = i === 0 ? ' ' : text[i - 1];
|
|
137
|
+
const after = i + term.length >= text.length ? ' ' : text[i + term.length];
|
|
138
|
+
if (!/[a-z0-9]/.test(before) && !/[a-z0-9]/.test(after)) return true;
|
|
139
|
+
}
|
|
140
|
+
return false;
|
|
139
141
|
}
|
|
140
142
|
|
|
141
143
|
function exactMatchBoost(hit: { title: string; snippet: string }, query: string | undefined): number {
|
|
@@ -163,12 +165,13 @@ function bm25Score(hit: { title: string; snippet: string }, terms: string[]): nu
|
|
|
163
165
|
const tfSnip = countTerm(t, snip);
|
|
164
166
|
if (!tfTitle && !tfSnip) continue;
|
|
165
167
|
const idf = 1 + Math.min(1, 12 / Math.max(t.length, 1));
|
|
166
|
-
const saturate = (tf: number) => (tf * 2.2) / (tf + 1.2);
|
|
167
168
|
score += idf * (2 * saturate(tfTitle) + saturate(tfSnip));
|
|
168
169
|
}
|
|
169
170
|
return score;
|
|
170
171
|
}
|
|
171
172
|
|
|
173
|
+
const saturate = (tf: number) => (tf * 2.2) / (tf + 1.2);
|
|
174
|
+
|
|
172
175
|
function countTerm(term: string, text: string): number {
|
|
173
176
|
let n = 0;
|
|
174
177
|
let i = 0;
|
|
@@ -192,12 +195,19 @@ function normTitle(s: string): string {
|
|
|
192
195
|
.trim();
|
|
193
196
|
}
|
|
194
197
|
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
198
|
+
interface NormTitle {
|
|
199
|
+
title: string;
|
|
200
|
+
words: string[];
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function normFor(s: string): NormTitle {
|
|
204
|
+
const title = normTitle(s);
|
|
205
|
+
return { title, words: [...new Set(title.split(' ').filter(Boolean))] };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function isNearDuplicate(a: NormTitle, b: NormTitle): boolean {
|
|
209
|
+
const { title: A, words: ta } = a;
|
|
210
|
+
const { title: B, words: tb } = b;
|
|
201
211
|
if (ta.length < 3 || tb.length < 3) return false;
|
|
202
212
|
if (A === B) return true;
|
|
203
213
|
const inter = ta.filter((w) => tb.includes(w)).length;
|
|
@@ -208,9 +218,12 @@ function isNearDuplicate(a: { title: string }, b: { title: string }): boolean {
|
|
|
208
218
|
|
|
209
219
|
function collapseNearDuplicates(fused: FusedHit[]): { kept: FusedHit[]; twins: number[] } {
|
|
210
220
|
const kept: FusedHit[] = [];
|
|
221
|
+
const norms = new Map<FusedHit, NormTitle>();
|
|
211
222
|
const twins: number[] = [];
|
|
212
223
|
for (const h of fused) {
|
|
213
|
-
const
|
|
224
|
+
const hn = normFor(h.title);
|
|
225
|
+
norms.set(h, hn);
|
|
226
|
+
const twin = kept.find((k) => isNearDuplicate(norms.get(k)!, hn));
|
|
214
227
|
if (twin) {
|
|
215
228
|
if (hostOf(h.url) !== hostOf(twin.url)) {
|
|
216
229
|
const idx = kept.indexOf(twin);
|
|
@@ -252,8 +265,7 @@ export function coverageLow(hits: { title: string; snippet: string }[], query: s
|
|
|
252
265
|
return hits.map((h) => queryTermCoverage([h], query) < COVERAGE_LOW_THRESHOLD);
|
|
253
266
|
}
|
|
254
267
|
|
|
255
|
-
export function fuse(outcomes: EngineOutcome[],
|
|
256
|
-
const { query } = opts;
|
|
268
|
+
export function fuse(outcomes: EngineOutcome[], query?: string): FusedHit[] {
|
|
257
269
|
const terms = query ? queryTerms(query) : [];
|
|
258
270
|
const rank: Record<string, { hit: SearchHit; ranks: { engine: string; rank: number }[] }> = {};
|
|
259
271
|
let answering = 0;
|
package/src/keys.ts
CHANGED
|
@@ -5,16 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { mkdirSync, readFileSync, writeFileSync, renameSync, chmodSync, existsSync } from 'node:fs';
|
|
7
7
|
import { dirname } from 'node:path';
|
|
8
|
-
import {
|
|
9
|
-
getConfig,
|
|
10
|
-
keyFilePath,
|
|
11
|
-
keyOrigins,
|
|
12
|
-
keyStatus,
|
|
13
|
-
parseEnvLine,
|
|
14
|
-
PROVIDERS as PROVIDER_LIST,
|
|
15
|
-
disabledFlagPath,
|
|
16
|
-
type ProviderInfo,
|
|
17
|
-
} from './config';
|
|
8
|
+
import { keyFilePath, parseEnvLine, type ProviderInfo } from './config';
|
|
18
9
|
|
|
19
10
|
function providerNames(p: ProviderInfo): string[] {
|
|
20
11
|
return [p.envName, ...p.aliases];
|
|
@@ -40,7 +31,7 @@ function mutateFile(mutations: Map<string, string | undefined>): void {
|
|
|
40
31
|
for (const [name, val] of mutations) {
|
|
41
32
|
if (val) incoming.push(`${name}=${val}`);
|
|
42
33
|
}
|
|
43
|
-
if (incoming.length) text = (text.trimEnd() ? text.trimEnd()
|
|
34
|
+
if (incoming.length) text = (text.trimEnd() ? `${text.trimEnd()}\n` : '') + `${incoming.join('\n')}\n`;
|
|
44
35
|
writeAtomic(path, text);
|
|
45
36
|
}
|
|
46
37
|
|
|
@@ -67,14 +58,3 @@ export function writeKey(p: ProviderInfo, value: string): void {
|
|
|
67
58
|
export function removeKey(p: ProviderInfo): void {
|
|
68
59
|
mutateFile(allNamesRemoved(p));
|
|
69
60
|
}
|
|
70
|
-
|
|
71
|
-
function originLabel(id: string, origin?: string): string {
|
|
72
|
-
if (origin === 'wsearch') return 'wsearch env';
|
|
73
|
-
if (origin === 'env') return `env: ${envNameFor(id)}`;
|
|
74
|
-
return 'shell';
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
function envNameFor(id: string): string {
|
|
78
|
-
const spec = PROVIDER_LIST.find((p) => p.id === id);
|
|
79
|
-
return spec?.envName ?? id;
|
|
80
|
-
}
|
package/src/login.ts
CHANGED
|
@@ -45,7 +45,7 @@ function sortProviders(list: ProviderInfo[]): ProviderInfo[] {
|
|
|
45
45
|
class ProviderPicker implements Component {
|
|
46
46
|
private filtered: ProviderPick[];
|
|
47
47
|
private selectedIndex = 0;
|
|
48
|
-
private searchInput = new Input({ placeholder: '
|
|
48
|
+
private searchInput = new Input({ placeholder: 'Filter', placeholderStyle: (t) => this.theme.fg('dim', t) });
|
|
49
49
|
private cached: string[] | undefined;
|
|
50
50
|
|
|
51
51
|
constructor(
|