termsearch 0.3.11 → 0.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/frontend/dist/app.js +39 -20
- package/frontend/dist/style.css +9 -1
- package/package.json +1 -1
- package/src/autostart/manager.js +14 -2
- package/src/search/engine.js +32 -4
- package/src/search/providers/ecosia.js +83 -0
- package/src/search/providers/qwant.js +72 -0
- package/src/search/providers/startpage.js +106 -0
- package/src/server.js +6 -0
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# TermSearch
|
|
2
2
|
|
|
3
|
-
[](https://www.npmjs.com/package/termsearch)
|
|
4
4
|
[](LICENSE)
|
|
5
5
|
[](https://nodejs.org)
|
|
6
6
|
[](#)
|
|
@@ -21,9 +21,9 @@ Opens `http://localhost:3000`. That's it.
|
|
|
21
21
|
|
|
22
22
|
## What You Get
|
|
23
23
|
|
|
24
|
-
**Search** — DuckDuckGo
|
|
24
|
+
**Search** — DuckDuckGo, Wikipedia, Startpage, Qwant, Ecosia, GitHub, Yandex, Marginalia, Ahmia out of the box. Add Brave, Mojeek, or your own SearXNG for more coverage. Engine picker lets you mix and match per-search.
|
|
25
25
|
|
|
26
|
-
**AI Summaries** — Connect any OpenAI-compatible endpoint (Ollama, LM Studio, llama.cpp, Chutes.ai, Anthropic, OpenAI). 2-phase agentic flow: AI picks sources, reads pages, synthesizes an answer. Session memory carries context across queries.
|
|
26
|
+
**AI Summaries** — Connect any OpenAI-compatible endpoint (Ollama, LM Studio, llama.cpp, Chutes.ai, OpenRoute.ai, Anthropic, OpenAI). 2-phase agentic flow: AI picks sources, reads pages, synthesizes an answer. Session memory carries context across queries.
|
|
27
27
|
|
|
28
28
|
**Social Profiler** — Paste a GitHub/Bluesky/Reddit/Twitter URL or @handle, get a profile card with stats, top repos, similar accounts.
|
|
29
29
|
|
|
@@ -35,7 +35,7 @@ Opens `http://localhost:3000`. That's it.
|
|
|
35
35
|
|
|
36
36
|
| Level | Config | What works |
|
|
37
37
|
|-------|--------|------------|
|
|
38
|
-
| **0** | None | DuckDuckGo + Wikipedia + Yandex + Marginalia + Ahmia |
|
|
38
|
+
| **0** | None | DuckDuckGo + Wikipedia + Startpage + Qwant + Ecosia + GitHub + Yandex + Marginalia + Ahmia |
|
|
39
39
|
| **1** | API keys (Settings) | + Brave, Mojeek |
|
|
40
40
|
| **2** | AI endpoint (Settings) | + AI summaries, query refinement, session memory |
|
|
41
41
|
| **3** | SearXNG URL | + 40 engines via your SearXNG instance |
|
|
@@ -67,6 +67,7 @@ Configure in **Settings > AI** from the browser. Presets auto-fill endpoint and
|
|
|
67
67
|
| LM Studio | `localhost:1234/v1` | no | — |
|
|
68
68
|
| llama.cpp | `localhost:8080/v1` | no | — |
|
|
69
69
|
| Chutes.ai TEE | `llm.chutes.ai/v1` | yes | `DeepSeek-V3.2-TEE` |
|
|
70
|
+
| OpenRoute.ai | `openroute.ai/api/v1` | yes | `deepseek/deepseek-chat` |
|
|
70
71
|
| Anthropic | `api.anthropic.com/v1` | yes | `claude-3-5-haiku-latest` |
|
|
71
72
|
| OpenAI | `api.openai.com/v1` | yes | `gpt-4o-mini` |
|
|
72
73
|
| Custom | any OpenAI-compatible | optional | — |
|
|
@@ -75,7 +76,7 @@ Load Models button auto-discovers available models from the endpoint.
|
|
|
75
76
|
|
|
76
77
|
## Search Engines
|
|
77
78
|
|
|
78
|
-
**Zero-config** (no API key): DuckDuckGo, Wikipedia, GitHub, Yandex, Ahmia, Marginalia
|
|
79
|
+
**Zero-config** (no API key): DuckDuckGo, Wikipedia, Startpage, Qwant, Ecosia, GitHub, Yandex, Ahmia, Marginalia
|
|
79
80
|
|
|
80
81
|
**API key** (toggle in Settings): Brave Search, Mojeek
|
|
81
82
|
|
|
@@ -107,7 +108,7 @@ src/
|
|
|
107
108
|
config/ manager + defaults + env overrides
|
|
108
109
|
search/
|
|
109
110
|
engine.js fan-out, merge, rank, health tracking
|
|
110
|
-
providers/ ddg, wikipedia, brave, mojeek, searxng, github, yandex, ahmia, marginalia
|
|
111
|
+
providers/ ddg, wikipedia, startpage, qwant, ecosia, brave, mojeek, searxng, github, yandex, ahmia, marginalia
|
|
111
112
|
ranking.js source diversity ranking
|
|
112
113
|
cache.js tiered L1+L2 cache
|
|
113
114
|
ai/
|
package/frontend/dist/app.js
CHANGED
|
@@ -265,13 +265,14 @@ const AI_PRESETS = [
|
|
|
265
265
|
{ id: 'ollama', label: 'LocalHost — Ollama', api_base: 'http://127.0.0.1:11434/v1', keyRequired: false, defaultModel: 'qwen3.5:4b' },
|
|
266
266
|
{ id: 'lmstudio', label: 'LocalHost — LM Studio', api_base: 'http://127.0.0.1:1234/v1', keyRequired: false, defaultModel: '' },
|
|
267
267
|
{ id: 'llamacpp', label: 'LocalHost — llama.cpp', api_base: 'http://127.0.0.1:8080/v1', keyRequired: false, defaultModel: '' },
|
|
268
|
-
{ id: 'chutes',
|
|
269
|
-
{ id: '
|
|
270
|
-
{ id: '
|
|
268
|
+
{ id: 'chutes', label: 'Chutes.ai TEE', api_base: 'https://llm.chutes.ai/v1', keyRequired: true, defaultModel: 'deepseek-ai/DeepSeek-V3.2-TEE' },
|
|
269
|
+
{ id: 'openroute', label: 'OpenRoute.ai', api_base: 'https://openroute.ai/api/v1', keyRequired: true, defaultModel: 'deepseek/deepseek-chat' },
|
|
270
|
+
{ id: 'anthropic', label: 'Anthropic', api_base: 'https://api.anthropic.com/v1', keyRequired: true, defaultModel: 'claude-3-5-haiku-latest' },
|
|
271
|
+
{ id: 'openai', label: 'OpenAI', api_base: 'https://api.openai.com/v1', keyRequired: true, defaultModel: 'gpt-4o-mini' },
|
|
271
272
|
];
|
|
272
273
|
|
|
273
274
|
const ENGINE_GROUPS = [
|
|
274
|
-
{ label: 'Web Core', items: ['duckduckgo', 'wikipedia', '
|
|
275
|
+
{ label: 'Web Core', items: ['duckduckgo', 'wikipedia', 'startpage', 'qwant', 'ecosia', 'brave', 'mojeek', 'bing', 'google', 'yahoo'] },
|
|
275
276
|
{ label: 'Uncensored', items: ['yandex', 'marginalia', 'ahmia'] },
|
|
276
277
|
{ label: 'Code & Dev', items: ['github', 'github-api', 'hackernews', 'reddit'] },
|
|
277
278
|
{ label: 'Media', items: ['youtube', 'sepiasearch'] },
|
|
@@ -288,7 +289,7 @@ const ENGINE_PRESETS = [
|
|
|
288
289
|
|
|
289
290
|
// ─── Engine availability (requires config) ────────────────────────────────
|
|
290
291
|
const SEARXNG_ROUTED = new Set([
|
|
291
|
-
'bing', 'google', 'yahoo',
|
|
292
|
+
'bing', 'google', 'yahoo',
|
|
292
293
|
'youtube', 'reddit', 'hackernews', 'sepiasearch',
|
|
293
294
|
'wikidata', 'crossref', 'openalex', 'openlibrary',
|
|
294
295
|
'mastodon users', 'mastodon hashtags', 'tootfinder',
|
|
@@ -303,6 +304,7 @@ function isEngineAvailable(engine) {
|
|
|
303
304
|
if (engine === 'yandex') return cfg.yandex?.enabled !== false;
|
|
304
305
|
if (engine === 'ahmia') return cfg.ahmia?.enabled !== false;
|
|
305
306
|
if (engine === 'marginalia') return cfg.marginalia?.enabled !== false;
|
|
307
|
+
if (engine === 'startpage' || engine === 'qwant' || engine === 'ecosia') return true;
|
|
306
308
|
if (SEARXNG_ROUTED.has(engine)) return Boolean(cfg.searxng?.enabled && cfg.searxng?.url);
|
|
307
309
|
return true; // duckduckgo, wikipedia, github, github-api — sempre disponibili
|
|
308
310
|
}
|
|
@@ -592,7 +594,7 @@ function renderAiPanel() {
|
|
|
592
594
|
// Steps
|
|
593
595
|
const showSteps = isLoading && state.aiSteps.length > 0;
|
|
594
596
|
const stepsEl = showSteps ? el('div', { className: 'ai-steps' },
|
|
595
|
-
...state.aiSteps.slice(-
|
|
597
|
+
...state.aiSteps.slice(-2).map(s => el('div', { className: 'ai-step' }, s)),
|
|
596
598
|
) : null;
|
|
597
599
|
|
|
598
600
|
// Content
|
|
@@ -708,9 +710,17 @@ function ProfilerPanel(data) {
|
|
|
708
710
|
info.append(nameEl);
|
|
709
711
|
if (profile.handle) info.append(el('div', { className: 'profile-handle' }, '@' + profile.handle));
|
|
710
712
|
if (profile.bio) info.append(el('div', { className: 'profile-bio' }, profile.bio));
|
|
713
|
+
if (profile.blog) {
|
|
714
|
+
const safeBlog = sanitizeHttpUrl(profile.blog.startsWith('http') ? profile.blog : 'https://' + profile.blog);
|
|
715
|
+
if (safeBlog) info.append(el('a', { className: 'profile-blog', href: safeBlog, target: '_blank', rel: 'noopener' }, profile.blog.replace(/^https?:\/\//, '')));
|
|
716
|
+
}
|
|
711
717
|
|
|
712
718
|
// Extra metadata
|
|
713
719
|
const extras = [profile.location, profile.company].filter(Boolean);
|
|
720
|
+
if (profile.createdAt) {
|
|
721
|
+
const d = new Date(profile.createdAt);
|
|
722
|
+
if (!isNaN(d)) extras.push('Joined ' + d.toLocaleDateString(undefined, { year: 'numeric', month: 'short' }));
|
|
723
|
+
}
|
|
714
724
|
if (extras.length) {
|
|
715
725
|
info.append(el('div', { style: 'font-size:10px;color:var(--text3);margin-top:4px' }, extras.join(' · ')));
|
|
716
726
|
}
|
|
@@ -719,13 +729,17 @@ function ProfilerPanel(data) {
|
|
|
719
729
|
|
|
720
730
|
// Stats
|
|
721
731
|
const statsFields = [
|
|
722
|
-
{ key: 'followers',
|
|
723
|
-
{ key: 'following',
|
|
724
|
-
{ key: 'repos',
|
|
725
|
-
{ key: '
|
|
726
|
-
{ key: '
|
|
727
|
-
{ key: '
|
|
728
|
-
{ key: '
|
|
732
|
+
{ key: 'followers', label: 'Followers' },
|
|
733
|
+
{ key: 'following', label: 'Following' },
|
|
734
|
+
{ key: 'repos', label: 'Repos' },
|
|
735
|
+
{ key: 'publicRepos', label: 'Repos' },
|
|
736
|
+
{ key: 'karma', label: 'Karma' },
|
|
737
|
+
{ key: 'linkKarma', label: 'Link↑' },
|
|
738
|
+
{ key: 'commentKarma', label: 'Comment↑' },
|
|
739
|
+
{ key: 'posts', label: 'Posts' },
|
|
740
|
+
{ key: 'postsCount', label: 'Posts' },
|
|
741
|
+
{ key: 'subscribers', label: 'Subscribers' },
|
|
742
|
+
{ key: 'likes', label: 'Likes' },
|
|
729
743
|
];
|
|
730
744
|
const statsData = statsFields.filter(f => profile[f.key] != null);
|
|
731
745
|
if (statsData.length) {
|
|
@@ -1132,6 +1146,10 @@ function renderApp() {
|
|
|
1132
1146
|
|
|
1133
1147
|
const main = el('div', { className: 'main' });
|
|
1134
1148
|
|
|
1149
|
+
// Profiler panel appears first (above AI panel)
|
|
1150
|
+
const profPanel = ProfilerPanel(state.profilerData || (state.profilerLoading ? { target: null, profile: null } : null));
|
|
1151
|
+
if (profPanel) main.append(profPanel);
|
|
1152
|
+
|
|
1135
1153
|
// AI panel placeholder
|
|
1136
1154
|
const aiPanel = el('div', { id: 'ai-panel', className: 'panel panel-ai', style: 'display:none' });
|
|
1137
1155
|
main.append(aiPanel);
|
|
@@ -1152,10 +1170,6 @@ function renderApp() {
|
|
|
1152
1170
|
main.append(meta);
|
|
1153
1171
|
}
|
|
1154
1172
|
|
|
1155
|
-
// 1. Profiler
|
|
1156
|
-
const profPanel = ProfilerPanel(state.profilerData || (state.profilerLoading ? { target: null, profile: null } : null));
|
|
1157
|
-
if (profPanel) main.append(profPanel);
|
|
1158
|
-
|
|
1159
1173
|
// 2. Torrent panel (only if there are magnets)
|
|
1160
1174
|
const torPanel = TorrentPanel(state.torrentData);
|
|
1161
1175
|
if (torPanel) main.append(torPanel);
|
|
@@ -1180,11 +1194,12 @@ function renderApp() {
|
|
|
1180
1194
|
|
|
1181
1195
|
// ─── Homepage ─────────────────────────────────────────────────────────────
|
|
1182
1196
|
function addToBrowser() {
|
|
1197
|
+
let attemptedLegacyAdd = false;
|
|
1183
1198
|
// Try legacy Firefox API
|
|
1184
1199
|
try {
|
|
1185
1200
|
if (window.external?.AddSearchProvider) {
|
|
1201
|
+
attemptedLegacyAdd = true;
|
|
1186
1202
|
window.external.AddSearchProvider(location.origin + '/opensearch.xml');
|
|
1187
|
-
return;
|
|
1188
1203
|
}
|
|
1189
1204
|
} catch {}
|
|
1190
1205
|
// Show inline hint
|
|
@@ -1199,6 +1214,7 @@ function addToBrowser() {
|
|
|
1199
1214
|
urlBox.append(urlText, copyBtn);
|
|
1200
1215
|
const hint = el('div', { className: 'add-browser-hint' },
|
|
1201
1216
|
el('div', { className: 'add-browser-title' }, 'Add TermSearch to your browser'),
|
|
1217
|
+
attemptedLegacyAdd ? el('div', { className: 'add-browser-note' }, 'If no browser prompt appears, add it manually with the URL below.') : null,
|
|
1202
1218
|
el('div', { className: 'add-browser-steps' },
|
|
1203
1219
|
el('div', {}, el('span', { className: 'add-browser-badge' }, 'Firefox'), ' Address bar → click TermSearch icon → "Add"'),
|
|
1204
1220
|
el('div', {}, el('span', { className: 'add-browser-badge' }, 'Chrome'), ' Settings → Search engine → Manage → Add'),
|
|
@@ -1207,7 +1223,10 @@ function addToBrowser() {
|
|
|
1207
1223
|
el('div', { className: 'add-browser-label' }, 'Search URL (paste in browser settings):'),
|
|
1208
1224
|
urlBox,
|
|
1209
1225
|
);
|
|
1210
|
-
document.querySelector('.footer')
|
|
1226
|
+
const footer = document.querySelector('.footer');
|
|
1227
|
+
if (footer) footer.before(hint);
|
|
1228
|
+
else document.getElementById('app')?.append(hint);
|
|
1229
|
+
hint.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
|
1211
1230
|
setTimeout(() => hint.remove(), 20000);
|
|
1212
1231
|
}
|
|
1213
1232
|
|
|
@@ -1594,7 +1613,7 @@ async function renderSettings() {
|
|
|
1594
1613
|
el('label', { className: 'form-label', for: 'ai-base' }, 'API Endpoint'),
|
|
1595
1614
|
makeInput('ai-base', ai.api_base, 'http://localhost:11434/v1'),
|
|
1596
1615
|
el('div', { className: 'form-hint' },
|
|
1597
|
-
'Included presets: LocalHost (Ollama · LM Studio · llama.cpp) · Chutes.ai TEE · Anthropic · OpenAI',
|
|
1616
|
+
'Included presets: LocalHost (Ollama · LM Studio · llama.cpp) · Chutes.ai TEE · OpenRoute.ai · Anthropic · OpenAI',
|
|
1598
1617
|
el('br', {}),
|
|
1599
1618
|
'You can also keep custom OpenAI-compatible endpoints.',
|
|
1600
1619
|
),
|
package/frontend/dist/style.css
CHANGED
|
@@ -688,9 +688,10 @@ a:hover { color: var(--link-h); }
|
|
|
688
688
|
}
|
|
689
689
|
.ai-content.ai-content-collapsed {
|
|
690
690
|
display: -webkit-box;
|
|
691
|
-
-webkit-line-clamp:
|
|
691
|
+
-webkit-line-clamp: 5;
|
|
692
692
|
-webkit-box-orient: vertical;
|
|
693
693
|
overflow: hidden;
|
|
694
|
+
max-height: 8.5em;
|
|
694
695
|
}
|
|
695
696
|
|
|
696
697
|
/* Sources */
|
|
@@ -740,6 +741,8 @@ a:hover { color: var(--link-h); }
|
|
|
740
741
|
.profile-name:hover { color: var(--prof-text); }
|
|
741
742
|
.profile-handle { font-size: 11px; color: var(--text3); margin-top: 1px; }
|
|
742
743
|
.profile-bio { font-size: 12px; color: var(--text2); margin-top: 6px; line-height: 1.5; }
|
|
744
|
+
.profile-blog { display: block; font-size: 11px; color: var(--accent); margin-top: 3px; text-decoration: none; opacity: 0.85; }
|
|
745
|
+
.profile-blog:hover { opacity: 1; text-decoration: underline; }
|
|
743
746
|
|
|
744
747
|
.profile-stats { display: flex; flex-wrap: wrap; gap: 16px; padding-top: 12px; border-top: 1px solid var(--prof-dim); }
|
|
745
748
|
.stat { display: flex; flex-direction: column; gap: 1px; }
|
|
@@ -1005,6 +1008,11 @@ a:hover { color: var(--link-h); }
|
|
|
1005
1008
|
color: var(--text);
|
|
1006
1009
|
margin-bottom: 10px;
|
|
1007
1010
|
}
|
|
1011
|
+
.add-browser-note {
|
|
1012
|
+
margin-bottom: 10px;
|
|
1013
|
+
font-size: 11px;
|
|
1014
|
+
color: #c4b5fd;
|
|
1015
|
+
}
|
|
1008
1016
|
.add-browser-steps {
|
|
1009
1017
|
display: flex;
|
|
1010
1018
|
flex-direction: column;
|
package/package.json
CHANGED
package/src/autostart/manager.js
CHANGED
|
@@ -49,8 +49,20 @@ function termuxStatus() {
|
|
|
49
49
|
|
|
50
50
|
function termuxEnable() {
|
|
51
51
|
fs.mkdirSync(TERMUX_BOOT_DIR, { recursive: true });
|
|
52
|
-
|
|
53
|
-
const
|
|
52
|
+
// Termux:Boot runs scripts with a minimal PATH — resolve everything to absolute paths now
|
|
53
|
+
const prefix = process.env.PREFIX || '/data/data/com.termux/files/usr';
|
|
54
|
+
const homeDir = os.homedir();
|
|
55
|
+
const nodeBin = process.execPath; // absolute path to node binary
|
|
56
|
+
const binPath = findBin(); // absolute path to termsearch script
|
|
57
|
+
const sh = [
|
|
58
|
+
'#!/data/data/com.termux/files/usr/bin/sh',
|
|
59
|
+
'# TermSearch autostart (generated by termsearch autostart enable)',
|
|
60
|
+
`export HOME="${homeDir}"`,
|
|
61
|
+
`export PREFIX="${prefix}"`,
|
|
62
|
+
`export PATH="${prefix}/bin:${homeDir}/.npm-global/bin:\$PATH"`,
|
|
63
|
+
`"${nodeBin}" "${binPath}" start --fg &`,
|
|
64
|
+
'',
|
|
65
|
+
].join('\n');
|
|
54
66
|
fs.writeFileSync(TERMUX_BOOT_FILE, sh, { mode: 0o755 });
|
|
55
67
|
}
|
|
56
68
|
|
package/src/search/engine.js
CHANGED
|
@@ -12,6 +12,9 @@ import * as github from './providers/github.js';
|
|
|
12
12
|
import * as yandex from './providers/yandex.js';
|
|
13
13
|
import * as ahmia from './providers/ahmia.js';
|
|
14
14
|
import * as marginalia from './providers/marginalia.js';
|
|
15
|
+
import * as startpage from './providers/startpage.js';
|
|
16
|
+
import * as qwant from './providers/qwant.js';
|
|
17
|
+
import * as ecosia from './providers/ecosia.js';
|
|
15
18
|
|
|
16
19
|
let _searchCache = null;
|
|
17
20
|
let _docCache = null;
|
|
@@ -52,6 +55,10 @@ export const ALLOWED_ENGINES = new Set([
|
|
|
52
55
|
'1337x',
|
|
53
56
|
'piratebay',
|
|
54
57
|
'nyaa',
|
|
58
|
+
// native scrapers
|
|
59
|
+
'startpage',
|
|
60
|
+
'qwant',
|
|
61
|
+
'ecosia',
|
|
55
62
|
// uncensored / alternative index engines
|
|
56
63
|
'yandex',
|
|
57
64
|
'ahmia',
|
|
@@ -97,6 +104,24 @@ const PROVIDER_REGISTRY = {
|
|
|
97
104
|
run: searxng.search,
|
|
98
105
|
defaultProvider: true,
|
|
99
106
|
},
|
|
107
|
+
startpage: {
|
|
108
|
+
aliases: new Set(['startpage']),
|
|
109
|
+
enabled: (_cfg) => true,
|
|
110
|
+
run: startpage.search,
|
|
111
|
+
defaultProvider: true,
|
|
112
|
+
},
|
|
113
|
+
qwant: {
|
|
114
|
+
aliases: new Set(['qwant']),
|
|
115
|
+
enabled: (_cfg) => true,
|
|
116
|
+
run: qwant.search,
|
|
117
|
+
defaultProvider: true,
|
|
118
|
+
},
|
|
119
|
+
ecosia: {
|
|
120
|
+
aliases: new Set(['ecosia']),
|
|
121
|
+
enabled: (_cfg) => true,
|
|
122
|
+
run: ecosia.search,
|
|
123
|
+
defaultProvider: true,
|
|
124
|
+
},
|
|
100
125
|
github: {
|
|
101
126
|
aliases: new Set(['github', 'github-api']),
|
|
102
127
|
enabled: (_cfg) => true,
|
|
@@ -107,19 +132,19 @@ const PROVIDER_REGISTRY = {
|
|
|
107
132
|
aliases: new Set(['yandex']),
|
|
108
133
|
enabled: (cfg) => cfg?.yandex?.enabled !== false,
|
|
109
134
|
run: yandex.search,
|
|
110
|
-
defaultProvider:
|
|
135
|
+
defaultProvider: true,
|
|
111
136
|
},
|
|
112
137
|
ahmia: {
|
|
113
138
|
aliases: new Set(['ahmia']),
|
|
114
139
|
enabled: (cfg) => cfg?.ahmia?.enabled !== false,
|
|
115
140
|
run: ahmia.search,
|
|
116
|
-
defaultProvider:
|
|
141
|
+
defaultProvider: true,
|
|
117
142
|
},
|
|
118
143
|
marginalia: {
|
|
119
144
|
aliases: new Set(['marginalia']),
|
|
120
145
|
enabled: (cfg) => cfg?.marginalia?.enabled !== false,
|
|
121
146
|
run: marginalia.search,
|
|
122
|
-
defaultProvider:
|
|
147
|
+
defaultProvider: true,
|
|
123
148
|
},
|
|
124
149
|
};
|
|
125
150
|
|
|
@@ -485,7 +510,10 @@ export async function search({ query, lang = 'en-US', safe = '1', page = 1, cate
|
|
|
485
510
|
};
|
|
486
511
|
});
|
|
487
512
|
|
|
488
|
-
|
|
513
|
+
// Don't cache empty results — likely a transient block/CAPTCHA; retry on next request
|
|
514
|
+
if (response.results.length > 0) {
|
|
515
|
+
_searchCache.set(cacheKey, response, cfg.search.cache_ttl_search_ms);
|
|
516
|
+
}
|
|
489
517
|
return response;
|
|
490
518
|
}
|
|
491
519
|
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// Ecosia HTML scraper — Bing-powered, no API key required
|
|
2
|
+
// Privacy-focused, plants trees with ad revenue
|
|
3
|
+
|
|
4
|
+
const ECOSIA_ENDPOINT = 'https://www.ecosia.org/search';
|
|
5
|
+
|
|
6
|
+
const USER_AGENTS = [
|
|
7
|
+
'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
|
|
8
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
|
9
|
+
];
|
|
10
|
+
|
|
11
|
+
function randomUA() {
|
|
12
|
+
return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function ent(s) {
|
|
16
|
+
return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/ /g, ' ').replace(/&#\d+;/g, '').trim();
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function parseEcosia(html) {
|
|
20
|
+
const results = [];
|
|
21
|
+
|
|
22
|
+
// Primary: result__body blocks (standard Ecosia layout)
|
|
23
|
+
const blockRe = /class="[^"]*result(?:__body|-body)[^"]*"[^>]*>([\s\S]*?)(?=class="[^"]*result(?:__body|-body)|class="[^"]*pagination|<\/main)/gi;
|
|
24
|
+
let m;
|
|
25
|
+
while ((m = blockRe.exec(html)) !== null && results.length < 15) {
|
|
26
|
+
const block = m[1];
|
|
27
|
+
// Title anchor — skip ecosia.org internal links
|
|
28
|
+
const aMatch = block.match(/<a[^>]+href="(https?:\/\/(?!(?:[^/]*\.)?ecosia\.org)[^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
|
29
|
+
if (!aMatch) continue;
|
|
30
|
+
const url = aMatch[1];
|
|
31
|
+
if (url.length > 500) continue;
|
|
32
|
+
const title = ent(aMatch[2].replace(/<[^>]+>/g, ''));
|
|
33
|
+
if (!title) continue;
|
|
34
|
+
const snipM = block.match(/class="[^"]*(?:result-snippet|result__description|result__body-text|result__content)[^"]*"[^>]*>([\s\S]*?)<\/(?:p|div|span)>/i);
|
|
35
|
+
const snippet = snipM ? ent(snipM[1].replace(/<[^>]+>/g, '')).slice(0, 300) : '';
|
|
36
|
+
results.push({ title, url, snippet, engine: 'ecosia', score: 0 });
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Fallback: external links that have a heading nearby
|
|
40
|
+
if (results.length === 0) {
|
|
41
|
+
const linkRe = /<a[^>]+href="(https?:\/\/(?!(?:[^/]*\.)?ecosia\.org)[^"]+)"[^>]*>[\s]*<(?:h[1-4]|strong)[^>]*>([\s\S]*?)<\/(?:h[1-4]|strong)>/gi;
|
|
42
|
+
while ((m = linkRe.exec(html)) !== null && results.length < 15) {
|
|
43
|
+
const url = m[1];
|
|
44
|
+
const title = ent(m[2].replace(/<[^>]+>/g, ''));
|
|
45
|
+
if (!title || url.length > 500) continue;
|
|
46
|
+
results.push({ title, url, snippet: '', engine: 'ecosia', score: 0 });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
return results;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function search({ query, lang = 'en-US', safe = '1', page = 1, timeoutMs = 12000 }) {
|
|
54
|
+
const params = new URLSearchParams({ q: query, c: 'web' });
|
|
55
|
+
if (page > 1) params.set('p', String(page - 1));
|
|
56
|
+
|
|
57
|
+
const ac = new AbortController();
|
|
58
|
+
const timer = setTimeout(() => ac.abort(), timeoutMs);
|
|
59
|
+
try {
|
|
60
|
+
const r = await fetch(`${ECOSIA_ENDPOINT}?${params}`, {
|
|
61
|
+
headers: {
|
|
62
|
+
'User-Agent': randomUA(),
|
|
63
|
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
64
|
+
'Accept-Language': lang.slice(0, 2) + ',en;q=0.5',
|
|
65
|
+
'Referer': 'https://www.ecosia.org/',
|
|
66
|
+
'DNT': '1',
|
|
67
|
+
},
|
|
68
|
+
signal: ac.signal,
|
|
69
|
+
});
|
|
70
|
+
clearTimeout(timer);
|
|
71
|
+
if (!r.ok) return { results: [], _meta: { error: `ecosia_http_${r.status}` } };
|
|
72
|
+
const html = await r.text();
|
|
73
|
+
if (html.length < 2000 || html.includes('cf-challenge') || html.includes('captcha')) {
|
|
74
|
+
return { results: [], _meta: { error: 'ecosia_blocked' } };
|
|
75
|
+
}
|
|
76
|
+
const results = parseEcosia(html);
|
|
77
|
+
if (results.length === 0) return { results: [], _meta: { empty: true, skipHealth: true } };
|
|
78
|
+
return { results, _meta: {} };
|
|
79
|
+
} catch {
|
|
80
|
+
clearTimeout(timer);
|
|
81
|
+
return { results: [], _meta: { error: 'ecosia_unreachable' } };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Qwant JSON API — no API key required, semi-public endpoint
|
|
2
|
+
// Independent EU index; privacy-focused; results from own crawler + Bing blend
|
|
3
|
+
|
|
4
|
+
const QWANT_API = 'https://api.qwant.com/v3/search/web';
|
|
5
|
+
|
|
6
|
+
const UA = 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0';
|
|
7
|
+
|
|
8
|
+
function localeParam(lang) {
|
|
9
|
+
const map = {
|
|
10
|
+
'en-US': 'en_US', 'en-GB': 'en_GB',
|
|
11
|
+
'it-IT': 'it_IT', 'de-DE': 'de_DE', 'fr-FR': 'fr_FR',
|
|
12
|
+
'es-ES': 'es_ES', 'pt-PT': 'pt_PT', 'nl-NL': 'nl_NL',
|
|
13
|
+
'pl-PL': 'pl_PL', 'ru-RU': 'ru_RU', 'ja-JP': 'ja_JP',
|
|
14
|
+
};
|
|
15
|
+
return map[lang] || 'en_US';
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function stripTags(s) {
|
|
19
|
+
return s.replace(/<[^>]+>/g, '').replace(/&/g, '&').replace(/ /g, ' ').trim();
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export async function search({ query, lang = 'en-US', safe = '1', page = 1, timeoutMs = 10000 }) {
|
|
23
|
+
const offset = (Math.max(1, Number(page)) - 1) * 10;
|
|
24
|
+
const params = new URLSearchParams({
|
|
25
|
+
q: query,
|
|
26
|
+
count: '10',
|
|
27
|
+
locale: localeParam(lang),
|
|
28
|
+
offset: String(offset),
|
|
29
|
+
safesearch: safe === '0' ? '0' : '1',
|
|
30
|
+
t: 'web',
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
const ac = new AbortController();
|
|
34
|
+
const timer = setTimeout(() => ac.abort(), timeoutMs);
|
|
35
|
+
try {
|
|
36
|
+
const r = await fetch(`${QWANT_API}?${params}`, {
|
|
37
|
+
headers: {
|
|
38
|
+
'Accept': 'application/json',
|
|
39
|
+
'User-Agent': UA,
|
|
40
|
+
'Referer': 'https://www.qwant.com/',
|
|
41
|
+
'Accept-Language': 'en-US,en;q=0.5',
|
|
42
|
+
},
|
|
43
|
+
signal: ac.signal,
|
|
44
|
+
});
|
|
45
|
+
clearTimeout(timer);
|
|
46
|
+
if (!r.ok) return { results: [], _meta: { error: `qwant_http_${r.status}` } };
|
|
47
|
+
const json = await r.json();
|
|
48
|
+
if (json.status !== 'success') return { results: [], _meta: { error: `qwant_api_${json.status || 'error'}` } };
|
|
49
|
+
|
|
50
|
+
// Flatten mainline sections of type "web"
|
|
51
|
+
const mainline = json?.data?.result?.items?.mainline || [];
|
|
52
|
+
const results = [];
|
|
53
|
+
for (const section of mainline) {
|
|
54
|
+
if (section.type !== 'web') continue;
|
|
55
|
+
for (const item of (section.items || [])) {
|
|
56
|
+
const url = String(item.url || '').trim();
|
|
57
|
+
const title = stripTags(String(item.title || ''));
|
|
58
|
+
const snippet = stripTags(String(item.desc || item.snippet || '')).slice(0, 300);
|
|
59
|
+
if (!url.startsWith('http') || !title) continue;
|
|
60
|
+
results.push({ title, url, snippet, engine: 'qwant', score: 0 });
|
|
61
|
+
if (results.length >= 10) break;
|
|
62
|
+
}
|
|
63
|
+
if (results.length >= 10) break;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (results.length === 0) return { results: [], _meta: { empty: true, skipHealth: true } };
|
|
67
|
+
return { results, _meta: {} };
|
|
68
|
+
} catch {
|
|
69
|
+
clearTimeout(timer);
|
|
70
|
+
return { results: [], _meta: { error: 'qwant_unreachable' } };
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Startpage HTML scraper — proxies Google index, privacy-first, no API key required
|
|
2
|
+
// Uses /sp/search POST form (maintained for non-JS clients)
|
|
3
|
+
|
|
4
|
+
const SP_ENDPOINT = 'https://www.startpage.com/sp/search';
|
|
5
|
+
|
|
6
|
+
const USER_AGENTS = [
|
|
7
|
+
'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
|
|
8
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0',
|
|
9
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
|
|
10
|
+
];
|
|
11
|
+
|
|
12
|
+
function randomUA() {
|
|
13
|
+
return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function langParam(lang) {
|
|
17
|
+
const map = {
|
|
18
|
+
'en-US': 'english', 'en-GB': 'english',
|
|
19
|
+
'it-IT': 'italian', 'de-DE': 'german', 'fr-FR': 'french',
|
|
20
|
+
'es-ES': 'spanish', 'pt-PT': 'portuguese', 'ru-RU': 'russian',
|
|
21
|
+
'nl-NL': 'dutch', 'pl-PL': 'polish',
|
|
22
|
+
};
|
|
23
|
+
return map[lang] || 'english';
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function ent(s) {
|
|
27
|
+
return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/ /g, ' ').replace(/&#\d+;/g, '').trim();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function parseStartpage(html) {
|
|
31
|
+
const results = [];
|
|
32
|
+
|
|
33
|
+
// Primary pattern: w-gl__result blocks (classic Startpage layout)
|
|
34
|
+
const blockRe = /<(?:article|section|div)[^>]+class="[^"]*w-gl__result[^"]*"[^>]*>([\s\S]*?)(?=<(?:article|section|div)[^>]+class="[^"]*w-gl__result|<\/section|id="pagination|id="new-feature-banner)/gi;
|
|
35
|
+
let m;
|
|
36
|
+
while ((m = blockRe.exec(html)) !== null && results.length < 15) {
|
|
37
|
+
const block = m[1];
|
|
38
|
+
const aMatch = block.match(/<a[^>]+class="[^"]*(?:w-gl__result-title-anchor|result-title)[^"]*"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
|
39
|
+
if (!aMatch) continue;
|
|
40
|
+
const url = aMatch[1];
|
|
41
|
+
if (!url.startsWith('http') || url.includes('startpage.com')) continue;
|
|
42
|
+
const title = ent(aMatch[2].replace(/<[^>]+>/g, ''));
|
|
43
|
+
if (!title) continue;
|
|
44
|
+
const snipM = block.match(/class="[^"]*(?:w-gl__result-description|result-desc)[^"]*"[^>]*>([\s\S]*?)<\/[a-z]+>/i);
|
|
45
|
+
const snippet = snipM ? ent(snipM[1].replace(/<[^>]+>/g, '')).slice(0, 300) : '';
|
|
46
|
+
results.push({ title, url, snippet, engine: 'startpage', score: 0 });
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Fallback: any external link preceded by a heading tag
|
|
50
|
+
if (results.length === 0) {
|
|
51
|
+
const linkRe = /<a[^>]+href="(https?:\/\/(?!(?:[^/]*\.)?startpage\.com)[^"]+)"[^>]*>[\s\S]*?<(?:h[1-4]|strong)[^>]*>([\s\S]*?)<\/(?:h[1-4]|strong)>/gi;
|
|
52
|
+
while ((m = linkRe.exec(html)) !== null && results.length < 15) {
|
|
53
|
+
const url = m[1];
|
|
54
|
+
const title = ent(m[2].replace(/<[^>]+>/g, ''));
|
|
55
|
+
if (!title || url.length > 500) continue;
|
|
56
|
+
results.push({ title, url, snippet: '', engine: 'startpage', score: 0 });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return results;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export async function search({ query, lang = 'en-US', safe = '1', page = 1, timeoutMs = 12000 }) {
|
|
64
|
+
const startat = (Math.max(1, Number(page)) - 1) * 10;
|
|
65
|
+
const body = new URLSearchParams({
|
|
66
|
+
query,
|
|
67
|
+
language: langParam(lang),
|
|
68
|
+
startat: String(startat),
|
|
69
|
+
cat: 'web',
|
|
70
|
+
cmd: 'process_search',
|
|
71
|
+
nj: '1',
|
|
72
|
+
abp: '1',
|
|
73
|
+
t: 'device',
|
|
74
|
+
with_date: '',
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
const ac = new AbortController();
|
|
78
|
+
const timer = setTimeout(() => ac.abort(), timeoutMs);
|
|
79
|
+
try {
|
|
80
|
+
const r = await fetch(SP_ENDPOINT, {
|
|
81
|
+
method: 'POST',
|
|
82
|
+
headers: {
|
|
83
|
+
'User-Agent': randomUA(),
|
|
84
|
+
'Content-Type': 'application/x-www-form-urlencoded',
|
|
85
|
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
86
|
+
'Accept-Language':'en-US,en;q=0.5',
|
|
87
|
+
'Origin': 'https://www.startpage.com',
|
|
88
|
+
'Referer': 'https://www.startpage.com/',
|
|
89
|
+
},
|
|
90
|
+
body: body.toString(),
|
|
91
|
+
signal: ac.signal,
|
|
92
|
+
});
|
|
93
|
+
clearTimeout(timer);
|
|
94
|
+
if (!r.ok) return { results: [], _meta: { error: `startpage_http_${r.status}` } };
|
|
95
|
+
const html = await r.text();
|
|
96
|
+
if (html.length < 2000 || html.includes('captcha') || html.includes('robot-check')) {
|
|
97
|
+
return { results: [], _meta: { error: 'startpage_blocked' } };
|
|
98
|
+
}
|
|
99
|
+
const results = parseStartpage(html);
|
|
100
|
+
if (results.length === 0) return { results: [], _meta: { empty: true, skipHealth: true } };
|
|
101
|
+
return { results, _meta: {} };
|
|
102
|
+
} catch {
|
|
103
|
+
clearTimeout(timer);
|
|
104
|
+
return { results: [], _meta: { error: 'startpage_unreachable' } };
|
|
105
|
+
}
|
|
106
|
+
}
|
package/src/server.js
CHANGED
|
@@ -76,6 +76,12 @@ function shutdown(signal) {
|
|
|
76
76
|
}
|
|
77
77
|
process.on('SIGINT', () => shutdown('SIGINT'));
|
|
78
78
|
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
|
79
|
+
process.on('uncaughtException', (err) => {
|
|
80
|
+
console.error('[termsearch] uncaughtException:', err);
|
|
81
|
+
});
|
|
82
|
+
process.on('unhandledRejection', (reason) => {
|
|
83
|
+
console.error('[termsearch] unhandledRejection:', reason);
|
|
84
|
+
});
|
|
79
85
|
|
|
80
86
|
export default app;
|
|
81
87
|
export { port, host };
|