search-web-api 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +307 -0
- package/demo/index.ts +38 -0
- package/demo/openapi.ts +402 -0
- package/demo/routes/autocomplete.ts +124 -0
- package/demo/routes/search.ts +61 -0
- package/docs/AUTOCOMPLETE.md +296 -0
- package/docs/CATEGORY_SEARCH.md +265 -0
- package/package.json +32 -0
- package/src/autocomplete/autocomplete-ai-next-word-predictor.ts +47 -0
- package/src/autocomplete/autocomplete-search-engine-backends.ts +231 -0
- package/src/category-registry.ts +6 -0
- package/src/config/search-engine-constants.ts +216 -0
- package/src/constants.ts +6 -0
- package/src/engine-descriptions.ts +7 -0
- package/src/engine-status.ts +7 -0
- package/src/engine.ts +6 -0
- package/src/registry/search-engine-category-registry.ts +200 -0
- package/src/registry/search-engine-descriptions.ts +121 -0
- package/src/registry/search-engine-status-tracker.ts +240 -0
- package/src/result-container.ts +8 -0
- package/src/search/search-engines-registry-list.ts +205 -0
- package/src/search/search-query-executor.ts +168 -0
- package/src/search/search-result-container.ts +421 -0
- package/src/search-web-types.ts +7 -0
- package/src/search.ts +10 -0
- package/src/sources/academic/arxiv.ts +72 -0
- package/src/sources/academic/core.ts +65 -0
- package/src/sources/academic/crossref.ts +75 -0
- package/src/sources/academic/doaj.ts +63 -0
- package/src/sources/academic/google_scholar.ts +53 -0
- package/src/sources/academic/openalex.ts +75 -0
- package/src/sources/academic/pubmed.ts +96 -0
- package/src/sources/academic/semantic_scholar.ts +77 -0
- package/src/sources/academic/wikidata.ts +44 -0
- package/src/sources/general/baidu.ts +59 -0
- package/src/sources/general/bing.ts +30 -0
- package/src/sources/general/brave.ts +54 -0
- package/src/sources/general/duckduckgo.ts +49 -0
- package/src/sources/general/google.ts +68 -0
- package/src/sources/general/mojeek.ts +56 -0
- package/src/sources/general/qwant.ts +37 -0
- package/src/sources/general/startpage.ts +44 -0
- package/src/sources/general/yahoo.ts +41 -0
- package/src/sources/general/yandex.ts +56 -0
- package/src/sources/images/bing_images.ts +68 -0
- package/src/sources/images/deviantart.ts +71 -0
- package/src/sources/images/flickr.ts +132 -0
- package/src/sources/images/google_images.ts +101 -0
- package/src/sources/images/imgur.ts +58 -0
- package/src/sources/images/openclipart.ts +52 -0
- package/src/sources/images/pixabay.ts +56 -0
- package/src/sources/images/unsplash.ts +38 -0
- package/src/sources/images/wallhaven.ts +50 -0
- package/src/sources/it/crates.ts +41 -0
- package/src/sources/it/dockerhub.ts +47 -0
- package/src/sources/it/github.ts +37 -0
- package/src/sources/it/gitlab.ts +55 -0
- package/src/sources/it/npm.ts +36 -0
- package/src/sources/it/packagist.ts +43 -0
- package/src/sources/it/pypi.ts +30 -0
- package/src/sources/it/rubygems.ts +43 -0
- package/src/sources/it/stackoverflow.ts +39 -0
- package/src/sources/maps/apple_maps.ts +105 -0
- package/src/sources/maps/openstreetmap.ts +34 -0
- package/src/sources/maps/photon.ts +77 -0
- package/src/sources/news/bing_news.ts +95 -0
- package/src/sources/news/google_news.ts +80 -0
- package/src/sources/news/hackernews.ts +94 -0
- package/src/sources/news/yahoo_news.ts +77 -0
- package/src/sources/shopping/ebay.ts +96 -0
- package/src/sources/social/mastodon.ts +46 -0
- package/src/sources/social/medium.ts +52 -0
- package/src/sources/social/reddit.ts +48 -0
- package/src/sources/social/soundcloud.ts +64 -0
- package/src/sources/social/twitter.ts +56 -0
- package/src/sources/specialized/annas_archive.ts +97 -0
- package/src/sources/specialized/archive.ts +48 -0
- package/src/sources/specialized/genius.ts +43 -0
- package/src/sources/specialized/goodreads.ts +62 -0
- package/src/sources/specialized/imdb.ts +55 -0
- package/src/sources/specialized/openlibrary.ts +59 -0
- package/src/sources/specialized/wikipedia.ts +37 -0
- package/src/sources/specialized/wttr.ts +98 -0
- package/src/sources/torrents/1337x.ts +43 -0
- package/src/sources/torrents/eztv.ts +47 -0
- package/src/sources/torrents/kickass.ts +63 -0
- package/src/sources/torrents/nyaa.ts +53 -0
- package/src/sources/torrents/solidtorrents.ts +68 -0
- package/src/sources/torrents/thepiratebay.ts +57 -0
- package/src/sources/torrents/yts.ts +54 -0
- package/src/sources/videos/bing_videos.ts +91 -0
- package/src/sources/videos/dailymotion.ts +101 -0
- package/src/sources/videos/invidious.ts +86 -0
- package/src/sources/videos/peertube.ts +76 -0
- package/src/sources/videos/vimeo.ts +67 -0
- package/src/sources/videos/youtube.ts +78 -0
- package/src/suggest-next-words/autocomplete-ai.ts +38 -0
- package/src/suggest-next-words/autocomplete-search-engines.ts +384 -0
- package/src/suggest-next-words/misspelled-typos-8k.json +1 -0
- package/src/types/search-engine-interface.ts +27 -0
- package/src/types/search-result-types.ts +405 -0
- package/test/api.test.ts +128 -0
- package/test/autocomplete-ai.test.ts +20 -0
- package/test/autocomplete-engines.test.ts +131 -0
- package/test/engine-health-suite.test.ts +350 -0
- package/test/search.test.ts +69 -0
- package/test/sources-unit.test.ts +1152 -0
- package/test/sources.test.ts +182 -0
- package/test/test-utils.ts +81 -0
- package/tsconfig.json +16 -0
- package/vitest.config.ts +20 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Autocomplete Search Engine Backends — Provides query-completion suggestions
|
|
3
|
+
* by querying the native autocomplete APIs of eight major search engines: Baidu,
|
|
4
|
+
* Brave, DuckDuckGo, Google, Qwant, Startpage, Wikipedia, and Yandex. Each
|
|
5
|
+
* backend normalises locale codes to the format expected by its API and returns
|
|
6
|
+
* a plain string array. `searchAutocompleteMulti` fans out to multiple backends
|
|
7
|
+
* in parallel and merges the deduplicated results.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import grab from "grab-url";
|
|
11
|
+
import { parseHTML } from "linkedom";
|
|
12
|
+
|
|
13
|
+
type AutocompleteFunction = (query: string, locale?: string) => Promise<string[]>;
|
|
14
|
+
|
|
15
|
+
async function getSuggestions(url: string, options: any = {}): Promise<any> {
|
|
16
|
+
try {
|
|
17
|
+
const response = await grab(url, {
|
|
18
|
+
timeout: 3,
|
|
19
|
+
responseType: options.responseType || "json",
|
|
20
|
+
...options,
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
if (typeof response === "object" && "data" in response) return response.data;
|
|
24
|
+
return response;
|
|
25
|
+
} catch (error) {
|
|
26
|
+
console.error("Autocomplete error:", error);
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Baidu autocomplete — returns suggestions from Baidu's `sugrec` endpoint.
|
|
33
|
+
*/
|
|
34
|
+
export async function baidu(query: string, _locale?: string): Promise<string[]> {
|
|
35
|
+
const params = new URLSearchParams({ ie: "utf-8", json: "1", prod: "pc", wd: query });
|
|
36
|
+
const data = await getSuggestions(`https://www.baidu.com/sugrec?${params}`);
|
|
37
|
+
const results: string[] = [];
|
|
38
|
+
if (data && data.g) for (const item of data.g) results.push(item.q);
|
|
39
|
+
return results;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Brave autocomplete — returns suggestions from Brave Search's suggest API.
|
|
44
|
+
*/
|
|
45
|
+
export async function brave(query: string, _locale?: string): Promise<string[]> {
|
|
46
|
+
const params = new URLSearchParams({ q: query });
|
|
47
|
+
const data = await getSuggestions(`https://search.brave.com/api/suggest?${params}`, {
|
|
48
|
+
headers: { Cookie: "country=all" },
|
|
49
|
+
});
|
|
50
|
+
if (data && Array.isArray(data) && data.length > 1) return data[1];
|
|
51
|
+
return [];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* DuckDuckGo autocomplete — maps `locale` (e.g. `en-US`) to a `kl` region code.
|
|
56
|
+
*/
|
|
57
|
+
export async function duckduckgo(query: string, locale: string = "en-US"): Promise<string[]> {
|
|
58
|
+
const region = locale.toLowerCase().split("-").reverse().join("-");
|
|
59
|
+
const params = new URLSearchParams({ q: query, kl: region });
|
|
60
|
+
const data = await getSuggestions(`https://duckduckgo.com/ac/?type=list&${params}`);
|
|
61
|
+
if (data && Array.isArray(data) && data.length > 1) return data[1];
|
|
62
|
+
return [];
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Google autocomplete — routes to the locale-appropriate Google subdomain and
|
|
67
|
+
* strips HTML entities from suggestion text using linkedom.
|
|
68
|
+
*/
|
|
69
|
+
export async function google(query: string, locale: string = "en"): Promise<string[]> {
|
|
70
|
+
const subdomainMap: { [key: string]: string } = {
|
|
71
|
+
de: "google.de", fr: "google.fr", es: "google.es", it: "google.it",
|
|
72
|
+
nl: "google.nl", pt: "google.pt", ru: "google.ru",
|
|
73
|
+
ja: "google.co.jp", zh: "google.com.hk", ko: "google.co.kr",
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
const lang = locale.split("-")[0];
|
|
77
|
+
const subdomain = subdomainMap[lang] || "google.com";
|
|
78
|
+
const params = new URLSearchParams({ q: query, client: "gws-wiz", hl: lang });
|
|
79
|
+
const response = await getSuggestions(
|
|
80
|
+
`https://${subdomain}/complete/search?${params}`,
|
|
81
|
+
{ responseType: "text" },
|
|
82
|
+
);
|
|
83
|
+
|
|
84
|
+
const results: string[] = [];
|
|
85
|
+
if (response) {
|
|
86
|
+
try {
|
|
87
|
+
const jsonStart = response.indexOf("[");
|
|
88
|
+
const jsonEnd = response.lastIndexOf("]") + 1;
|
|
89
|
+
if (jsonStart >= 0 && jsonEnd > jsonStart) {
|
|
90
|
+
const data = JSON.parse(response.substring(jsonStart, jsonEnd));
|
|
91
|
+
if (data[0]) {
|
|
92
|
+
for (const item of data[0]) {
|
|
93
|
+
const { document } = parseHTML(item[0]);
|
|
94
|
+
const text = document.body?.textContent?.trim();
|
|
95
|
+
if (text) results.push(text);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
} catch {
|
|
100
|
+
// Ignore parse errors
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return results;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Qwant autocomplete — uses Qwant's v3 suggest API with locale normalisation.
|
|
109
|
+
*/
|
|
110
|
+
export async function qwant(query: string, locale: string = "en_US"): Promise<string[]> {
|
|
111
|
+
const params = new URLSearchParams({
|
|
112
|
+
q: query,
|
|
113
|
+
locale: locale.replace("-", "_"),
|
|
114
|
+
version: "2",
|
|
115
|
+
});
|
|
116
|
+
const data = await getSuggestions(`https://api.qwant.com/v3/suggest?${params}`);
|
|
117
|
+
const results: string[] = [];
|
|
118
|
+
if (data && data.status === "success" && data.data?.items)
|
|
119
|
+
for (const item of data.data.items) results.push(item.value);
|
|
120
|
+
return results;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Startpage autocomplete — maps locale to the `lui` language parameter.
|
|
125
|
+
*/
|
|
126
|
+
export async function startpage(query: string, locale: string = "en"): Promise<string[]> {
|
|
127
|
+
const langMap: { [key: string]: string } = {
|
|
128
|
+
da: "dansk", de: "deutsch", en: "english", es: "espanol",
|
|
129
|
+
fr: "francais", nb: "norsk", nl: "nederlands",
|
|
130
|
+
pl: "polski", pt: "portugues", sv: "svenska",
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
const lui = langMap[locale.split("-")[0]] || "english";
|
|
134
|
+
const params = new URLSearchParams({
|
|
135
|
+
q: query, format: "opensearch", segment: "startpage.defaultffx", lui,
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
const data = await getSuggestions(`https://www.startpage.com/suggestions?${params}`, {
|
|
139
|
+
headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" },
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
if (data && Array.isArray(data) && data.length >= 2 && Array.isArray(data[1])) return data[1];
|
|
143
|
+
return [];
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Wikipedia autocomplete — uses the OpenSearch API on the locale-appropriate Wikipedia subdomain.
|
|
148
|
+
*/
|
|
149
|
+
export async function wikipedia(query: string, locale: string = "en"): Promise<string[]> {
|
|
150
|
+
const langMap: { [key: string]: string } = {
|
|
151
|
+
en: "en.wikipedia.org", de: "de.wikipedia.org", fr: "fr.wikipedia.org",
|
|
152
|
+
es: "es.wikipedia.org", it: "it.wikipedia.org", nl: "nl.wikipedia.org",
|
|
153
|
+
pt: "pt.wikipedia.org", ru: "ru.wikipedia.org", ja: "ja.wikipedia.org",
|
|
154
|
+
zh: "zh.wikipedia.org", ar: "ar.wikipedia.org", ko: "ko.wikipedia.org",
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
const netloc = langMap[locale.split("-")[0]] || "en.wikipedia.org";
|
|
158
|
+
const params = new URLSearchParams({
|
|
159
|
+
action: "opensearch", format: "json", formatversion: "2",
|
|
160
|
+
search: query, namespace: "0", limit: "10",
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
const data = await getSuggestions(`https://${netloc}/w/api.php?${params}`);
|
|
164
|
+
if (data && Array.isArray(data) && data.length > 1) return data[1];
|
|
165
|
+
return [];
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Yandex autocomplete — uses Yandex's `suggest-ff.cgi` endpoint.
|
|
170
|
+
*/
|
|
171
|
+
export async function yandex(query: string, _locale?: string): Promise<string[]> {
|
|
172
|
+
const params = new URLSearchParams({ part: query });
|
|
173
|
+
const data = await getSuggestions(
|
|
174
|
+
`https://suggest.yandex.com/suggest-ff.cgi?${params}`,
|
|
175
|
+
);
|
|
176
|
+
if (data && Array.isArray(data) && data.length > 1) return data[1];
|
|
177
|
+
return [];
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Map of all available autocomplete backend names to their functions. */
|
|
181
|
+
export const backends: { [key: string]: AutocompleteFunction } = {
|
|
182
|
+
baidu, brave, duckduckgo, google, qwant, startpage, wikipedia, yandex,
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Get autocomplete suggestions from a specific backend.
|
|
187
|
+
*
|
|
188
|
+
* @param backendName - Name of the backend (e.g. `"google"`, `"duckduckgo"`).
|
|
189
|
+
* @param query - Partial search query to complete.
|
|
190
|
+
* @param locale - BCP-47 locale code such as `"en-US"` or `"de-DE"`.
|
|
191
|
+
*/
|
|
192
|
+
export async function searchAutocomplete(
|
|
193
|
+
backendName: string,
|
|
194
|
+
query: string,
|
|
195
|
+
locale: string = "en-US",
|
|
196
|
+
): Promise<string[]> {
|
|
197
|
+
const backend = backends[backendName];
|
|
198
|
+
if (!backend) {
|
|
199
|
+
console.warn(`Autocomplete backend '${backendName}' not found`);
|
|
200
|
+
return [];
|
|
201
|
+
}
|
|
202
|
+
try {
|
|
203
|
+
return await backend(query, locale);
|
|
204
|
+
} catch (error) {
|
|
205
|
+
console.error(`Autocomplete error for ${backendName}:`, error);
|
|
206
|
+
return [];
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Fan out to multiple autocomplete backends in parallel and return a
|
|
212
|
+
* deduplicated union of all suggestions.
|
|
213
|
+
*
|
|
214
|
+
* @param backendNames - Array of backend names to query.
|
|
215
|
+
* @param query - Partial search query.
|
|
216
|
+
* @param locale - BCP-47 locale code.
|
|
217
|
+
*/
|
|
218
|
+
export async function searchAutocompleteMulti(
|
|
219
|
+
backendNames: string[],
|
|
220
|
+
query: string,
|
|
221
|
+
locale: string = "en-US",
|
|
222
|
+
): Promise<string[]> {
|
|
223
|
+
const results = await Promise.all(
|
|
224
|
+
backendNames.map((name) => searchAutocomplete(name, query, locale)),
|
|
225
|
+
);
|
|
226
|
+
|
|
227
|
+
const merged = new Set<string>();
|
|
228
|
+
for (const result of results) for (const suggestion of result) merged.add(suggestion);
|
|
229
|
+
|
|
230
|
+
return Array.from(merged);
|
|
231
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Category Registry — Re-exports the CategoryRegistry class, the CATEGORIES
|
|
3
|
+
* config map (with per-category scoring weights), and the global
|
|
4
|
+
* `categoryRegistry` singleton from the canonical registry module.
|
|
5
|
+
*/
|
|
6
|
+
export * from "./registry/search-engine-category-registry.js";
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Search Engine Constants — Static configuration used across the search API:
|
|
3
|
+
* the list of valid result categories, allowed recency filter values, a curated
|
|
4
|
+
* list of public SearXNG instance domains, and a per-category directory of
|
|
5
|
+
* engine names with short descriptions for documentation and UI rendering.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export const CATEGORY_LIST = [
|
|
9
|
+
"general",
|
|
10
|
+
"news",
|
|
11
|
+
"videos",
|
|
12
|
+
"images",
|
|
13
|
+
"science",
|
|
14
|
+
"it",
|
|
15
|
+
"files",
|
|
16
|
+
"social+media",
|
|
17
|
+
"map",
|
|
18
|
+
"music",
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
export const RECENCY_ALLOWED_LIST = ["day", "week", "month", "year"];
|
|
22
|
+
|
|
23
|
+
export const SEARX_DOMAINS = [
|
|
24
|
+
"baresearch.org",
|
|
25
|
+
"copp.gg",
|
|
26
|
+
"darmarit.org",
|
|
27
|
+
"etsi.me",
|
|
28
|
+
"fairsuch.net",
|
|
29
|
+
"nogoo.me",
|
|
30
|
+
"northboot.xyz",
|
|
31
|
+
"nyc1.sx.ggtyler.dev",
|
|
32
|
+
"ooglester.com",
|
|
33
|
+
"opnxng.com",
|
|
34
|
+
"paulgo.io",
|
|
35
|
+
"priv.au",
|
|
36
|
+
"s.trung.fun",
|
|
37
|
+
"search.blitzw.in",
|
|
38
|
+
"search.charliewhiskey.net",
|
|
39
|
+
"search.citw.lgbt",
|
|
40
|
+
"search.darkness.services",
|
|
41
|
+
"search.datura.network",
|
|
42
|
+
"search.dotone.nl",
|
|
43
|
+
"search.gcomm.ch",
|
|
44
|
+
"search.hbubli.cc",
|
|
45
|
+
"search.im-in.space",
|
|
46
|
+
"search.incogniweb.net",
|
|
47
|
+
"search.inetol.net",
|
|
48
|
+
"search.leptons.xyz",
|
|
49
|
+
"search.nadeko.net",
|
|
50
|
+
"search.ngn.tf",
|
|
51
|
+
"search.ononoki.org",
|
|
52
|
+
"search.privacyredirect.com",
|
|
53
|
+
"search.sapti.me",
|
|
54
|
+
"search.rowie.at",
|
|
55
|
+
"search.projectsegfau.lt",
|
|
56
|
+
"search.tommy-tran.com",
|
|
57
|
+
"searx.aleteoryx.me",
|
|
58
|
+
"searx.ankha.ac",
|
|
59
|
+
"searx.be",
|
|
60
|
+
"searx.colbster937.dev",
|
|
61
|
+
"searx.daetalytica.io",
|
|
62
|
+
"searx.dresden.network",
|
|
63
|
+
"searx.foss.family",
|
|
64
|
+
"searx.hu",
|
|
65
|
+
"searx.juancord.xyz",
|
|
66
|
+
"searx.lunar.icu",
|
|
67
|
+
"searx.mxchange.org",
|
|
68
|
+
"searx.namejeff.xyz",
|
|
69
|
+
"searx.oakleycord.dev",
|
|
70
|
+
"searx.ro",
|
|
71
|
+
"searx.sev.monster",
|
|
72
|
+
"searx.thefloatinglab.world",
|
|
73
|
+
"searx.tiekoetter.com",
|
|
74
|
+
"searx.tuxcloud.net",
|
|
75
|
+
"searx.work",
|
|
76
|
+
"searx.zhenyapav.com",
|
|
77
|
+
"searxng.hweeren.com",
|
|
78
|
+
"searxng.online",
|
|
79
|
+
"searxng.shreven.org",
|
|
80
|
+
"searxng.site",
|
|
81
|
+
"skyrimhater.com",
|
|
82
|
+
"sx.ca.zorby.top",
|
|
83
|
+
"sx.catgirl.cloud",
|
|
84
|
+
"sx.thatxtreme.dev",
|
|
85
|
+
"sx.zorby.top",
|
|
86
|
+
"xo.wtf",
|
|
87
|
+
];
|
|
88
|
+
|
|
89
|
+
/** Directory of engine names and short descriptions grouped by category. */
|
|
90
|
+
export const engines: Record<string, [string, string][]> = {
|
|
91
|
+
general: [
|
|
92
|
+
["google", "The world's most popular web search engine."],
|
|
93
|
+
["bing", "Microsoft's web search engine for the general web."],
|
|
94
|
+
["brave", "Privacy-focused web search integrated with the Brave browser."],
|
|
95
|
+
["duckduckgo", "Privacy-oriented search engine that does not track users."],
|
|
96
|
+
["qwant", "European privacy-focused web search engine."],
|
|
97
|
+
["startpage", "Search engine that proxies Google results with enhanced privacy."],
|
|
98
|
+
["yahoo", "Web portal providing search, email, and news."],
|
|
99
|
+
["ask", "Question-answering oriented web search engine."],
|
|
100
|
+
["wikipedia", "Free, community-edited online encyclopedia."],
|
|
101
|
+
["wikidata", "Structured, collaborative knowledge base from Wikimedia."],
|
|
102
|
+
["wolframalpha", "Computational knowledge engine for factual queries and calculations."],
|
|
103
|
+
],
|
|
104
|
+
news: [
|
|
105
|
+
["duckduckgo news", "Aggregated news search powered by DuckDuckGo."],
|
|
106
|
+
["mojeek news", "News search vertical from the independent Mojeek engine."],
|
|
107
|
+
["bing news", "News aggregation and search from Microsoft Bing."],
|
|
108
|
+
["brave.news", "Privacy-focused news search from Brave Search."],
|
|
109
|
+
["google news", "Google's personalized news aggregation and search service."],
|
|
110
|
+
["qwant news", "News search and headlines from Qwant."],
|
|
111
|
+
["yahoo news", "News portal and search from Yahoo."],
|
|
112
|
+
["tagesschau (DE)", "German-language news search for Tagesschau, a major public broadcaster."],
|
|
113
|
+
],
|
|
114
|
+
videos: [
|
|
115
|
+
["bing videos", "Video search vertical from Microsoft Bing."],
|
|
116
|
+
["brave.videos", "Video search vertical of Brave Search."],
|
|
117
|
+
["duckduckgo videos", "Video search results from DuckDuckGo."],
|
|
118
|
+
["google videos", "Video-focused search results from Google."],
|
|
119
|
+
["qwant videos", "Video search vertical from Qwant."],
|
|
120
|
+
["bilibili", "Chinese video sharing and streaming platform popular for animation and games."],
|
|
121
|
+
["dailymotion", "Global video-sharing and hosting platform."],
|
|
122
|
+
["invidious", "Alternative, privacy-friendly front-end for YouTube content."],
|
|
123
|
+
["odysee", "Decentralized, blockchain-based video hosting platform."],
|
|
124
|
+
["peertube", "Federated, open source video hosting network."],
|
|
125
|
+
["piped", "Privacy-respecting alternative front-end for YouTube."],
|
|
126
|
+
["rumble", "Video sharing platform focusing on independent creators."],
|
|
127
|
+
["vimeo", "Ad-free video hosting platform used by filmmakers and businesses."],
|
|
128
|
+
["youtube", "The largest global video sharing and streaming platform."],
|
|
129
|
+
],
|
|
130
|
+
images: [
|
|
131
|
+
["bing images", "Image search engine by Microsoft Bing."],
|
|
132
|
+
["brave.images", "Image search vertical of Brave Search."],
|
|
133
|
+
["duckduckgo images", "Image search results from DuckDuckGo."],
|
|
134
|
+
["google images", "Google's dedicated image search service."],
|
|
135
|
+
["qwant images", "Image search vertical from Qwant."],
|
|
136
|
+
["deviantart", "Online community and gallery for digital art and illustrations."],
|
|
137
|
+
["flickr", "Image and short video hosting platform for photographers."],
|
|
138
|
+
["imgur", "Image hosting and sharing site popular for memes and galleries."],
|
|
139
|
+
["pinterest", "Visual discovery and bookmarking platform centered on images."],
|
|
140
|
+
["unsplash", "Library of freely usable, high-resolution photos."],
|
|
141
|
+
["wallhaven", "Community-driven wallpaper and background image repository."],
|
|
142
|
+
["wikicommons.images", "Image search across media stored on Wikimedia Commons."],
|
|
143
|
+
],
|
|
144
|
+
science: [
|
|
145
|
+
["arxiv", "Repository of preprint research papers in physics, math, CS and more."],
|
|
146
|
+
["crossref", "Infrastructure service providing DOIs and metadata for scholarly content."],
|
|
147
|
+
["google scholar", "Google's search engine for scholarly literature and citations."],
|
|
148
|
+
["internetarchivescholar", "Internet Archive search for digitized scholarly works."],
|
|
149
|
+
["pubmed", "Database of biomedical and life sciences literature from NCBI."],
|
|
150
|
+
["semantic scholar", "AI-powered literature search and discovery tool for research papers."],
|
|
151
|
+
],
|
|
152
|
+
it: [
|
|
153
|
+
["alpine linux packages", "Search index for Alpine Linux software packages."],
|
|
154
|
+
["crates.io", "Official package registry for Rust crates."],
|
|
155
|
+
["docker hub", "Central repository of Docker container images."],
|
|
156
|
+
["hex", "Package manager and repository for the Erlang and Elixir ecosystem."],
|
|
157
|
+
["hoogle", "API search engine for Haskell libraries and functions."],
|
|
158
|
+
["lib.rs", "Alternative index and front-end for Rust crates."],
|
|
159
|
+
["metacpan", "Search engine for Perl modules hosted on CPAN."],
|
|
160
|
+
["npm", "Package registry and manager for JavaScript and Node.js."],
|
|
161
|
+
["packagist", "Primary package repository for PHP Composer."],
|
|
162
|
+
["pkg.go.dev", "Documentation and discovery service for Go modules."],
|
|
163
|
+
["pub.dev", "Package repository for Dart and Flutter packages."],
|
|
164
|
+
["pypi", "Python Package Index, the main repository for Python packages."],
|
|
165
|
+
["rubygems", "Package manager and repository for Ruby gems."],
|
|
166
|
+
["voidlinux", "Package search for the Void Linux distribution."],
|
|
167
|
+
["askubuntu", "Q&A site focused on Ubuntu usage and troubleshooting."],
|
|
168
|
+
["stackoverflow", "Question-and-answer site for programmers and software development."],
|
|
169
|
+
["superuser", "Q&A site for advanced computer users and system administration."],
|
|
170
|
+
["bitbucket", "Git-based source code hosting and collaboration platform by Atlassian."],
|
|
171
|
+
["codeberg", "Non-profit, open source Git hosting platform."],
|
|
172
|
+
["gitea.com", "Hosted instance of the lightweight Gitea Git service."],
|
|
173
|
+
["github", "Popular Git-based code hosting and collaboration platform."],
|
|
174
|
+
["gitlab", "DevOps platform offering Git hosting, CI/CD, and project management."],
|
|
175
|
+
["sourcehut", "Minimalist suite of tools for software development and mailing lists."],
|
|
176
|
+
["arch linux wiki", "Extensive documentation wiki for Arch Linux and related tools."],
|
|
177
|
+
["gentoo", "Documentation wiki for the Gentoo Linux distribution."],
|
|
178
|
+
["mdn", "Mozilla Developer Network documentation for web technologies."],
|
|
179
|
+
["searchcode code", "Search engine indexing source code across public repositories."],
|
|
180
|
+
],
|
|
181
|
+
files: [
|
|
182
|
+
["library of congress", "Catalog and digital collections from the U.S. Library of Congress."],
|
|
183
|
+
["wikibooks", "Wikimedia project hosting free textbooks and manuals."],
|
|
184
|
+
["wikisource", "Wikimedia digital library of source texts and documents."],
|
|
185
|
+
["wikiversity", "Wikimedia project for educational resources and courses."],
|
|
186
|
+
["wikivoyage", "Free, community-created travel guide from Wikimedia."],
|
|
187
|
+
["google play movies", "Google's store for purchasing and renting movies and TV series."],
|
|
188
|
+
["mediathekviewweb (DE)", "Web interface to German public TV media libraries."],
|
|
189
|
+
["ina (FR)", "French National Audiovisual Institute archive search."],
|
|
190
|
+
["wikicommons.videos", "Search for video media stored on Wikimedia Commons."],
|
|
191
|
+
["wikicommons.audio", "Search for audio files stored on Wikimedia Commons."],
|
|
192
|
+
["apk mirror", "Repository for Android APK files outside official app stores."],
|
|
193
|
+
["apple app store", "Official iOS and iPadOS app distribution platform."],
|
|
194
|
+
["fdroid", "Catalog of free and open source Android applications."],
|
|
195
|
+
["google play apps", "Official Android app store operated by Google."],
|
|
196
|
+
],
|
|
197
|
+
"social+media": [
|
|
198
|
+
["pinterest", "Visual discovery, bookmarking, and inspiration social platform."],
|
|
199
|
+
["reddit", "Social news aggregation and discussion website organized by communities."],
|
|
200
|
+
["9gag", "Humor-focused social media site for sharing memes and short posts."],
|
|
201
|
+
["lemmy posts", "Post search across Lemmy, a federated link-aggregation platform."],
|
|
202
|
+
],
|
|
203
|
+
map: [
|
|
204
|
+
["apple maps", "Apple's mapping and navigation service for its devices."],
|
|
205
|
+
["openstreetmap", "Collaborative, open data world map project."],
|
|
206
|
+
],
|
|
207
|
+
music: [
|
|
208
|
+
["genius", "Platform for song lyrics, annotations, and music commentary."],
|
|
209
|
+
["radio browser", "Directory and search engine for online radio stations."],
|
|
210
|
+
["bandcamp", "Music platform for independent artists to sell and stream releases."],
|
|
211
|
+
["deezer", "On-demand music streaming service."],
|
|
212
|
+
["mixcloud", "Streaming platform for DJ mixes, radio shows, and podcasts."],
|
|
213
|
+
["soundcloud", "Audio distribution and music sharing platform for creators."],
|
|
214
|
+
["youtube", "Video platform heavily used for streaming music and music videos."],
|
|
215
|
+
],
|
|
216
|
+
};
|
package/src/constants.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Constants — Re-exports all static search API constants from the canonical
|
|
3
|
+
* config module: CATEGORY_LIST, RECENCY_ALLOWED_LIST, SEARX_DOMAINS, and the
|
|
4
|
+
* per-category engine directory used for documentation and UI rendering.
|
|
5
|
+
*/
|
|
6
|
+
export * from "./config/search-engine-constants.js";
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Engine Descriptions — Re-exports the `engineDescriptions` map and helper
|
|
3
|
+
* functions (`getEngineDescription`, `getEnginesWithDescriptions`) from the
|
|
4
|
+
* canonical registry module. Descriptions are sourced from SearXNG's
|
|
5
|
+
* engine_descriptions.json and cover all 75+ supported engines.
|
|
6
|
+
*/
|
|
7
|
+
export * from "./registry/search-engine-descriptions.js";
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Engine Status — Re-exports the EngineStatusTracker class and the global
|
|
3
|
+
* `engineStatusTracker` singleton from the canonical registry module.
|
|
4
|
+
* The tracker records per-engine success/failure counts, response times,
|
|
5
|
+
* and health status, and gates which engines are eligible to run.
|
|
6
|
+
*/
|
|
7
|
+
export * from "./registry/search-engine-status-tracker.js";
|
package/src/engine.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Engine — Re-exports the core engine types (EngineResult, Engine, EngineFunction)
|
|
3
|
+
* and the `extractResponseData` helper from the canonical types module. All
|
|
4
|
+
* engine source adapters import from `types/search-engine-interface` directly.
|
|
5
|
+
*/
|
|
6
|
+
export * from "./types/search-engine-interface.js";
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Search Engine Category Registry — Manages the mapping of search engines to
|
|
3
|
+
* categories (general, images, videos, news, maps, academic, etc.) and stores
|
|
4
|
+
* per-category default scoring weights. Based on SearXNG's engine categorisation
|
|
5
|
+
* system, it supports multi-category searches and weighted result combination.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { Engine } from "../types/search-engine-interface.js";
|
|
9
|
+
import { CategoryConfig } from "../types/search-result-types.js";
|
|
10
|
+
|
|
11
|
+
export const CATEGORIES: { [key: string]: CategoryConfig } = {
|
|
12
|
+
general: {
|
|
13
|
+
name: "general",
|
|
14
|
+
displayName: "General",
|
|
15
|
+
description: "General web search",
|
|
16
|
+
defaultWeight: 1.0,
|
|
17
|
+
},
|
|
18
|
+
images: {
|
|
19
|
+
name: "images",
|
|
20
|
+
displayName: "Images",
|
|
21
|
+
description: "Image search",
|
|
22
|
+
defaultWeight: 1.0,
|
|
23
|
+
},
|
|
24
|
+
videos: {
|
|
25
|
+
name: "videos",
|
|
26
|
+
displayName: "Videos",
|
|
27
|
+
description: "Video search",
|
|
28
|
+
defaultWeight: 1.0,
|
|
29
|
+
},
|
|
30
|
+
news: {
|
|
31
|
+
name: "news",
|
|
32
|
+
displayName: "News",
|
|
33
|
+
description: "News search",
|
|
34
|
+
defaultWeight: 1.1,
|
|
35
|
+
},
|
|
36
|
+
maps: {
|
|
37
|
+
name: "maps",
|
|
38
|
+
displayName: "Maps",
|
|
39
|
+
description: "Maps and locations",
|
|
40
|
+
defaultWeight: 1.0,
|
|
41
|
+
},
|
|
42
|
+
music: {
|
|
43
|
+
name: "music",
|
|
44
|
+
displayName: "Music",
|
|
45
|
+
description: "Music search",
|
|
46
|
+
defaultWeight: 1.0,
|
|
47
|
+
},
|
|
48
|
+
it: {
|
|
49
|
+
name: "it",
|
|
50
|
+
displayName: "IT",
|
|
51
|
+
description: "Programming and IT resources",
|
|
52
|
+
defaultWeight: 1.2,
|
|
53
|
+
},
|
|
54
|
+
academic: {
|
|
55
|
+
name: "academic",
|
|
56
|
+
displayName: "Academic",
|
|
57
|
+
description: "Scientific papers and academic resources",
|
|
58
|
+
defaultWeight: 1.3,
|
|
59
|
+
},
|
|
60
|
+
social: {
|
|
61
|
+
name: "social",
|
|
62
|
+
displayName: "Social Media",
|
|
63
|
+
description: "Social media content",
|
|
64
|
+
defaultWeight: 0.9,
|
|
65
|
+
},
|
|
66
|
+
files: {
|
|
67
|
+
name: "files",
|
|
68
|
+
displayName: "Files",
|
|
69
|
+
description: "File search",
|
|
70
|
+
defaultWeight: 1.0,
|
|
71
|
+
},
|
|
72
|
+
torrents: {
|
|
73
|
+
name: "torrents",
|
|
74
|
+
displayName: "Torrents",
|
|
75
|
+
description: "Torrent search",
|
|
76
|
+
defaultWeight: 0.8,
|
|
77
|
+
},
|
|
78
|
+
shopping: {
|
|
79
|
+
name: "shopping",
|
|
80
|
+
displayName: "Shopping",
|
|
81
|
+
description: "Shopping and products",
|
|
82
|
+
defaultWeight: 1.0,
|
|
83
|
+
},
|
|
84
|
+
specialized: {
|
|
85
|
+
name: "specialized",
|
|
86
|
+
displayName: "Specialized",
|
|
87
|
+
description: "Specialized search engines",
|
|
88
|
+
defaultWeight: 1.1,
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* CategoryRegistry manages engines organised by category.
|
|
94
|
+
* Supports multi-category search with proper result combination.
|
|
95
|
+
*/
|
|
96
|
+
export class CategoryRegistry {
|
|
97
|
+
private enginesByCategory: Map<string, Engine[]> = new Map();
|
|
98
|
+
private allEngines: Engine[] = [];
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Register a single engine in the registry.
|
|
102
|
+
*/
|
|
103
|
+
registerEngine(engine: Engine): void {
|
|
104
|
+
if (!this.allEngines.find((e) => e.name === engine.name)) {
|
|
105
|
+
this.allEngines.push(engine);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const categories = engine.categories || ["general"];
|
|
109
|
+
for (const category of categories) {
|
|
110
|
+
if (!this.enginesByCategory.has(category)) {
|
|
111
|
+
this.enginesByCategory.set(category, []);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const engines = this.enginesByCategory.get(category)!;
|
|
115
|
+
if (!engines.find((e) => e.name === engine.name)) {
|
|
116
|
+
engines.push(engine);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Register multiple engines at once.
|
|
123
|
+
*/
|
|
124
|
+
registerEngines(engines: Engine[]): void {
|
|
125
|
+
for (const engine of engines) {
|
|
126
|
+
this.registerEngine(engine);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Get all engines that belong to any of the given categories.
|
|
132
|
+
*/
|
|
133
|
+
getEnginesByCategories(categories: string[]): Engine[] {
|
|
134
|
+
const engines = new Set<Engine>();
|
|
135
|
+
|
|
136
|
+
for (const category of categories) {
|
|
137
|
+
const categoryEngines = this.enginesByCategory.get(category) || [];
|
|
138
|
+
for (const engine of categoryEngines) {
|
|
139
|
+
engines.add(engine);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
return Array.from(engines);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Get engines by their names.
|
|
148
|
+
*/
|
|
149
|
+
getEnginesByNames(names: string[]): Engine[] {
|
|
150
|
+
return this.allEngines.filter((e) => names.includes(e.name));
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Get all available category names.
|
|
155
|
+
*/
|
|
156
|
+
getCategories(): string[] {
|
|
157
|
+
return Array.from(this.enginesByCategory.keys());
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Get all registered engines.
|
|
162
|
+
*/
|
|
163
|
+
getAllEngines(): Engine[] {
|
|
164
|
+
return this.allEngines;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Get engines for a single category.
|
|
169
|
+
*/
|
|
170
|
+
getCategoryEngines(category: string): Engine[] {
|
|
171
|
+
return this.enginesByCategory.get(category) || [];
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Get the configuration object for a category.
|
|
176
|
+
*/
|
|
177
|
+
getCategoryConfig(category: string): CategoryConfig | undefined {
|
|
178
|
+
return CATEGORIES[category];
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Get a count of registered engines per category plus totals.
|
|
183
|
+
*/
|
|
184
|
+
getStats() {
|
|
185
|
+
const stats: { [category: string]: number } = {};
|
|
186
|
+
|
|
187
|
+
for (const [category, engines] of this.enginesByCategory.entries()) {
|
|
188
|
+
stats[category] = engines.length;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
totalEngines: this.allEngines.length,
|
|
193
|
+
categories: this.getCategories().length,
|
|
194
|
+
enginesByCategory: stats,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** Global singleton instance used throughout the application. */
|
|
200
|
+
export const categoryRegistry = new CategoryRegistry();
|