search-web-api 1.0.13

package/docs/AUTOCOMPLETE.md

@@ -0,0 +1,296 @@
+# Autocomplete API Documentation
+
+The autocomplete module provides search query suggestions from multiple search engines, helping users quickly find what they're looking for.
+
+## Features
+
+- **8 Search Engine Backends**: Google, DuckDuckGo, Brave, Wikipedia, Baidu, Qwant, Startpage, Yandex
+- **Locale Support**: Most backends support different languages and regions
+- **Multi-Backend Queries**: Combine suggestions from multiple engines
+- **Fast Responses**: Optimized with 3-second timeout for quick results
+
+## API Endpoints
+
+### GET /autocomplete
+
+Get autocomplete suggestions from a specific backend.
+
+**Query Parameters:**
+
+- `q` (required): Search query string
+- `backend` (optional): Backend name (default: 'google')
+- `locale` (optional): Language/locale code (default: 'en-US')
+
+**Example:**
+
+```bash
+curl "http://localhost:3000/autocomplete?q=typescript&backend=google&locale=en-US"
+```
+
+**Response:**
+
+```json
+{
+  "query": "typescript",
+  "backend": "google",
+  "locale": "en-US",
+  "suggestions": [
+    "typescript tutorial",
+    "typescript vs javascript",
+    "typescript generics",
+    "typescript interface",
+    "typescript types"
+  ]
+}
+```
+
+### GET /autocomplete/multi
+
+Get autocomplete suggestions from multiple backends and merge them.
+
+**Query Parameters:**
+
+- `q` (required): Search query string
+- `backends` (optional): Comma-separated backend names (default: 'google,duckduckgo')
+- `locale` (optional): Language/locale code (default: 'en-US')
+
+**Example:**
+
+```bash
+curl "http://localhost:3000/autocomplete/multi?q=python&backends=google,duckduckgo,wikipedia"
+```
+
+**Response:**
+
+```json
+{
+  "query": "python",
+  "backends": ["google", "duckduckgo", "wikipedia"],
+  "locale": "en-US",
+  "count": 12,
+  "suggestions": [
+    "python tutorial",
+    "python download",
+    "python programming",
+    "python (programming language)",
+    "python snake",
+    ...
+  ]
+}
+```
+
+### GET /autocomplete/backends
+
+List all available autocomplete backends.
+
+**Example:**
+
+```bash
+curl "http://localhost:3000/autocomplete/backends"
+```
+
+**Response:**
+
+```json
+{
+  "backends": [
+    {
+      "name": "baidu",
+      "description": "Baidu (Chinese search engine)"
+    },
+    {
+      "name": "brave",
+      "description": "Brave Search"
+    },
+    ...
+  ],
+  "count": 8
+}
+```
+
+## Available Backends
+
+| Backend              | Description   | Locale Support   |
+| -------------------- | ------------- | ---------------- |
+| **google**     | Google Search | Yes (subdomains) |
+| **duckduckgo** | DuckDuckGo    | Yes (regions)    |
+| **brave**      | Brave Search  | Limited          |
+| **wikipedia**  | Wikipedia     | Yes (languages)  |
+| **baidu**      | Baidu         | Chinese          |
+| **qwant**      | Qwant         | Yes (locales)    |
+| **startpage**  | Startpage     | Yes (languages)  |
+| **yandex**     | Yandex        | Russian          |
+
+## Programmatic Usage
+
+### TypeScript/JavaScript
+
+```typescript
+import { searchAutocomplete, searchAutocompleteMulti } from './app/lib/autocomplete';
+
+// Single backend
+const suggestions = await searchAutocomplete('google', 'typescript');
+console.log(suggestions);
+
+// Multiple backends
+const merged = await searchAutocompleteMulti(
+    ['google', 'duckduckgo', 'wikipedia'],
+    'javascript',
+    'en-US'
+);
+console.log(merged);
+```
+
+### Individual Backend Functions
+
+```typescript
+import { google, duckduckgo, wikipedia } from './app/lib/autocomplete';
+
+// Google autocomplete
+const googleSuggestions = await google('react', 'en-US');
+
+// DuckDuckGo autocomplete
+const ddgSuggestions = await duckduckgo('vue');
+
+// Wikipedia autocomplete (German)
+const wikiSuggestions = await wikipedia('quantum physics', 'de');
+```
+
+## Locale Support
+
+### Google
+
+Google uses subdomains based on language:
+
+```typescript
+// English (google.com)
+await google('search query', 'en');
+
+// German (google.de)
+await google('Suchanfrage', 'de');
+
+// French (google.fr)
+await google('requête de recherche', 'fr');
+
+// Japanese (google.co.jp)
+await google('検索クエリ', 'ja');
+```
+
+### Wikipedia
+
+Wikipedia uses language-specific domains:
+
+```typescript
+// English Wikipedia
+await wikipedia('physics', 'en');
+
+// German Wikipedia
+await wikipedia('Physik', 'de');
+
+// French Wikipedia
+await wikipedia('physique', 'fr');
+```
+
+### DuckDuckGo
+
+DuckDuckGo uses region codes:
+
+```typescript
+// US English
+await duckduckgo('search', 'en-US');
+
+// UK English
+await duckduckgo('search', 'en-GB');
+
+// German
+await duckduckgo('Suche', 'de-DE');
+```
+
+## Implementation Details
+
+### Response Timeout
+
+All autocomplete requests have a 3-second timeout to ensure fast responses. If a backend doesn't respond within this time, an empty array is returned.
+
+### Error Handling
+
+Errors are caught and logged, but don't prevent the API from responding:
+
+```typescript
+try {
+    return await backend(query, locale);
+} catch (error) {
+    console.error('Autocomplete error:', error);
+    return [];
+}
+```
+
+### Deduplication
+
+When using multi-backend queries, results are automatically deduplicated:
+
+```typescript
+const merged = new Set<string>();
+for (const result of results) {
+    for (const suggestion of result) {
+        merged.add(suggestion);
+    }
+}
+return Array.from(merged);
+```
+
+## Examples
+
+See `examples/autocomplete-example.ts` for comprehensive usage examples including:
+
+1. Single backend autocomplete
+2. Multi-backend queries
+3. Locale-specific suggestions
+4. Direct backend function calls
+5. Comparing results across backends
+6. Asian language autocomplete (Baidu, Yandex)
+
+Run the examples:
+
+```bash
+bun run examples/autocomplete-example.ts
+```
+
+## Performance Considerations
+
+1. **Parallel Requests**: Multi-backend queries run in parallel using `Promise.all()`
+2. **Fast Timeout**: 3-second timeout prevents slow backends from blocking
+3. **Lightweight Parsing**: Minimal HTML parsing using linkedom for Google
+4. **No Caching**: Suggestions are fresh on each request (can be added if needed)
+
+## Future Enhancements
+
+Potential improvements:
+
+- [ ] Add caching layer for frequent queries
+- [ ] Support for more backends (Bing, Yahoo, etc.)
+- [ ] Ranked suggestions based on popularity
+- [ ] User query history integration
+- [ ] Personalized suggestions
+- [ ] A/B testing different backends
+
+## Troubleshooting
+
+### No suggestions returned
+
+1. Check if the backend is supported: `GET /autocomplete/backends`
+2. Verify the query is not empty
+3. Check network connectivity
+4. Try a different backend
+
+### Slow responses
+
+1. Use single backend instead of multi-backend
+2. Check backend-specific issues (rate limiting, etc.)
+3. Consider implementing caching
+
+### Locale not working
+
+1. Verify locale format (e.g., 'en-US', 'de-DE')
+2. Check if backend supports the locale
+3. Refer to backend-specific locale documentation above

package/docs/CATEGORY_SEARCH.md

@@ -0,0 +1,265 @@
+# Category-Based Search with Weighted Ranking
+
+This document explains how the TypeScript search API implements category-based search with weighted ranking, based on the Python SearXNG implementation.
+
+## Overview
+
+The search API now supports:
+
+- **Multi-category search**: Search across specific categories (general, news, academic, etc.)
+- **Result combination**: Automatically combine and deduplicate results from multiple engines
+- **Weighted ranking**: Score results based on engine quality, category importance, and position
+- **Category grouping**: Results are grouped by category for better organization
+
+## Architecture
+
+### Key Components
+
+1. **CategoryRegistry** (`app/lib/category-registry.ts`)
+
+   - Manages engines organized by category
+   - Supports 13 categories: general, images, videos, news, maps, music, it, academic, social, files, torrents, shopping, specialized
+   - Each category has configurable default weights
+
+2. **ResultContainer** (`app/lib/result-container.ts`)
+
+   - Deduplicates results by URL hash
+   - Merges duplicate results from different engines
+   - Calculates weighted scores
+   - Groups results by category
+
+3. **Search Class** (`app/lib/search.ts`)
+   - Main search interface
+   - Queries multiple engines in parallel
+   - Applies category and engine filters
+   - Returns combined, scored, and sorted results
+
+## How Scoring Works
+
+The scoring algorithm is based on SearXNG's Python implementation:
+
+```
+score = Σ (weight / position) for each position
+```
+
+Where:
+
+- **weight** = `base_weight × engine_weight × category_weight × num_occurrences`
+- **position**: The position of the result in the engine's results (1st, 2nd, 3rd, etc.)
+- **num_occurrences**: How many engines found this result
+
+### Engine Weights
+
+Higher quality engines get higher weights:
+
+- Google: 1.5
+- Bing: 1.3
+- DuckDuckGo: 1.2
+- Google Scholar: 1.4
+- Semantic Scholar: 1.3
+- Default: 1.0
+
+### Category Weights
+
+Different categories can have different importance:
+
+- Academic: 1.3 (higher credibility)
+- IT: 1.2 (technical accuracy)
+- News: 1.1
+- Social: 0.9 (lower due to potential duplicates)
+- Torrents: 0.8 (lower due to potential spam)
+- Default: 1.0
+
+### Example Score Calculation
+
+If a result appears at:
+
+- Position 1 in Google (weight 1.5)
+- Position 3 in Bing (weight 1.3)
+- Category: general (weight 1.0)
+
+```
+score = (1.5 × 1.0 × 2 / 1) + (1.3 × 1.0 × 2 / 3)
+      = 3.0 + 0.87
+      = 3.87
+```
+
+The multiplier of 2 is because the result appeared in 2 engines.
+
+## Usage Examples
+
+### 1. Search a Single Category
+
+```typescript
+import { Search } from "./app/lib/search.js";
+
+const search = new Search();
+
+// Search only general web engines
+const results = await search.searchByCategories("typescript", ["general"]);
+```
+
+### 2. Search Multiple Categories
+
+```typescript
+// Search across general, IT, and academic categories
+const results = await search.searchByCategories("machine learning", [
+  "general",
+  "it",
+  "academic",
+]);
+```
+
+### 3. Use the Standard Search with Category Filter
+
+```typescript
+// More explicit control
+const results = await search.search(
+  "climate change",
+  1, // page number
+  undefined, // engine names
+  ["news", "general"] // categories
+);
+```
+
+### 4. Search Specific Engines
+
+```typescript
+// Search only specific engines
+const results = await search.search(
+  "nodejs",
+  1,
+  ["google", "duckduckgo", "github"] // specific engines
+);
+```
+
+## Available Categories
+
+| Category    | Description         | Default Weight | Example Engines                    |
+| ----------- | ------------------- | -------------- | ---------------------------------- |
+| general     | General web search  | 1.0            | Google, Bing, DuckDuckGo           |
+| images      | Image search        | 1.0            | Unsplash, Bing Images, Flickr      |
+| videos      | Video search        | 1.0            | YouTube, Vimeo, Dailymotion        |
+| news        | News search         | 1.1            | Google News, HackerNews, Bing News |
+| maps        | Maps and locations  | 1.0            | OpenStreetMap, Photon              |
+| it          | Programming/IT      | 1.2            | GitHub, StackOverflow, NPM         |
+| academic    | Scientific papers   | 1.3            | Google Scholar, arXiv, PubMed      |
+| social      | Social media        | 0.9            | Reddit, Twitter, Mastodon          |
+| torrents    | Torrent search      | 0.8            | 1337x, ThePirateBay, YTS           |
+| shopping    | Shopping            | 1.0            | eBay                               |
+| specialized | Specialized engines | 1.1            | Wikipedia, IMDB, Genius            |
+
+## Result Deduplication
+
+Results with the same URL (normalized) are automatically merged:
+
+1. **URL normalization**: Removes protocol, www, trailing slashes
+2. **Title matching**: Includes partial title match for disambiguation
+3. **Result merging**:
+   - Combines engine names
+   - Tracks all positions
+   - Uses longer content/title
+   - Prefers HTTPS URLs
+   - Accumulates scores from all engines
+
+Example:
+
+```
+Result from Google at position 1
++ Same URL from Bing at position 2
+= Merged result with:
+  - engines: ['google', 'bing']
+  - positions: [1, 2]
+  - Higher combined score
+```
+
+## Category Grouping
+
+Results are grouped by category in the final output:
+
+1. **First pass**: Sort all results by score (descending)
+2. **Second pass**: Group similar categories together
+   - Maximum 8 results per group
+   - Maximum distance of 20 positions to look back
+   - Groups by: category + template + has_image
+
+This ensures related results appear together while maintaining score-based relevance.
+
+## API Reference
+
+### Search Class Methods
+
+#### `search(query, pageno, engineNames?, categories?)`
+
+Main search method with full control.
+
+**Parameters:**
+
+- `query`: Search query string
+- `pageno`: Page number (default: 1)
+- `engineNames`: Optional array of specific engine names
+- `categories`: Optional array of category names
+
+**Returns:** `Promise<MergedResult[]>`
+
+#### `searchByCategories(query, categories, pageno?)`
+
+Convenience method for multi-category search.
+
+**Parameters:**
+
+- `query`: Search query string
+- `categories`: Array of category names
+- `pageno`: Page number (default: 1)
+
+**Returns:** `Promise<MergedResult[]>`
+
+#### `getCategories()`
+
+Get list of available categories.
+
+**Returns:** `string[]`
+
+#### `getEnginesByCategory(category)`
+
+Get engines for a specific category.
+
+**Parameters:**
+
+- `category`: Category name
+
+**Returns:** `Engine[]`
+
+#### `getCategoryStats()`
+
+Get statistics about engines and categories.
+
+**Returns:** Object with category statistics
+
+### MergedResult Interface
+
+```typescript
+interface MergedResult {
+  url?: string;
+  title: string;
+  content: string;
+  engines: string[]; // List of engines that found this result
+  positions: number[]; // Positions in each engine's results
+  score: number; // Calculated weighted score
+  priority: "low" | "normal" | "high";
+  category?: string; // Primary category
+  template?: string; // Result template type
+  thumbnail?: string;
+  publishedDate?: string;
+  author?: string;
+  // ... other fields
+}
+```
+
+## Performance Considerations
+
+1. **Parallel execution**: All engines are queried in parallel using `Promise.all()`
+2. **Health tracking**: Unhealthy engines are automatically skipped
+3. **Timeout handling**: Individual engine timeouts don't block the overall search
+4. **Deduplication**: Hash-based deduplication is O(1) using Map
+5. **Sorting**: Results are sorted once using efficient native sort

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "search-web-api",
+  "description": "70+ search engines across 10 categories and Scrape Extract API",
+  "type": "module",
+  "version": "1.0.13",
+  "author": "vtempest",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/OpenSourceAGI/qwksearch-research-agent.git",
+    "directory": "packages/search-web-api"
+  },
+  "scripts": {
+    "dev": "bun --watch demo/index.ts",
+    "start": "bun demo/index.ts",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^3.8.1",
+    "@scalar/hono-api-reference": "^0.9.26",
+    "grab-url": "^1.0.8",
+    "hono": "^4.6.12",
+    "linkedom": "^0.18.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.9.3",
+    "@vitest/coverage-v8": "^4.0.18",
+    "tsx": "^4.7.1",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.18"
+  }
+}

package/src/autocomplete/autocomplete-ai-next-word-predictor.ts

@@ -0,0 +1,47 @@
+/**
+ * Autocomplete AI Next-Word Predictor — Local, CPU-based next-word prediction
+ * using the Xenova/distilgpt2 model via the Hugging Face Transformers.js library.
+ * DistilGPT-2 is a distilled 82 M-parameter variant of GPT-2 (~80 MB) that
+ * runs efficiently on CPU with FP16 precision and is well-suited for lightweight
+ * inline query completion without any network calls.
+ *
+ * @see https://huggingface.co/Xenova/distilgpt2
+ */
+
+import { pipeline, type TextGenerationSingle } from "@huggingface/transformers";
+
+/**
+ * Predict the next words for a given text prompt using a small local language model.
+ *
+ * @param prompt - The input text to continue.
+ * @param options.maxTokens - Maximum number of new tokens to generate (default 16).
+ * @param options.model - Hugging Face model ID to use (default `"Xenova/distilgpt2"`).
+ * @param options.modelParams - Additional pipeline constructor options.
+ * @returns The generated continuation text with the original prompt removed.
+ */
+export async function predictNextWordsWithSmallLocalModel(
+  prompt: string,
+  {
+    model = "Xenova/distilgpt2",
+    maxTokens = 16,
+    modelParams = {},
+  }: {
+    model?: string;
+    maxTokens?: number;
+    modelParams?: Record<string, unknown>;
+  } = {},
+): Promise<string> {
+  if (!prompt) throw new Error("Prompt is required");
+
+  const generator = await pipeline("text-generation", model, {
+    dtype: "fp16",
+    device: "cpu",
+    ...modelParams,
+  });
+
+  return (await generator(prompt, { max_new_tokens: maxTokens }))
+    .map((res) => (res as TextGenerationSingle).generated_text)
+    .join(" ")
+    .replace(prompt, "")
+    .trim();
+}