ubersearch 0.0.0-development

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Brian Sunter
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,374 @@
+# Multi-Search
+
+Unified, Bun-first search interface across multiple providers with credit tracking, pluggable strategies, and optional Docker-managed SearXNG.
+
+## Highlights
+
+- 🔍 Providers: Tavily, Brave, Linkup, SearXNG (local, Docker auto-start)
+- 🔌 Extensible: Add custom providers via TypeScript plugin system
+- 🤝 Single interface: shared types + CLI + programmatic API
+- 💳 Credits: per-engine quotas with snapshots and low-credit warnings
+- 🧠 Strategies: `all` (merge) or `first-success` (fastest win)
+- ⚙️ Config: JSON or TypeScript (`defineConfig`), XDG-aware resolution
+- 🐳 Auto-start: optional Docker lifecycle for local SearXNG
+
+## Install & Run (Bun)
+
+```bash
+cd /path/to/allsearch
+bun install
+
+# CLI (direct)
+bun run src/cli.ts "best TypeScript ORM 2025"
+
+# Or use bun link (works from any directory)
+bun link
+allsearch "llm observability" --json
+```
+
+## Usage
+
+### Basic Search
+
+```bash
+allsearch "your search query"
+```
+
+### Options
+
+```bash
+allsearch "query" [options]
+
+Options:
+  --json                        Output results as JSON
+  --engines engine1,engine2     Use specific engines
+  --strategy all|first-success  Search strategy (default: all)
+  --limit number                Max results per engine
+  --include-raw                 Include raw provider responses
+  --help, -h                    Show help
+  health                        Run provider health checks (starts Docker-backed ones if needed)
+```
+
+### Examples
+
+```bash
+# Search with specific engines
+allsearch "hawaii dev meetups" --engines tavily,brave --json
+
+# Use first-success strategy (stop after first working provider)
+allsearch "emerging web frameworks" --strategy first-success
+
+# Limit results per provider
+allsearch "rust async patterns" --limit 3
+
+# Check credit status
+allsearch credits
+```
+
+## Configuration
+
+Resolution order (first wins):
+1. Explicit path passed to CLI/API (`--config /path/to/config.json`)
+2. `./allsearch.config.(ts|json)` (current directory)
+3. `$XDG_CONFIG_HOME/allsearch/config.(ts|json)` (default: `~/.config/allsearch/`)
+
+### XDG Directory Structure
+
+```
+~/.config/allsearch/
+├── config.json              # Main configuration
+└── searxng/
+    └── config/
+        └── settings.yml     # SearXNG settings (auto-copied on first run)
+
+~/.local/share/allsearch/
+└── searxng/
+    └── data/                # SearXNG cache (auto-created)
+```
+
+- Example config: see `docs/config/allsearch.config.json`
+- Schema: `docs/config/config.schema.json` (generated from Zod)
+- TS helper: `defineConfig`, `defineTavily`, `defineBrave`, `defineLinkup`, `defineSearchxng`
+
+### SearXNG Configuration
+
+SearXNG uses Docker with volumes mounted to XDG directories. On first run, the default `settings.yml` is copied to `~/.config/allsearch/searxng/config/`. You can customize this file to:
+- Enable/disable search engines
+- Adjust rate limiting
+- Configure output formats
+
+## Custom Providers
+
+Add your own search providers using the plugin system.
+
+### TypeScript Config with Custom Provider
+
+```typescript
+// ~/.config/allsearch/config.ts
+import { defineConfig, definePlugin } from "allsearch/config";
+
+// 1. Define your provider class
+class PerplexityProvider {
+  constructor(private config: any) {}
+
+  get id() { return this.config.id; }
+
+  async search(query: { query: string; limit?: number }) {
+    const response = await fetch("https://api.perplexity.ai/search", {
+      method: "POST",
+      headers: {
+        "Authorization": `Bearer ${process.env[this.config.apiKeyEnv]}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({ query: query.query }),
+    });
+    const data = await response.json();
+
+    return {
+      engineId: this.id,
+      items: data.results.map((r: any) => ({
+        title: r.title,
+        url: r.url,
+        snippet: r.snippet,
+        sourceEngine: this.id,
+      })),
+    };
+  }
+}
+
+// 2. Create the plugin
+const perplexityPlugin = definePlugin({
+  type: "perplexity",
+  displayName: "Perplexity AI",
+  hasLifecycle: false,
+  factory: (config) => new PerplexityProvider(config),
+});
+
+// 3. Export config with plugin
+export default defineConfig({
+  plugins: [perplexityPlugin],
+  defaultEngineOrder: ["perplexity", "searxng"],
+  engines: [
+    {
+      id: "perplexity",
+      type: "perplexity",  // matches plugin type
+      enabled: true,
+      apiKeyEnv: "PERPLEXITY_API_KEY",
+      endpoint: "https://api.perplexity.ai/search",
+      monthlyQuota: 1000,
+      creditCostPerSearch: 1,
+      lowCreditThresholdPercent: 10,
+    },
+    // ... other engines
+  ],
+});
+```
+
+### Provider Interface
+
+Your provider must implement:
+
+```typescript
+interface ISearchProvider {
+  id: string;
+  search(query: SearchQuery): Promise<SearchResponse>;
+}
+
+interface SearchQuery {
+  query: string;
+  limit?: number;
+  includeRaw?: boolean;
+}
+
+interface SearchResponse {
+  engineId: string;
+  items: SearchResultItem[];
+  raw?: unknown;
+  tookMs?: number;
+}
+
+interface SearchResultItem {
+  title: string;
+  url: string;
+  snippet: string;
+  score?: number;
+  sourceEngine: string;
+}
+```
+
+### Plugin Helpers
+
+- `definePlugin({ type, displayName, hasLifecycle, factory })` - Create a plugin
+- `defineConfig({ plugins, engines, ... })` - Config with plugins
+- `defineEngine<T>(config)` - Type-safe custom engine config
+
+## Architecture (short)
+
+- Config resolved and validated (`src/config`), plugins registered
+- DI container bootstraps orchestrator, credit manager, provider registry (`src/bootstrap/container.ts`)
+- Providers registered via plugins (`src/plugin`, `src/providers/*`)
+- Orchestrator runs strategies (`src/core/strategy/*`) and aggregates results
+- Docker-backed providers (SearXNG) use lifecycle manager with auto-start/health checks (`src/core/docker/*`)
+
+## Output Formats
+
+### Human-Readable (Default)
+
+```
+Query: "rust async patterns"
+Found 15 results
+
+============================================================
+tavily (10 results)
+============================================================
+
+1. Async programming in Rust - Tokio
+   https://tokio.rs/
+   Score: 0.95
+   Tokio is a runtime for writing reliable asynchronous applications with Rust.
+
+2. Asynchronous Programming in Rust
+   https://rust-lang.github.io/async-book/
+   Score: 0.92
+   A book explaining async/await in Rust...
+```
+
+### JSON (`--json`)
+
+```json
+{
+  "query": "rust async patterns",
+  "items": [
+    {
+      "title": "Async programming in Rust - Tokio",
+      "url": "https://tokio.rs/",
+      "snippet": "Tokio is a runtime...",
+      "score": 0.95,
+      "sourceEngine": "tavily"
+    }
+  ],
+  "enginesTried": [
+    {
+      "engineId": "tavily",
+      "success": true
+    }
+  ],
+  "credits": [...]
+}
+```
+
+## Search Strategies
+
+### All (Default)
+
+Queries all configured/enabled providers and combines results.
+
+```bash
+allsearch "topic" --strategy all
+```
+
+- Pro: Gets maximum coverage, see different perspectives
+- Con: Uses more credits, slower
+- Best for: Research, comparison, getting API formats
+
+### First Success
+
+Stops after the first provider returns results.
+
+```bash
+allsearch "topic" --strategy first-success
+```
+
+- Pro: Saves credits, faster
+- Con: Misses results from other providers
+- Best for: Quick lookups, production use
+
+## Development
+
+### Source layout
+
+```
+src/
+├── app/                  # Public surface (bootstrap + API exports)
+├── bootstrap/            # DI container wiring
+├── config/               # Config types, schema, loaders
+├── core/                 # Orchestrator, strategy, credits, docker helpers
+│   ├── docker/           # Docker compose helper, lifecycle manager
+│   ├── paths.ts          # XDG path utilities
+│   └── ...
+├── plugin/               # Plugin registry and built-ins
+├── providers/            # Provider implementations + shared helpers
+├── tool/                 # CLI-facing tool + interfaces
+└── cli.ts                # CLI entry
+
+providers/
+└── searxng/
+    ├── docker-compose.yml  # SearXNG Docker config (uses env var volumes)
+    └── config/
+        └── settings.yml    # Default SearXNG settings (copied to XDG on first run)
+```
+
+### Building
+
+```bash
+# Bundle to dist/
+bun run build
+
+# Creates:
+# dist/cli.js              - Bundled CLI
+# dist/providers/searxng/  - Docker compose + default settings
+```
+
+### Testing (Bun)
+
+- All: `SKIP_DOCKER_TESTS=true bun test --preload ./test/setup.ts test/`
+- Unit only: `bun run test:unit`
+- Integration (Docker optional): `SKIP_DOCKER_TESTS=false bun run test:integration`
+- Coverage: `SKIP_DOCKER_TESTS=true bun run test:coverage`
+
+See `docs/testing/README.md` for suite layout.
+
+## Troubleshooting
+
+- **Missing config**: Copy `docs/config/allsearch.config.json` to `~/.config/allsearch/config.json`
+- **Missing API key**: Set `TAVILY_API_KEY`, `BRAVE_API_KEY`, `LINKUP_API_KEY` environment variables
+- **SearXNG not healthy**: Ensure Docker is running. Check `~/.config/allsearch/searxng/config/settings.yml` exists
+- **SearXNG settings missing**: Run `allsearch health` once to bootstrap default config to XDG directory
+- **Path issues after bun link**: The CLI resolves paths relative to XDG directories, not the working directory
+
+## Providers
+
+| Provider | Type | API Key Required | Free Tier | Notes |
+|----------|------|------------------|-----------|-------|
+| **SearXNG** | `searchxng` | No | Unlimited (local) | Self-hosted, Docker auto-start |
+| **Tavily** | `tavily` | Yes | 1000/month | Best for AI/research queries |
+| **Brave** | `brave` | Yes | 2000/month | General web search |
+| **Linkup** | `linkup` | Yes | 1000/month | AI-powered search |
+
+### Getting API Keys
+
+- **Tavily**: https://tavily.com/ → Sign up → Dashboard → API Keys
+- **Brave**: https://brave.com/search/api/ → Get Started → Create App
+- **Linkup**: https://linkup.so/ → Sign up → API Keys
+- **SearXNG**: No key needed (runs locally via Docker)
+
+## Environment Variables
+
+### API Keys (required per enabled engine)
+
+```bash
+# Add to ~/.bashrc, ~/.zshrc, or use a secrets manager
+export TAVILY_API_KEY="tvly-..."      # From tavily.com dashboard
+export BRAVE_API_KEY="BSA..."          # From brave.com/search/api
+export LINKUP_API_KEY="xxxxxxxx-..."   # UUID from linkup.so
+# SEARXNG_API_KEY not needed (local Docker)
+```
+
+### XDG Directories (optional)
+- `XDG_CONFIG_HOME` - Config directory (default: `~/.config`)
+- `XDG_DATA_HOME` - Data directory (default: `~/.local/share`)
+- `XDG_STATE_HOME` - State directory (default: `~/.local/state`)
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "ubersearch",
+  "version": "0.0.0-development",
+  "module": "src/index.ts",
+  "type": "module",
+  "bin": {
+    "ubersearch": "src/cli.ts"
+  },
+  "exports": {
+    ".": {
+      "import": "./src/index.ts"
+    },
+    "./cli": "./src/cli.ts",
+    "./config": "./src/config/defineConfig.ts",
+    "./types": "./src/app/index.ts"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "devDependencies": {
+    "@biomejs/biome": "^2.3.8",
+    "@semantic-release/commit-analyzer": "^13.0.0",
+    "@semantic-release/github": "^11.0.1",
+    "@semantic-release/npm": "^12.0.1",
+    "@semantic-release/release-notes-generator": "^14.0.1",
+    "@types/bun": "latest",
+    "semantic-release": "^24.2.0"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  },
+  "dependencies": {
+    "zod": "^4.1.12"
+  },
+  "scripts": {
+    "lint": "bunx biome check .",
+    "lint:fix": "bunx biome check --write .",
+    "format": "bunx biome format --write .",
+    "test": "SKIP_DOCKER_TESTS=true bun test --preload ./test/setup.ts test/",
+    "test:unit": "bun test --preload ./test/setup.ts test/unit/",
+    "test:integration": "bun test --preload ./test/setup.ts test/integration/",
+    "test:e2e": "bun test --preload ./test/setup.ts test/e2e/",
+    "test:coverage": "SKIP_DOCKER_TESTS=true bun test --preload ./test/setup.ts --coverage test/",
+    "test:watch": "SKIP_DOCKER_TESTS=true bun test --preload ./test/setup.ts --watch test/",
+    "test:verbose": "DEBUG_TESTS=1 SKIP_DOCKER_TESTS=true bun test --preload ./test/setup.ts test/",
+    "test:docker": "SKIP_DOCKER_TESTS=false bun test --preload ./test/setup.ts test/integration/",
+    "mcp": "bun run mcp-server.ts",
+    "mcp:test": "bun run scripts/test-mcp.ts",
+    "build": "bun run scripts/build.ts",
+    "build:binary": "bun build --compile --minify src/cli.ts --outfile dist/allsearch"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/briansunter/allsearch.git"
+  },
+  "bugs": {
+    "url": "https://github.com/briansunter/allsearch/issues"
+  },
+  "homepage": "https://github.com/briansunter/allsearch#readme",
+  "keywords": [
+    "search",
+    "ai",
+    "search-api",
+    "search-engine",
+    "tavily",
+    "brave",
+    "searxng"
+  ],
+  "author": "Brian Sunter",
+  "license": "MIT"
+}

package/src/app/index.ts

@@ -0,0 +1,30 @@
+// Public API surface for consumers importing the library (non-CLI).
+
+export { bootstrapContainer } from "../bootstrap/container";
+// Config helpers
+export {
+  createConfig,
+  defineBrave,
+  defineConfig,
+  defineEngine,
+  defineLinkup,
+  definePlugin,
+  defineSearchxng,
+  defineTavily,
+} from "../config/defineConfig";
+export type {
+  BraveConfig,
+  EngineConfig,
+  LinkupConfig,
+  AllSearchConfig,
+  SearchxngConfig,
+  TavilyConfig,
+} from "../config/types";
+export type { EngineId, SearchQuery, SearchResponse, SearchResultItem } from "../core/types";
+// Types
+export type { AllSearchInput, AllSearchOutput, AllSearchOutputItem } from "../tool/interface";
+export type {
+  GetCreditStatusOptions,
+  AllSearchOptions as ToolAllSearchOptions,
+} from "../tool/multiSearchTool";
+export { getCreditStatus, multiSearch } from "../tool/multiSearchTool";

package/src/bootstrap/container.ts

@@ -0,0 +1,157 @@
+/**
+ * Bootstrap the Dependency Injection Container
+ *
+ * Sets up all services and their dependencies
+ */
+
+import { loadConfig } from "../config/load";
+import type { EngineConfig, AllSearchConfig } from "../config/types";
+import { type Container, container } from "../core/container";
+import { CreditManager } from "../core/credits";
+import { FileCreditStateProvider } from "../core/credits/FileCreditStateProvider";
+import { createLogger } from "../core/logger";
+import { AllSearchOrchestrator } from "../core/orchestrator";
+import type { ILifecycleProvider, SearchProvider } from "../core/provider";
+import { ProviderRegistry } from "../core/provider";
+import { ProviderFactory } from "../core/provider/ProviderFactory";
+import { ServiceKeys } from "../core/serviceKeys";
+import { StrategyFactory } from "../core/strategy/StrategyFactory";
+import { PluginRegistry as PluginReg, registerBuiltInPlugins } from "../plugin";
+
+const log = createLogger("Bootstrap");
+
+/**
+ * Bootstrap options
+ */
+export interface BootstrapOptions {
+  /** Custom credit state path (overrides config) */
+  creditStatePath?: string;
+  /** Skip plugin registration (for testing) */
+  skipPluginRegistration?: boolean;
+}
+
+/**
+ * Bootstrap the DI container with all services
+ *
+ * @param configOrPath - Either a config file path (string) or a config object directly
+ * @param options - Bootstrap options (or legacy creditStatePath string)
+ */
+export async function bootstrapContainer(
+  configOrPath?: string | AllSearchConfig,
+  options?: BootstrapOptions | string,
+): Promise<Container> {
+  // Clear existing registrations (useful for testing)
+  container.reset();
+
+  // Handle legacy second argument (creditStatePath)
+  const opts: BootstrapOptions =
+    typeof options === "string" ? { creditStatePath: options } : (options ?? {});
+
+  // Load or use provided configuration
+  let config: AllSearchConfig;
+  if (typeof configOrPath === "object" && configOrPath !== null) {
+    // Config object provided directly (useful for testing)
+    config = configOrPath;
+
+    // Ensure plugins are registered when config is provided directly
+    if (!opts.skipPluginRegistration) {
+      await registerBuiltInPlugins(PluginReg.getInstance());
+    }
+  } else {
+    // Load from file (plugins registered by loadConfig)
+    config = await loadConfig(configOrPath);
+  }
+
+  // Register configuration as singleton
+  container.singleton(ServiceKeys.CONFIG, () => config);
+
+  // Register credit state provider
+  container.singleton(ServiceKeys.CREDIT_STATE_PROVIDER, () => {
+    const creditStatePath = opts.creditStatePath ?? config.storage?.creditStatePath;
+    return new FileCreditStateProvider(creditStatePath);
+  });
+
+  // Register credit manager
+  container.singleton(ServiceKeys.CREDIT_MANAGER, () => {
+    const enabledEngines = config.engines.filter((e) => e.enabled);
+    const stateProvider = container.get<FileCreditStateProvider>(ServiceKeys.CREDIT_STATE_PROVIDER);
+    return new CreditManager(enabledEngines, stateProvider);
+  });
+
+  // Register provider registry
+  container.singleton(ServiceKeys.PROVIDER_REGISTRY, () => {
+    const registry = new ProviderRegistry();
+    const failedProviders: string[] = [];
+
+    // Register all enabled providers
+    for (const engineConfig of config.engines) {
+      if (!engineConfig.enabled) {
+        continue;
+      }
+
+      try {
+        const provider = createProvider(engineConfig);
+        registry.register(provider);
+        log.info(`Registered provider: ${engineConfig.id}`);
+      } catch (error) {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        log.warn(`Failed to register provider ${engineConfig.id}: ${errorMsg}`);
+        failedProviders.push(engineConfig.id);
+      }
+    }
+
+    const availableProviders = registry.list();
+    if (availableProviders.length === 0) {
+      throw new Error(
+        `No providers could be registered. Failed providers: ${failedProviders.join(", ")}. ` +
+          "Check your configuration and environment variables.",
+      );
+    }
+
+    if (failedProviders.length > 0) {
+      log.warn(`Some providers failed to initialize: ${failedProviders.join(", ")}`);
+    }
+
+    return registry;
+  });
+
+  // Register strategy factory
+  container.singleton(ServiceKeys.STRATEGY_FACTORY, () => StrategyFactory);
+
+  // Register orchestrator
+  container.singleton(ServiceKeys.ORCHESTRATOR, () => {
+    const creditManager = container.get<CreditManager>(ServiceKeys.CREDIT_MANAGER);
+    const providerRegistry = container.get<ProviderRegistry>(ServiceKeys.PROVIDER_REGISTRY);
+    return new AllSearchOrchestrator(config, creditManager, providerRegistry);
+  });
+
+  // Initialize services that need async setup
+  const creditManager = container.get<CreditManager>(ServiceKeys.CREDIT_MANAGER);
+  await creditManager.initialize();
+
+  // Resolve provider registry to trigger validation (throws if no providers registered)
+  container.get<ProviderRegistry>(ServiceKeys.PROVIDER_REGISTRY);
+
+  return container;
+}
+
+/**
+ * Create a provider instance based on engine configuration
+ */
+function createProvider(engineConfig: EngineConfig): SearchProvider {
+  return ProviderFactory.createProvider(engineConfig, container);
+}
+
+/**
+ * Helper function to check if a provider implements ILifecycleProvider
+ */
+export function isLifecycleProvider(provider: unknown): provider is ILifecycleProvider {
+  return (
+    provider != null &&
+    typeof provider === "object" &&
+    "init" in provider &&
+    typeof provider.init === "function" &&
+    "healthcheck" in provider &&
+    typeof provider.healthcheck === "function"
+  );
+}