ubersearch 0.0.0-development

package/src/config/load.ts

@@ -0,0 +1,368 @@
+/**
+ * Configuration loader with multi-path resolution and validation
+ *
+ * Supports multiple config formats:
+ * - JSON: allsearch.config.json
+ * - TypeScript: allsearch.config.ts (with defineConfig helper)
+ *
+ * Config resolution order:
+ * 1. Explicit path (if provided)
+ * 2. Local directory (./allsearch.config.{ts,json})
+ * 3. XDG config ($XDG_CONFIG_HOME/allsearch/config.{ts,json})
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, isAbsolute, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { PluginRegistry, registerBuiltInPlugins } from "../plugin";
+import type { ConfigFactory, ExtendedSearchConfig } from "./defineConfig";
+import { formatValidationErrors, validateConfigSafe } from "./validation";
+
+/**
+ * Get package root directory
+ * Handles both development (src/) and bundled (dist/) environments
+ */
+function getPackageRoot(): string {
+  const currentFile = fileURLToPath(import.meta.url);
+  const currentDir = dirname(currentFile);
+
+  // Check if we're in dist/ or src/
+  if (currentDir.includes("/dist")) {
+    // Bundled: dist/cli.js -> go up 1 level
+    return dirname(currentDir);
+  }
+  // Development: src/config/load.ts -> go up 2 levels
+  return dirname(dirname(currentDir));
+}
+
+/** Supported config file extensions */
+const CONFIG_EXTENSIONS = [".ts", ".json"] as const;
+
+/** Config file base names */
+const CONFIG_BASENAMES = {
+  local: "allsearch.config",
+  xdg: "config",
+} as const;
+
+/**
+ * Get local config paths (current directory)
+ * Returns paths for both .ts and .json variants
+ */
+function getLocalConfigPaths(): string[] {
+  const base = join(process.cwd(), CONFIG_BASENAMES.local);
+  return CONFIG_EXTENSIONS.map((ext) => `${base}${ext}`);
+}
+
+/**
+ * Get XDG config paths ($XDG_CONFIG_HOME/allsearch/config.{ts,json})
+ */
+function getXdgConfigPaths(): string[] {
+  const xdg = process.env.XDG_CONFIG_HOME;
+  const baseDir = xdg ?? join(homedir(), ".config");
+  const base = join(baseDir, "allsearch", CONFIG_BASENAMES.xdg);
+  return CONFIG_EXTENSIONS.map((ext) => `${base}${ext}`);
+}
+
+/**
+ * Get all possible config file paths in order of preference
+ * TypeScript files are preferred over JSON when both exist
+ */
+export function getConfigPaths(explicitPath?: string): string[] {
+  const paths: string[] = [];
+
+  // Explicit path first (if provided)
+  if (explicitPath) {
+    paths.push(explicitPath);
+  }
+
+  // Local directory (prefer .ts over .json)
+  paths.push(...getLocalConfigPaths());
+
+  // XDG config (prefer .ts over .json)
+  paths.push(...getXdgConfigPaths());
+
+  return paths;
+}
+
+/**
+ * Check if a path is a TypeScript config
+ */
+function isTypeScriptConfig(filePath: string): boolean {
+  return filePath.endsWith(".ts");
+}
+
+/**
+ * Load a TypeScript config file
+ * Uses Bun's native TS support for importing
+ */
+async function loadTypeScriptConfig(path: string): Promise<ExtendedSearchConfig> {
+  try {
+    // Use dynamic import for TS files
+    // Bun natively supports importing .ts files
+    const module = await import(path);
+
+    // Config can be default export or named 'config'
+    const configOrFactory = module.default ?? module.config;
+
+    if (!configOrFactory) {
+      throw new Error(`Config file must export a default configuration or named 'config' export`);
+    }
+
+    // Handle factory functions (async config)
+    if (typeof configOrFactory === "function") {
+      return await (configOrFactory as ConfigFactory)();
+    }
+
+    return configOrFactory as ExtendedSearchConfig;
+  } catch (error) {
+    throw new Error(
+      `Failed to load TypeScript config from ${path}: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+}
+
+/**
+ * Load a JSON config file
+ */
+function loadJsonConfig(path: string): ExtendedSearchConfig {
+  const raw = readFileSync(path, "utf8");
+  return JSON.parse(raw);
+}
+
+/**
+ * Resolve relative paths in config to absolute paths
+ * This ensures paths like ./providers/searxng/docker-compose.yml work
+ * when running from any directory
+ *
+ * @param config - The loaded configuration
+ * @param configFilePath - The absolute path to the config file
+ * @returns Config with resolved absolute paths
+ */
+function resolveConfigPaths(
+  config: ExtendedSearchConfig,
+  configFilePath: string,
+): ExtendedSearchConfig {
+  // Handle empty or malformed config
+  if (!config.engines || !Array.isArray(config.engines)) {
+    return config;
+  }
+
+  const configDir = dirname(configFilePath);
+
+  return {
+    ...config,
+    engines: config.engines.map((engine) => {
+      // Check if engine has composeFile property (SearXNG type)
+      if ("composeFile" in engine && engine.composeFile) {
+        // Only resolve if it's a relative path
+        if (!isAbsolute(engine.composeFile)) {
+          return {
+            ...engine,
+            composeFile: join(configDir, engine.composeFile),
+          };
+        }
+      }
+      return engine;
+    }),
+  };
+}
+
+/**
+ * Options for loading configuration
+ */
+export interface LoadConfigOptions {
+  /** Skip config validation */
+  skipValidation?: boolean;
+  /** Custom plugin registry (defaults to singleton) */
+  registry?: PluginRegistry;
+  /** Skip registering built-in plugins */
+  skipBuiltInPlugins?: boolean;
+}
+
+/**
+ * Load configuration from the first available config file
+ * Supports both JSON and TypeScript configurations
+ *
+ * @param explicitPath Optional explicit path to config file
+ * @param options Optional loading options
+ * @returns Parsed and validated configuration
+ * @throws Error if no config file is found or validation fails
+ *
+ * @example
+ * ```typescript
+ * // Load from default locations
+ * const config = await loadConfig();
+ *
+ * // Load from specific path
+ * const config = await loadConfig('./my-config.ts');
+ *
+ * // Load with options
+ * const config = await loadConfig(undefined, { skipValidation: true });
+ * ```
+ */
+/**
+ * Get default configuration when no config file is found
+ * Uses SearXNG as a sensible default (no API key required)
+ */
+function getDefaultConfig(): ExtendedSearchConfig {
+  const packageRoot = getPackageRoot();
+  const composeFile = join(packageRoot, "providers", "searxng", "docker-compose.yml");
+
+  return {
+    defaultEngineOrder: ["searchxng"],
+    engines: [
+      {
+        id: "searchxng",
+        type: "searchxng",
+        enabled: true,
+        displayName: "SearXNG (Local)",
+        apiKeyEnv: "SEARXNG_API_KEY",
+        endpoint: "http://localhost:8888/search",
+        composeFile,
+        containerName: "searxng",
+        healthEndpoint: "http://localhost:8888/healthz",
+        defaultLimit: 10,
+        monthlyQuota: 10000,
+        creditCostPerSearch: 0,
+        lowCreditThresholdPercent: 80,
+        autoStart: true,
+        autoStop: true,
+        initTimeoutMs: 60000,
+      },
+    ],
+  };
+}
+
+export async function loadConfig(
+  explicitPath?: string,
+  options: LoadConfigOptions = {},
+): Promise<ExtendedSearchConfig> {
+  const { skipValidation = false, registry, skipBuiltInPlugins = false } = options;
+  const paths = getConfigPaths(explicitPath);
+
+  for (const path of paths) {
+    if (existsSync(path)) {
+      let rawConfig: ExtendedSearchConfig;
+
+      try {
+        if (isTypeScriptConfig(path)) {
+          rawConfig = await loadTypeScriptConfig(path);
+        } else {
+          rawConfig = loadJsonConfig(path);
+        }
+      } catch (error) {
+        throw new Error(
+          `Failed to load config file at ${path}: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+
+      // Resolve relative paths (like composeFile) to absolute paths
+      // based on the config file's directory
+      rawConfig = resolveConfigPaths(rawConfig, path);
+
+      // Skip validation if explicitly requested (useful for testing)
+      if (!skipValidation) {
+        // Validate configuration against schema
+        const result = validateConfigSafe(rawConfig);
+        if (!result.success) {
+          const errors = formatValidationErrors(result.error);
+          throw new Error(
+            `Invalid configuration in ${path}:\n${errors.map((e) => `  - ${e}`).join("\n")}`,
+          );
+        }
+        rawConfig = result.data as ExtendedSearchConfig;
+      }
+
+      // Register plugins
+      const pluginRegistry = registry ?? PluginRegistry.getInstance();
+
+      // Register built-in plugins first (unless skipped)
+      if (!skipBuiltInPlugins) {
+        await registerBuiltInPlugins(pluginRegistry);
+      }
+
+      // Register any custom plugins from config
+      if (rawConfig.plugins) {
+        for (const plugin of rawConfig.plugins) {
+          await pluginRegistry.register(plugin);
+        }
+      }
+
+      return rawConfig;
+    }
+  }
+
+  // No config file found - use default configuration
+  console.warn("No config file found. Using default configuration (SearXNG).");
+  console.warn("Create a config file for custom providers: allsearch.config.json");
+
+  const defaultConfig = getDefaultConfig();
+
+  // Register plugins for default config
+  const pluginRegistry = registry ?? PluginRegistry.getInstance();
+  if (!skipBuiltInPlugins) {
+    await registerBuiltInPlugins(pluginRegistry);
+  }
+
+  return defaultConfig;
+}
+
+/**
+ * Synchronous version of loadConfig for JSON files only
+ * @deprecated Use loadConfig() for full TypeScript support
+ */
+export function loadConfigSync(
+  explicitPath?: string,
+  options: { skipValidation?: boolean } = {},
+): ExtendedSearchConfig {
+  const paths = getConfigPaths(explicitPath);
+
+  for (const path of paths) {
+    if (existsSync(path)) {
+      // Skip TypeScript files in sync mode
+      if (isTypeScriptConfig(path)) {
+        continue;
+      }
+
+      let rawConfig: ExtendedSearchConfig;
+
+      try {
+        rawConfig = loadJsonConfig(path);
+      } catch (error) {
+        throw new Error(
+          `Failed to parse config file at ${path}: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+
+      // Resolve relative paths (like composeFile) to absolute paths
+      rawConfig = resolveConfigPaths(rawConfig, path);
+
+      if (options.skipValidation) {
+        return rawConfig;
+      }
+
+      const result = validateConfigSafe(rawConfig);
+      if (!result.success) {
+        const errors = formatValidationErrors(result.error);
+        throw new Error(
+          `Invalid configuration in ${path}:\n${errors.map((e) => `  - ${e}`).join("\n")}`,
+        );
+      }
+
+      return result.data as ExtendedSearchConfig;
+    }
+  }
+
+  console.warn("No config file found. Using default configuration (SearXNG).");
+  console.warn("Create a config file for custom providers: allsearch.config.json");
+  return getDefaultConfig();
+}
+
+/**
+ * Check if a config file exists at any of the standard locations
+ */
+export function configExists(): boolean {
+  const paths = getConfigPaths();
+  return paths.some((path) => existsSync(path));
+}

package/src/config/types.ts

@@ -0,0 +1,86 @@
+/**
+ * Configuration types for allsearch
+ */
+
+import type { EngineId } from "../core/types";
+
+/**
+ * Docker configuration options for providers that can manage container lifecycle
+ */
+export interface DockerConfigurable {
+  /** Whether to auto-start the container on init */
+  autoStart?: boolean;
+  /** Whether to auto-stop the container on shutdown */
+  autoStop?: boolean;
+  /** Path to docker-compose file */
+  composeFile?: string;
+  /** Name of the container to manage */
+  containerName?: string;
+  /** Health check endpoint URL */
+  healthEndpoint?: string;
+  /** Timeout for initialization in milliseconds */
+  initTimeoutMs?: number;
+}
+
+export interface EngineConfigBase {
+  /** Unique identifier for this engine */
+  id: EngineId;
+
+  /** Whether this engine is enabled */
+  enabled: boolean;
+
+  /** Human-readable display name */
+  displayName: string;
+
+  /** Monthly query quota */
+  monthlyQuota: number;
+
+  /** Credit cost per search */
+  creditCostPerSearch: number;
+
+  /** Warning threshold (percentage of quota used) */
+  lowCreditThresholdPercent: number;
+}
+
+export interface TavilyConfig extends EngineConfigBase {
+  type: "tavily";
+  apiKeyEnv: string;
+  endpoint: string;
+  searchDepth: "basic" | "advanced";
+}
+
+export interface BraveConfig extends EngineConfigBase {
+  type: "brave";
+  apiKeyEnv: string;
+  endpoint: string;
+  defaultLimit: number;
+}
+
+export interface LinkupConfig extends EngineConfigBase, DockerConfigurable {
+  type: "linkup";
+  apiKeyEnv: string;
+  endpoint: string;
+}
+
+export interface SearchxngConfig extends EngineConfigBase, DockerConfigurable {
+  type: "searchxng";
+  apiKeyEnv?: string;
+  endpoint: string;
+  defaultLimit: number;
+}
+
+export type EngineConfig = TavilyConfig | BraveConfig | LinkupConfig | SearchxngConfig;
+
+export interface AllSearchConfig {
+  /** Default order to try engines (can be overridden per query) */
+  defaultEngineOrder: EngineId[];
+
+  /** Configuration for each search provider */
+  engines: EngineConfig[];
+
+  /** Storage settings */
+  storage?: {
+    /** Path to store credit state (defaults to ~/.local/state/allsearch/credits.json) */
+    creditStatePath?: string;
+  };
+}

package/src/config/validation.ts

@@ -0,0 +1,148 @@
+/**
+ * Zod schemas for configuration validation
+ */
+
+import { z } from "zod";
+
+// Base engine configuration (uses passthrough to preserve extra fields)
+export const EngineConfigBaseSchema = z
+  .object({
+    id: z.string().min(1),
+    enabled: z.boolean(),
+    displayName: z.string().min(1),
+    monthlyQuota: z.number().int().positive(),
+    creditCostPerSearch: z.number().int().nonnegative(), // Allow 0 for free engines
+    lowCreditThresholdPercent: z.number().min(0).max(100),
+  })
+  .passthrough();
+
+// Docker-specific configuration
+export const DockerConfigSchema = z.object({
+  autoStart: z.boolean().optional(),
+  autoStop: z.boolean().optional(),
+  composeFile: z.string().optional(),
+  containerName: z.string().optional(),
+  healthEndpoint: z.string().url().optional(),
+  initTimeoutMs: z.number().int().positive().optional(),
+});
+
+// Provider-specific schemas
+export const TavilyConfigSchema = EngineConfigBaseSchema.extend({
+  type: z.literal("tavily"),
+  apiKeyEnv: z.string(),
+  endpoint: z.string().url(),
+  searchDepth: z.enum(["basic", "advanced"]),
+});
+
+export const BraveConfigSchema = EngineConfigBaseSchema.extend({
+  type: z.literal("brave"),
+  apiKeyEnv: z.string(),
+  endpoint: z.string().url(),
+  defaultLimit: z.number().int().positive(),
+});
+
+export const LinkupConfigSchema = EngineConfigBaseSchema.extend({
+  type: z.literal("linkup"),
+  apiKeyEnv: z.string(),
+  endpoint: z.string().url(),
+}).merge(DockerConfigSchema);
+
+export const SearchxngConfigSchema = EngineConfigBaseSchema.extend({
+  type: z.literal("searchxng"),
+  apiKeyEnv: z.string().optional(),
+  endpoint: z.string().url(),
+  defaultLimit: z.number().int().positive(),
+}).merge(DockerConfigSchema);
+
+// Union type for all engine configs
+export const EngineConfigSchema = z.discriminatedUnion("type", [
+  TavilyConfigSchema,
+  BraveConfigSchema,
+  LinkupConfigSchema,
+  SearchxngConfigSchema,
+]);
+
+// Main configuration schema (uses passthrough to preserve extra fields)
+export const AllSearchConfigSchema = z
+  .object({
+    defaultEngineOrder: z.array(z.string()).min(1),
+    engines: z.array(EngineConfigSchema).min(1),
+    storage: z
+      .object({
+        creditStatePath: z.string().optional(),
+      })
+      .optional(),
+  })
+  .passthrough();
+
+// CLI input schema
+export const CliInputSchema = z.object({
+  query: z.string().min(1),
+  limit: z.number().int().positive().optional(),
+  engines: z.array(z.string()).min(1).optional(),
+  includeRaw: z.boolean().optional(),
+  strategy: z.enum(["all", "first-success"]).optional(),
+  json: z.boolean().optional(),
+});
+
+// Export inferred types from schemas
+export type ValidatedAllSearchConfig = z.infer<typeof AllSearchConfigSchema>;
+export type ValidatedEngineConfig = z.infer<typeof EngineConfigSchema>;
+export type ValidatedTavilyConfig = z.infer<typeof TavilyConfigSchema>;
+export type ValidatedBraveConfig = z.infer<typeof BraveConfigSchema>;
+export type ValidatedLinkupConfig = z.infer<typeof LinkupConfigSchema>;
+export type ValidatedSearchxngConfig = z.infer<typeof SearchxngConfigSchema>;
+export type ValidatedCliInput = z.infer<typeof CliInputSchema>;
+
+/**
+ * Validate configuration against schema
+ * @param config - Raw configuration object
+ * @returns Validated configuration
+ * @throws ZodError if validation fails
+ */
+export function validateConfig(config: unknown): ValidatedAllSearchConfig {
+  return AllSearchConfigSchema.parse(config);
+}
+
+/**
+ * Validate configuration safely (returns result object instead of throwing)
+ * @param config - Raw configuration object
+ * @returns Result object with success status and data or error
+ */
+export function validateConfigSafe(config: unknown):
+  | {
+      success: true;
+      data: ValidatedAllSearchConfig;
+    }
+  | {
+      success: false;
+      error: z.ZodError;
+    } {
+  const result = AllSearchConfigSchema.safeParse(config);
+  if (result.success) {
+    return { success: true, data: result.data };
+  }
+  return { success: false, error: result.error };
+}
+
+/**
+ * Validate CLI input against schema
+ * @param input - Raw CLI input object
+ * @returns Validated CLI input
+ * @throws ZodError if validation fails
+ */
+export function validateCliInput(input: unknown): ValidatedCliInput {
+  return CliInputSchema.parse(input);
+}
+
+/**
+ * Format Zod validation errors into human-readable messages
+ * @param error - ZodError from validation
+ * @returns Array of formatted error messages
+ */
+export function formatValidationErrors(error: z.ZodError): string[] {
+  return error.issues.map((err) => {
+    const path = err.path.join(".");
+    return path ? `${path}: ${err.message}` : err.message;
+  });
+}

package/src/core/cache.ts

@@ -0,0 +1,74 @@
+/**
+ * Request Cache for Multi-Search
+ *
+ * Simple in-memory cache with TTL support for caching search results
+ */
+
+/**
+ * Cache entry with expiration time
+ */
+interface CacheEntry<T> {
+  data: T;
+  expiresAt: number;
+}
+
+/**
+ * Simple in-memory cache with TTL support
+ */
+export class SearchCache {
+  private cache = new Map<string, CacheEntry<unknown>>();
+  private readonly DEFAULT_TTL_MS = 5 * 60 * 1000;
+
+  /**
+   * Get a value from the cache if it exists and hasn't expired
+   *
+   * @param key - Cache key
+   * @returns Cached value or undefined if not found/expired
+   */
+  get<T>(key: string): T | undefined {
+    const entry = this.cache.get(key);
+    if (!entry) {
+      return undefined;
+    }
+
+    if (Date.now() > entry.expiresAt) {
+      this.cache.delete(key);
+      return undefined;
+    }
+
+    return entry.data as T;
+  }
+
+  /**
+   * Set a value in the cache with TTL
+   *
+   * @param key - Cache key
+   * @param value - Value to cache
+   * @param ttlMs - Time to live in milliseconds (defaults to 5 minutes)
+   */
+  set<T>(key: string, value: T, ttlMs = this.DEFAULT_TTL_MS): void {
+    this.cache.set(key, {
+      data: value,
+      expiresAt: Date.now() + ttlMs,
+    });
+  }
+
+  /**
+   * Clear all cache entries
+   */
+  clear(): void {
+    this.cache.clear();
+  }
+
+  /**
+   * Remove expired entries from the cache
+   */
+  prune(): void {
+    const now = Date.now();
+    for (const [key, entry] of this.cache.entries()) {
+      if (now > entry.expiresAt) {
+        this.cache.delete(key);
+      }
+    }
+  }
+}