sxng-cli 1.0.5 → 1.0.7

package/dist/runCli.js

@@ -4,38 +4,65 @@
 import { createSuccessEnvelope, createErrorEnvelope } from './protocol.js';
 import { config } from './config.js';
 import { initConfig } from './init.js';
+import { runExtract } from './commands/extract.js';
+import { ContentExtractor } from './deep/extractor.js';
+import { rrf } from './deep/rrf.js';
+import { normalizeUrl } from './deep/dedupe.js';
+import { deserializeGraph, serializeGraph, graphStats, resultId } from './deep/graph.js';
+import { initSessionDir, resolveSessionPath, appendSessionResults, updateSessionGraph, loadSessionResults } from './deep/session.js';
+import { runSessionList, runSessionDelete } from './commands/session.js';
+import { DirectedGraph } from 'graphology';
+import { readFileSync, writeFileSync, existsSync } from 'fs';
 const HELP_TEXT = `SearXNG CLI - Web Search Tool
 
 Usage:
-  sxng <query> [options]
-  sxng init
-  sxng --engines <engine1,engine2> <query>
-  sxng --categories <category> <query>
-  sxng --health
+  sxng <query> [options]             Single search
+  sxng --queries "q1,q2,q3"          Multi-query search (RRF fusion)
+  sxng --session new <query>         Create new session and search (auto-named)
+  sxng --session <dir> <query>       Multi-round session with result accumulation
+  sxng extract --urls/--from-json    Extract article content
+  sxng extract --session <dir>       Extract content from session and merge back
+  sxng query-graph <path> --seeds    Query subgraph via BFS
+  sxng graph-add <path> --data       Add entities/edges to knowledge graph
+  sxng session-list                  List all sessions
+  sxng session-delete <names>        Delete sessions by name
+  sxng session-delete --older <h>    Delete sessions older than N hours
+  sxng init                          Interactive configuration setup
+  sxng --health                      Check SearXNG server health
 
-Commands:
-  init                         Interactive configuration setup
-  <query>                      Perform a web search with the given query
-
-Options:
+Search Options:
   -e, --engines <engines>      Comma-separated list of search engines
   -c, --categories <cats>      Comma-separated list of categories
   -l, --limit <n>              Maximum number of results (default: ${config.defaultLimit})
   -p, --page <n>               Page number for pagination
   --lang <code>                Language code (e.g., en, zh, ja)
   --time <range>               Time range: day, week, month, year, all
-  -f, --format <fmt>           Output format: md, json, csv, html (default: md)
-  --engines-list               List available search engines
-  --categories-list            List available categories
-  --health                     Check SearXNG server health
-  -h, --help                   Show this help message
+  -f, --format <fmt>           Output format: md (default), json
+  --queries <q1,q2,q3>        Multi-query with RRF fusion
+  --merge <file>               Merge new results with previous search JSON
+  --session <dir|new>          Session dir, or "new" to auto-create
+  --owner <name>               Session owner (stored in meta.json)
+  --desc <text>                Session description (stored in meta.json)
+  --graph <file>               Save search result metadata to knowledge graph file
+
+Extract Options:
+  --urls <url1,url2>           URLs to extract content from
+  --from-json <file>           Extract from search results JSON file
+  --session <dir>              Extract from session results and merge content back
+
+Graph Options:
+  --seeds <s1,s2>              Seed nodes for query-graph BFS
+  --depth <n>                  BFS depth for query-graph (default: 2)
+  --data <json>                JSON with entities/edges for graph-add
 
 Examples:
-  sxng init
   sxng "TypeScript tutorial"
-  sxng --engines google,github "react hooks"
-  sxng --categories it,science "machine learning"
-  sxng --limit 5 --time week "latest news"
+  sxng --session new "rust async" --owner "agent-1" --desc "async ecosystem research"
+  sxng --session /tmp/s "rust async"
+  sxng extract --session /tmp/s
+  sxng session-list
+  sxng session-delete ds_1745000000_abcd123
+  sxng session-delete --older 24
 
 Environment Variables:
   SEARXNG_BASE_URL             SearXNG server URL
@@ -49,6 +76,11 @@ Environment Variables:
 function parseArgs(args) {
     const options = {
         init: false,
+        extract: false,
+        queryGraph: false,
+        graphAdd: false,
+        sessionList: false,
+        sessionDelete: false,
         enginesList: false,
         categoriesList: false,
         health: false,
@@ -65,6 +97,21 @@ function parseArgs(args) {
             case 'init':
                 options.init = true;
                 break;
+            case 'extract':
+                options.extract = true;
+                break;
+            case 'query-graph':
+                options.queryGraph = true;
+                break;
+            case 'graph-add':
+                options.graphAdd = true;
+                break;
+            case 'session-list':
+                options.sessionList = true;
+                break;
+            case 'session-delete':
+                options.sessionDelete = true;
+                break;
             case '-e':
             case '--engines':
                 options.engines = args[++i]?.split(',').map(e => e.trim()).filter(Boolean);
@@ -93,10 +140,46 @@ function parseArgs(args) {
             case '-f':
             case '--format':
                 const fmt = args[++i];
-                if (['json', 'csv', 'html', 'md', 'markdown'].includes(fmt)) {
+                if (fmt === 'json' || fmt === 'md' || fmt === 'markdown') {
                     options.format = fmt === 'markdown' ? 'md' : fmt;
                 }
                 break;
+            case '--queries':
+                options.queries = args[++i]?.split(',').map(q => q.trim()).filter(Boolean);
+                break;
+            case '--urls':
+                options.urls = args[++i]?.split(',').map(u => u.trim()).filter(Boolean);
+                break;
+            case '--from-json':
+                options.fromJson = args[++i];
+                break;
+            case '--merge':
+                options.merge = args[++i];
+                break;
+            case '--graph':
+                options.graph = args[++i];
+                break;
+            case '--session':
+                options.session = args[++i];
+                break;
+            case '--seeds':
+                options.seeds = args[++i]?.split(',').map(s => s.trim()).filter(Boolean);
+                break;
+            case '--depth':
+                options.depth = parseInt(args[++i], 10);
+                break;
+            case '--data':
+                options.data = args[++i];
+                break;
+            case '--owner':
+                options.owner = args[++i];
+                break;
+            case '--desc':
+                options.desc = args[++i];
+                break;
+            case '--older':
+                options.older = parseFloat(args[++i]);
+                break;
             case '--engines-list':
                 options.enginesList = true;
                 break;
@@ -118,39 +201,6 @@ function parseArgs(args) {
     }
     return options;
 }
-function formatAsCsv(results) {
-    if (results.length === 0)
-        return '';
-    const headers = ['title', 'url', 'content', 'engine', 'category'];
-    const lines = [headers.join(',')];
-    for (const r of results) {
-        const row = headers.map(h => {
-            const val = String(r[h] || '').replace(/"/g, '""');
-            return `"${val}"`;
-        });
-        lines.push(row.join(','));
-    }
-    return lines.join('\n');
-}
-function formatAsHtml(results) {
-    const rows = results.map(r => `
-        <tr>
-            <td><a href="${r.url}">${r.title}</a></td>
-            <td>${r.content}</td>
-            <td>${r.engine}</td>
-        </tr>
-    `).join('');
-    return `<!DOCTYPE html>
-<html>
-<head><title>Search Results</title></head>
-<body>
-<table border="1">
-    <tr><th>Title</th><th>Content</th><th>Engine</th></tr>
-    ${rows}
-</table>
-</body>
-</html>`;
-}
 function formatAsMarkdown(data) {
     const lines = [];
     // Query info
@@ -163,10 +213,8 @@ function formatAsMarkdown(data) {
         lines.push('');
         for (let i = 0; i < results.length; i++) {
             const r = results[i];
-            // Title with score indicator
             let scoreIndicator = '';
             if (r.score !== undefined && r.score !== null) {
-                // Score might be already > 1 or in 0-1 range
                 const normalizedScore = r.score > 1 ? r.score : r.score * 100;
                 scoreIndicator = ` [${normalizedScore.toFixed(1)}]`;
             }
@@ -176,7 +224,6 @@ function formatAsMarkdown(data) {
                 lines.push(r.content);
                 lines.push('');
             }
-            // Metadata: engine, category, publishedDate (if available)
             const meta = [];
             meta.push(`Engine: ${r.engine || 'unknown'}`);
             if (r.category)
@@ -191,13 +238,13 @@ function formatAsMarkdown(data) {
         lines.push('No results found.');
         lines.push('');
     }
-    // Unresponsive engines (compact format)
+    // Unresponsive engines
     if (data.unresponsiveEngines && data.unresponsiveEngines.length > 0) {
         const unresponsive = data.unresponsiveEngines.map((item) => Array.isArray(item) ? item[0] : item).join(', ');
         lines.push(`*Unresponsive: ${unresponsive}*`);
         lines.push('');
     }
-    // Suggestions (if any)
+    // Suggestions
     if (data.suggestions && data.suggestions.length > 0) {
         lines.push('**Suggestions:** ' + data.suggestions.join(' · '));
         lines.push('');
@@ -205,17 +252,52 @@ function formatAsMarkdown(data) {
     return lines.join('\n');
 }
 function formatOutput(data, format) {
-    switch (format) {
-        case 'csv':
-            return formatAsCsv(data.results || []);
-        case 'html':
-            return formatAsHtml(data.results || []);
-        case 'md':
-            return formatAsMarkdown(data);
-        case 'json':
-        default:
-            return JSON.stringify(data, null, 2);
+    if (format === 'md')
+        return formatAsMarkdown(data);
+    return JSON.stringify(data, null, 2);
+}
+function addToGraph(graphFile, results) {
+    // Load existing graph or create new one
+    let graph;
+    if (existsSync(graphFile)) {
+        try {
+            const raw = readFileSync(graphFile, 'utf-8');
+            const parsed = JSON.parse(raw);
+            const graphData = parsed.status === 'ok' && parsed.data?.graph
+                ? parsed.data.graph
+                : (parsed.nodes && parsed.edges ? parsed : null);
+            if (graphData) {
+                graph = deserializeGraph(graphData);
+            }
+            else {
+                graph = new DirectedGraph();
+            }
+        }
+        catch {
+            graph = new DirectedGraph();
+        }
     }
+    else {
+        graph = new DirectedGraph();
+    }
+    // Save result metadata nodes (title, url, rank) into the graph
+    // Entity nodes and relationships are added by Agent via graph-add
+    for (let i = 0; i < results.length; i++) {
+        const r = results[i];
+        const id = resultId(r.url);
+        if (!graph.hasNode(id)) {
+            graph.mergeNode(id, {
+                type: 'result',
+                label: r.title,
+                url: r.url,
+                title: r.title,
+                rank: i + 1,
+            });
+        }
+    }
+    const serialized = serializeGraph(graph);
+    const stats = graphStats(graph);
+    writeFileSync(graphFile, JSON.stringify({ status: 'ok', data: { graph: serialized, stats } }, null, 2), 'utf-8');
 }
 export async function runCli(args, service) {
     const options = parseArgs(args);
@@ -226,6 +308,15 @@ export async function runCli(args, service) {
     if (options.init) {
         return await initConfig();
     }
+    if (options.extract) {
+        const extractor = new ContentExtractor();
+        const extractOptions = {
+            urls: options.urls,
+            fromJson: options.fromJson,
+            session: options.session,
+        };
+        return await runExtract(extractor, extractOptions);
+    }
     if (options.health) {
         const health = await service.healthCheck();
         const envelope = health.status === 'healthy'
@@ -276,8 +367,32 @@ export async function runCli(args, service) {
             return 1;
         }
     }
-    if (!options.query) {
-        const envelope = createErrorEnvelope('MISSING_QUERY', 'No search query provided', { hint: 'Use: sxng "your search query"' });
+    if (options.queryGraph) {
+        const { runQueryGraph } = await import('./commands/query-graph.js');
+        const graphFormat = options.format === 'json' ? 'json' : 'md';
+        return await runQueryGraph({
+            graphFile: options.query || options.fromJson || '',
+            seeds: options.seeds || [],
+            depth: options.depth ?? 2,
+            format: graphFormat,
+        });
+    }
+    if (options.graphAdd) {
+        const { runGraphAdd } = await import('./commands/graph-add.js');
+        return await runGraphAdd({
+            graphFile: options.query || options.fromJson || '',
+            data: options.data || '',
+        });
+    }
+    if (options.sessionList) {
+        return await runSessionList();
+    }
+    if (options.sessionDelete) {
+        const names = (options.query || '').split(',').map(n => n.trim()).filter(Boolean);
+        return await runSessionDelete(names, options.older);
+    }
+    if (!options.query && !options.queries) {
+        const envelope = createErrorEnvelope('MISSING_QUERY', 'No search query provided', { hint: 'Use: sxng "your search query" or sxng --queries "q1,q2,q3"' });
         console.log(JSON.stringify(envelope, null, 2));
         return 1;
     }
@@ -289,45 +404,224 @@ export async function runCli(args, service) {
             return 1;
         }
     }
-    const searchOptions = {
-        query: options.query,
-        engines: options.engines,
-        categories: options.categories,
-        limit: options.limit ?? config.defaultLimit,
-        page: options.page,
-        language: options.language,
-        timeRange: options.timeRange,
-        format: options.format || config.defaultFormat
-    };
+    const queries = options.queries || (options.query ? [options.query] : []);
+    const limit = options.limit ?? config.defaultLimit;
+    // Initialize session directory if --session specified
+    if (options.session) {
+        const resolved = resolveSessionPath(options.session);
+        options.session = resolved;
+        initSessionDir(resolved, options.owner, options.desc, options.query);
+    }
     try {
-        const results = await service.search(searchOptions);
-        const envelope = createSuccessEnvelope({
-            query: results.query,
-            totalResults: results.numberOfResults,
-            returnedResults: results.results.length,
-            results: results.results,
-            suggestions: results.suggestions,
-            answers: results.answers,
-            unresponsiveEngines: results.unresponsiveEngines
-        });
-        const outputFormat = options.format || config.defaultFormat;
-        if (outputFormat === 'md') {
-            // Markdown format: output clean markdown directly
-            console.log(formatOutput({
+        // Single query: use SearXNG directly (it already aggregates/sorts)
+        if (queries.length === 1 && !options.merge) {
+            const searchOptions = {
+                query: queries[0],
+                engines: options.engines,
+                categories: options.categories,
+                limit,
+                page: options.page,
+                language: options.language,
+                timeRange: options.timeRange,
+            };
+            const results = await service.search(searchOptions);
+            // Session: accumulate results and update graph
+            let sessionInfo = null;
+            if (options.session) {
+                sessionInfo = appendSessionResults(options.session, results.results);
+                for (const query of queries) {
+                    updateSessionGraph(options.session, query, results.results.map(r => ({
+                        url: r.url,
+                        title: r.title,
+                    })));
+                }
+            }
+            const outputFormat = options.format || config.defaultFormat;
+            // When session active, RRF-merge with all accumulated results for display
+            let displayResults = results.results;
+            if (options.session) {
+                const allSessionResults = loadSessionResults(options.session);
+                const rankings = [
+                    results.results.map(r => ({ id: normalizeUrl(r.url), ...r })),
+                    allSessionResults.map(r => ({ id: normalizeUrl(r.url), ...r })),
+                ];
+                const fused = rrf(rankings);
+                const urlMap = new Map();
+                for (const r of results.results)
+                    urlMap.set(normalizeUrl(r.url), r);
+                for (const r of allSessionResults)
+                    urlMap.set(normalizeUrl(r.url), r);
+                displayResults = fused
+                    .map(item => {
+                    const original = urlMap.get(item.id);
+                    return original ? { ...original, score: item.score } : null;
+                })
+                    .filter(Boolean);
+                if (limit > 0)
+                    displayResults = displayResults.slice(0, limit);
+            }
+            const displayData = {
                 query: results.query,
                 totalResults: results.numberOfResults,
-                results: results.results,
+                results: displayResults,
                 answers: results.answers,
                 suggestions: results.suggestions,
-                unresponsiveEngines: results.unresponsiveEngines
-            }, 'md'));
+                unresponsiveEngines: results.unresponsiveEngines,
+                ...(sessionInfo ? { session: { dir: options.session, added: sessionInfo.added, total: sessionInfo.total } } : {}),
+            };
+            if (outputFormat === 'md') {
+                console.log(formatOutput(displayData, 'md'));
+            }
+            else {
+                console.log(JSON.stringify(createSuccessEnvelope({
+                    ...displayData,
+                    returnedResults: displayResults.length,
+                }), null, 2));
+            }
+            // Auto-insert result metadata into knowledge graph if --graph specified
+            if (options.graph) {
+                addToGraph(options.graph, results.results);
+            }
+            return 0;
+        }
+        // Multi-query or --merge: RRF fusion across ranked lists
+        const allResponses = [];
+        for (const query of queries) {
+            const searchOptions = {
+                query,
+                engines: options.engines,
+                categories: options.categories,
+                limit,
+                page: options.page,
+                language: options.language,
+                timeRange: options.timeRange,
+            };
+            const response = await service.search(searchOptions);
+            allResponses.push(response);
+        }
+        // Build rankings for RRF: each query's results as a separate ranked list
+        let rankings = allResponses.map(resp => resp.results.map((r) => ({ id: normalizeUrl(r.url), ...r })));
+        // If --merge, load historical results and add as another ranking
+        let mergeData = null;
+        if (options.merge) {
+            try {
+                const raw = readFileSync(options.merge, 'utf-8');
+                mergeData = JSON.parse(raw);
+                let mergeResults = mergeData;
+                if (mergeData.status === 'ok' && mergeData.data) {
+                    mergeResults = mergeData.data;
+                }
+                const historicalResults = (mergeResults.results || []);
+                if (historicalResults.length > 0) {
+                    rankings.push(historicalResults.map((r) => ({ id: normalizeUrl(r.url), ...r })));
+                }
+            }
+            catch (error) {
+                const envelope = createErrorEnvelope('MERGE_FILE_FAILED', `Failed to read merge file: ${options.merge}`, { hint: 'Ensure the file exists and contains valid search results JSON' });
+                console.log(JSON.stringify(envelope, null, 2));
+                return 1;
+            }
+        }
+        // RRF fusion
+        const rrfFused = rrf(rankings);
+        // Map RRF scores back to original results
+        const allResults = [];
+        const allSuggestions = [];
+        const allAnswers = [];
+        const allUnresponsive = [];
+        for (const resp of allResponses) {
+            allResults.push(...resp.results);
+            for (const s of resp.suggestions) {
+                if (!allSuggestions.includes(s))
+                    allSuggestions.push(s);
+            }
+            for (const a of resp.answers) {
+                if (!allAnswers.includes(a))
+                    allAnswers.push(a);
+            }
+            for (const u of resp.unresponsiveEngines) {
+                const name = Array.isArray(u) ? u[0] : u;
+                if (!allUnresponsive.includes(name))
+                    allUnresponsive.push(name);
+            }
+        }
+        // Include historical results in the URL map if merging
+        if (mergeData) {
+            let mergeResults = mergeData;
+            if (mergeData.status === 'ok' && mergeData.data) {
+                mergeResults = mergeData.data;
+            }
+            const historical = mergeResults.results || [];
+            allResults.push(...historical);
+            if (mergeResults.suggestions) {
+                for (const s of mergeResults.suggestions) {
+                    if (!allSuggestions.includes(s))
+                        allSuggestions.push(s);
+                }
+            }
+            if (mergeResults.answers) {
+                for (const a of mergeResults.answers) {
+                    if (!allAnswers.includes(a))
+                        allAnswers.push(a);
+                }
+            }
+        }
+        // Dedup by URL to get unique results, then apply RRF order
+        const urlMap = new Map();
+        for (const r of allResults) {
+            const norm = normalizeUrl(r.url);
+            if (!urlMap.has(norm))
+                urlMap.set(norm, r);
+        }
+        let fusedResults = rrfFused
+            .map((item) => {
+            const original = urlMap.get(item.id);
+            if (!original)
+                return null;
+            return { ...original, score: item.score };
+        })
+            .filter(Boolean);
+        if (limit > 0) {
+            fusedResults = fusedResults.slice(0, limit);
+        }
+        // Session: accumulate results and update graph
+        let sessionInfo = null;
+        if (options.session) {
+            sessionInfo = appendSessionResults(options.session, fusedResults);
+            for (const query of queries) {
+                updateSessionGraph(options.session, query, fusedResults.map(r => ({
+                    url: r.url,
+                    title: r.title,
+                })));
+            }
+        }
+        const displayQuery = queries.length > 1 ? queries.join(' · ') : queries[0];
+        const outputFormat = options.format || config.defaultFormat;
+        const displayData = {
+            query: displayQuery,
+            queries,
+            totalResults: allResults.length,
+            results: fusedResults,
+            answers: allAnswers,
+            suggestions: allSuggestions,
+            unresponsiveEngines: allUnresponsive,
+            ...(sessionInfo ? { session: { dir: options.session, added: sessionInfo.added, total: sessionInfo.total } } : {}),
+        };
+        if (outputFormat === 'md') {
+            console.log(formatOutput(displayData, 'md'));
         }
         else if (outputFormat === 'json') {
-            console.log(JSON.stringify(envelope, null, 2));
+            console.log(JSON.stringify(createSuccessEnvelope({
+                ...displayData,
+                returnedResults: fusedResults.length,
+            }), null, 2));
         }
         else {
-            // csv or html
-            console.log(formatOutput(results, outputFormat));
+            console.log(formatOutput(displayData, outputFormat));
+        }
+        // Auto-insert result metadata into knowledge graph if --graph specified
+        if (options.graph) {
+            addToGraph(options.graph, fusedResults);
         }
         return 0;
     }