local-deep-research 0.3.12__py3-none-any.whl → 0.4.0__py3-none-any.whl

local_deep_research/benchmarks/cli.py

@@ -0,0 +1,347 @@
+"""
+Command-line interface for benchmarking functionality.
+
+This module provides a command-line interface for running parameter
+optimization, comparison, and benchmarking tasks.
+"""
+
+import argparse
+import logging
+import os
+import sys
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from .comparison import compare_configurations
+from .efficiency import ResourceMonitor, SpeedProfiler
+from .optimization import optimize_parameters
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Local Deep Research Benchmarking Tools",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run parameter optimization
+  python -m local_deep_research.benchmarks.cli optimize "What are the latest advancements in quantum computing?"
+
+  # Compare different configurations
+  python -m local_deep_research.benchmarks.cli compare "What are the effects of climate change?" --configs configs.json
+
+  # Run efficiency profiling
+  python -m local_deep_research.benchmarks.cli profile "How do neural networks work?"
+""",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Command to run")
+
+    # Optimizer parser
+    optimize_parser = subparsers.add_parser("optimize", help="Optimize parameters")
+    optimize_parser.add_argument("query", help="Research query to optimize for")
+    optimize_parser.add_argument(
+        "--output-dir",
+        default="data/optimization_results",
+        help="Directory to save results",
+    )
+    optimize_parser.add_argument("--model", help="Model name for the LLM")
+    optimize_parser.add_argument("--provider", help="Provider for the LLM")
+    optimize_parser.add_argument("--search-tool", help="Search tool to use")
+    optimize_parser.add_argument(
+        "--temperature", type=float, default=0.7, help="LLM temperature"
+    )
+    optimize_parser.add_argument(
+        "--n-trials",
+        type=int,
+        default=30,
+        help="Number of parameter combinations to try",
+    )
+    optimize_parser.add_argument(
+        "--timeout", type=int, help="Maximum seconds to run optimization"
+    )
+    optimize_parser.add_argument(
+        "--n-jobs", type=int, default=1, help="Number of parallel jobs for optimization"
+    )
+    optimize_parser.add_argument("--study-name", help="Name of the Optuna study")
+    optimize_parser.add_argument(
+        "--speed-focus", action="store_true", help="Focus optimization on speed"
+    )
+    optimize_parser.add_argument(
+        "--quality-focus", action="store_true", help="Focus optimization on quality"
+    )
+
+    # Comparison parser
+    compare_parser = subparsers.add_parser("compare", help="Compare configurations")
+    compare_parser.add_argument("query", help="Research query to compare with")
+    compare_parser.add_argument(
+        "--configs", required=True, help="JSON file with configurations to compare"
+    )
+    compare_parser.add_argument(
+        "--output-dir",
+        default="data/benchmark_results/comparison",
+        help="Directory to save results",
+    )
+    compare_parser.add_argument("--model", help="Model name for the LLM")
+    compare_parser.add_argument("--provider", help="Provider for the LLM")
+    compare_parser.add_argument("--search-tool", help="Search tool to use")
+    compare_parser.add_argument(
+        "--repetitions",
+        type=int,
+        default=1,
+        help="Number of repetitions for each configuration",
+    )
+
+    # Profiling parser
+    profile_parser = subparsers.add_parser("profile", help="Profile resource usage")
+    profile_parser.add_argument("query", help="Research query to profile")
+    profile_parser.add_argument(
+        "--output-dir",
+        default="data/benchmark_results/profiling",
+        help="Directory to save results",
+    )
+    profile_parser.add_argument("--model", help="Model name for the LLM")
+    profile_parser.add_argument("--provider", help="Provider for the LLM")
+    profile_parser.add_argument("--search-tool", help="Search tool to use")
+    profile_parser.add_argument(
+        "--iterations", type=int, default=2, help="Number of search iterations"
+    )
+    profile_parser.add_argument(
+        "--questions", type=int, default=2, help="Questions per iteration"
+    )
+    profile_parser.add_argument(
+        "--strategy", default="iterdrag", help="Search strategy to use"
+    )
+
+    return parser.parse_args()
+
+
+def run_optimization(args):
+    """Run parameter optimization."""
+    logger.info(f"Starting parameter optimization for query: {args.query}")
+
+    # Determine metric weights based on focus
+    metric_weights = None
+    if args.speed_focus:
+        metric_weights = {"speed": 0.8, "quality": 0.2}
+    elif args.quality_focus:
+        metric_weights = {"quality": 0.8, "speed": 0.2}
+
+    # Run optimization
+    best_params, best_score = optimize_parameters(
+        query=args.query,
+        output_dir=args.output_dir,
+        model_name=args.model,
+        provider=args.provider,
+        search_tool=args.search_tool,
+        temperature=args.temperature,
+        n_trials=args.n_trials,
+        timeout=args.timeout,
+        n_jobs=args.n_jobs,
+        study_name=args.study_name,
+        metric_weights=metric_weights,
+    )
+
+    # Print results
+    print("\nOptimization Results:")
+    print("====================")
+    print(f"Best Parameters: {best_params}")
+    print(f"Best Score: {best_score:.4f}")
+    print(f"Results saved to: {args.output_dir}")
+
+    return 0
+
+
+def run_comparison(args):
+    """Run configuration comparison."""
+    import json
+
+    logger.info(f"Comparing configurations for query: {args.query}")
+
+    # Load configurations from file
+    try:
+        with open(args.configs, "r") as f:
+            configurations = json.load(f)
+
+        if not isinstance(configurations, list):
+            logger.error("Configurations file must contain a JSON array")
+            return 1
+
+        if not configurations:
+            logger.error("No configurations found in the file")
+            return 1
+    except Exception as e:
+        logger.error(f"Error loading configurations file: {str(e)}")
+        return 1
+
+    # Run comparison
+    results = compare_configurations(
+        query=args.query,
+        configurations=configurations,
+        output_dir=args.output_dir,
+        model_name=args.model,
+        provider=args.provider,
+        search_tool=args.search_tool,
+        repetitions=args.repetitions,
+    )
+
+    # Print summary
+    print("\nComparison Results:")
+    print("==================")
+    print(f"Configurations tested: {results['configurations_tested']}")
+    print(f"Successful configurations: {results['successful_configurations']}")
+    print(f"Failed configurations: {results['failed_configurations']}")
+
+    # Print ranking
+    print("\nRanking by Overall Score:")
+    for i, result in enumerate(
+        [r for r in results["results"] if r.get("success", False)]
+    ):
+        print(f"{i+1}. {result['name']}: {result.get('overall_score', 0):.4f}")
+
+    print(f"\nResults saved to: {results.get('report_path', args.output_dir)}")
+
+    return 0
+
+
+def run_profiling(args):
+    """Run resource profiling."""
+    import json
+
+    from local_deep_research.config.llm_config import get_llm
+    from local_deep_research.config.search_config import get_search
+    from local_deep_research.search_system import AdvancedSearchSystem
+
+    logger.info(f"Profiling resource usage for query: {args.query}")
+
+    # Create output directory
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # Initialize profiling tools
+    speed_profiler = SpeedProfiler()
+    resource_monitor = ResourceMonitor(sampling_interval=0.5)
+
+    # Start profiling
+    speed_profiler.start()
+    resource_monitor.start()
+
+    try:
+        # Initialize system
+        with speed_profiler.timer("initialization"):
+            # Get LLM
+            llm = get_llm(model_name=args.model, provider=args.provider)
+
+            # Get search engine
+            search = None
+            if args.search_tool:
+                search = get_search(args.search_tool, llm_instance=llm)
+
+            # Create search system
+            system = AdvancedSearchSystem(llm=llm, search=search)
+            system.max_iterations = args.iterations
+            system.questions_per_iteration = args.questions
+            system.strategy_name = args.strategy
+
+        # Run analysis
+        with speed_profiler.timer("analysis"):
+            results = system.analyze_topic(args.query)
+
+        # Stop profiling
+        speed_profiler.stop()
+        resource_monitor.stop()
+
+        # Get profiling results
+        timing_results = speed_profiler.get_summary()
+        resource_results = resource_monitor.get_combined_stats()
+
+        # Print summary
+        print("\nProfiling Results:")
+        print("=================")
+
+        # Timing summary
+        print("\nTiming Summary:")
+        total_duration = timing_results.get("total_duration", 0)
+        print(f"Total execution time: {total_duration:.2f} seconds")
+
+        # Component breakdown
+        print("\nComponent Breakdown:")
+        for name, value in timing_results.items():
+            if name != "total_duration" and name.endswith("_duration"):
+                component = name.replace("_duration", "")
+                duration = value
+                percent = (duration / total_duration * 100) if total_duration > 0 else 0
+                print(f"- {component}: {duration:.2f}s ({percent:.1f}%)")
+
+        # Resource summary
+        print("\nResource Usage Summary:")
+        print(f"Peak memory: {resource_results.get('process_memory_max_mb', 0):.1f} MB")
+        print(
+            f"Average memory: {resource_results.get('process_memory_avg_mb', 0):.1f} MB"
+        )
+        print(f"Peak CPU: {resource_results.get('process_cpu_max', 0):.1f}%")
+        print(f"Average CPU: {resource_results.get('process_cpu_avg', 0):.1f}%")
+
+        # Save results
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        results_file = os.path.join(
+            args.output_dir, f"profiling_results_{timestamp}.json"
+        )
+
+        with open(results_file, "w") as f:
+            json.dump(
+                {
+                    "query": args.query,
+                    "configuration": {
+                        "model": args.model,
+                        "provider": args.provider,
+                        "search_tool": args.search_tool,
+                        "iterations": args.iterations,
+                        "questions_per_iteration": args.questions,
+                        "strategy": args.strategy,
+                    },
+                    "timing_results": timing_results,
+                    "resource_results": resource_results,
+                    "findings_count": len(results.get("findings", [])),
+                    "knowledge_length": len(results.get("current_knowledge", "")),
+                    "timestamp": timestamp,
+                },
+                f,
+                indent=2,
+            )
+
+        print(f"\nDetailed results saved to: {results_file}")
+
+        return 0
+
+    except Exception as e:
+        # Stop profiling on error
+        speed_profiler.stop()
+        resource_monitor.stop()
+
+        logger.error(f"Error during profiling: {str(e)}")
+        return 1
+
+
+def main():
+    """Main entry point for the CLI."""
+    args = parse_args()
+
+    if args.command == "optimize":
+        return run_optimization(args)
+    elif args.command == "compare":
+        return run_comparison(args)
+    elif args.command == "profile":
+        return run_profiling(args)
+    else:
+        print("Please specify a command. Use --help for more information.")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

local_deep_research/benchmarks/comparison/__init__.py

@@ -0,0 +1,12 @@
+"""
+Comparison submodule for evaluating different configurations of Local Deep Research.
+
+This module provides tools for comparing the performance of different
+parameters, models, and search engines.
+"""
+
+from local_deep_research.benchmarks.comparison.evaluator import compare_configurations
+
+__all__ = [
+    'compare_configurations',
+]