local-deep-research 0.5.9__py3-none-any.whl → 0.6.1__py3-none-any.whl

local_deep_research/web_search_engines/rate_limiting/cli.py

@@ -0,0 +1,209 @@
+"""
+Command-line interface for rate limiting monitoring and management.
+"""
+
+import argparse
+import sys
+from datetime import datetime
+
+from .tracker import get_tracker
+
+
+def format_stats_table(stats: list) -> str:
+    """Format rate limit statistics as a table."""
+    if not stats:
+        return "No rate limit data available."
+
+    # Header
+    lines = []
+    lines.append("Rate Limit Statistics:")
+    lines.append("-" * 80)
+    lines.append(
+        f"{'Engine':<20} {'Base Wait':<12} {'Range':<20} {'Success':<10} {'Attempts':<10} {'Updated':<15}"
+    )
+    lines.append("-" * 80)
+
+    # Data rows
+    for row in stats:
+        (
+            engine_type,
+            base_wait,
+            min_wait,
+            max_wait,
+            last_updated,
+            attempts,
+            success_rate,
+        ) = row
+
+        # Format time
+        updated_time = datetime.fromtimestamp(last_updated).strftime(
+            "%m-%d %H:%M"
+        )
+
+        # Format range
+        range_str = f"{min_wait:.1f}s - {max_wait:.1f}s"
+
+        lines.append(
+            f"{engine_type:<20} {base_wait:<12.2f} {range_str:<20} "
+            f"{success_rate:<10.1%} {attempts:<10} {updated_time:<15}"
+        )
+
+    return "\n".join(lines)
+
+
+def cmd_status(args):
+    """Show current rate limit status."""
+    tracker = get_tracker()
+
+    if args.engine:
+        stats = tracker.get_stats(args.engine)
+        if not stats:
+            print(f"No rate limit data found for engine: {args.engine}")
+            return
+    else:
+        stats = tracker.get_stats()
+
+    print(format_stats_table(stats))
+
+
+def cmd_reset(args):
+    """Reset rate limit data for an engine."""
+    tracker = get_tracker()
+
+    if args.engine:
+        tracker.reset_engine(args.engine)
+        print(f"Reset rate limit data for {args.engine}")
+    else:
+        print("Error: --engine parameter is required for reset command")
+        sys.exit(1)
+
+
+def cmd_cleanup(args):
+    """Clean up old rate limit data."""
+    tracker = get_tracker()
+    tracker.cleanup_old_data(days=args.days)
+    print(f"Cleaned up rate limit data older than {args.days} days")
+
+
+def cmd_export(args):
+    """Export rate limit data."""
+    tracker = get_tracker()
+
+    # Get all stats
+    stats = tracker.get_stats()
+
+    if args.format == "csv":
+        print(
+            "engine_type,base_wait_seconds,min_wait_seconds,max_wait_seconds,last_updated,total_attempts,success_rate"
+        )
+        for row in stats:
+            print(",".join(map(str, row)))
+    elif args.format == "json":
+        import json
+
+        data = []
+        for row in stats:
+            (
+                engine_type,
+                base_wait,
+                min_wait,
+                max_wait,
+                last_updated,
+                attempts,
+                success_rate,
+            ) = row
+            data.append(
+                {
+                    "engine_type": engine_type,
+                    "base_wait_seconds": base_wait,
+                    "min_wait_seconds": min_wait,
+                    "max_wait_seconds": max_wait,
+                    "last_updated": last_updated,
+                    "total_attempts": attempts,
+                    "success_rate": success_rate,
+                }
+            )
+        print(json.dumps(data, indent=2))
+    else:
+        print(format_stats_table(stats))
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Rate limiting monitoring and management",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Show all engine statistics
+  python -m local_deep_research.web_search_engines.rate_limiting.cli status
+
+  # Show specific engine statistics
+  python -m local_deep_research.web_search_engines.rate_limiting.cli status --engine DuckDuckGoSearchEngine
+
+  # Reset rate limit data for an engine
+  python -m local_deep_research.web_search_engines.rate_limiting.cli reset --engine SearXNGSearchEngine
+
+  # Export data as CSV
+  python -m local_deep_research.web_search_engines.rate_limiting.cli export --format csv
+
+  # Clean up old data
+  python -m local_deep_research.web_search_engines.rate_limiting.cli cleanup --days 30
+""",
+    )
+
+    subparsers = parser.add_subparsers(
+        dest="command", help="Available commands"
+    )
+
+    # Status command
+    status_parser = subparsers.add_parser(
+        "status", help="Show rate limit statistics"
+    )
+    status_parser.add_argument(
+        "--engine", help="Show stats for specific engine"
+    )
+    status_parser.set_defaults(func=cmd_status)
+
+    # Reset command
+    reset_parser = subparsers.add_parser("reset", help="Reset rate limit data")
+    reset_parser.add_argument("--engine", required=True, help="Engine to reset")
+    reset_parser.set_defaults(func=cmd_reset)
+
+    # Export command
+    export_parser = subparsers.add_parser(
+        "export", help="Export rate limit data"
+    )
+    export_parser.add_argument(
+        "--format",
+        choices=["table", "csv", "json"],
+        default="table",
+        help="Output format",
+    )
+    export_parser.set_defaults(func=cmd_export)
+
+    # Cleanup command
+    cleanup_parser = subparsers.add_parser("cleanup", help="Clean up old data")
+    cleanup_parser.add_argument(
+        "--days",
+        type=int,
+        default=30,
+        help="Remove data older than this many days",
+    )
+    cleanup_parser.set_defaults(func=cmd_cleanup)
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        args.func(args)
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

local_deep_research/web_search_engines/rate_limiting/exceptions.py

@@ -0,0 +1,21 @@
+"""
+Rate limiting specific exceptions for search engines.
+"""
+
+
+class RateLimitError(Exception):
+    """Raised when a search engine hits rate limits."""
+
+    pass
+
+
+class AdaptiveRetryError(Exception):
+    """Raised when adaptive retry fails after all attempts."""
+
+    pass
+
+
+class RateLimitConfigError(Exception):
+    """Raised when there's an issue with rate limit configuration."""
+
+    pass