local-deep-research 0.4.4__py3-none-any.whl → 0.5.2__py3-none-any.whl

local_deep_research/utilities/url_utils.py

@@ -49,6 +49,12 @@ def normalize_url(raw_url: str) -> str:
     # At this point, we should have hostname:port or just hostname
     # Determine if this is localhost or an external host
     hostname = raw_url.split(":")[0].split("/")[0]
+
+    # Handle IPv6 addresses in brackets
+    if hostname.startswith("[") and "]" in raw_url:
+        # Extract the IPv6 address including brackets
+        hostname = raw_url.split("]")[0] + "]"
+
     is_localhost = hostname in ("localhost", "127.0.0.1", "[::1]", "0.0.0.0")
 
     # Use http for localhost, https for external hosts

local_deep_research/web/api.py

@@ -0,0 +1,347 @@
+"""
+REST API for Local Deep Research.
+Provides HTTP access to programmatic search and research capabilities.
+"""
+
+import logging
+import time
+from functools import wraps
+
+from flask import Blueprint, jsonify, request
+
+from ..api.research_functions import analyze_documents
+from .services.settings_service import get_setting
+
+# Create a blueprint for the API
+api_blueprint = Blueprint("api_v1", __name__, url_prefix="/api/v1")
+logger = logging.getLogger(__name__)
+
+# Rate limiting data store: {ip_address: [timestamp1, timestamp2, ...]}
+rate_limit_data = {}
+
+
+def api_access_control(f):
+    """
+    Decorator to enforce API access control:
+    - Check if API is enabled
+    - Enforce rate limiting
+    """
+
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        # Check if API is enabled
+        api_enabled = get_setting("app.enable_api", True)  # Default to enabled
+        if not api_enabled:
+            return jsonify({"error": "API access is disabled"}), 403
+
+        # Implement rate limiting
+        rate_limit = get_setting(
+            "app.api_rate_limit", 60
+        )  # Default 60 requests per minute
+        if rate_limit:
+            client_ip = request.remote_addr
+            current_time = time.time()
+
+            # Initialize or clean up old requests for this IP
+            if client_ip not in rate_limit_data:
+                rate_limit_data[client_ip] = []
+
+            # Remove timestamps older than 1 minute
+            rate_limit_data[client_ip] = [
+                ts
+                for ts in rate_limit_data[client_ip]
+                if current_time - ts < 60
+            ]
+
+            # Check if rate limit is exceeded
+            if len(rate_limit_data[client_ip]) >= rate_limit:
+                return (
+                    jsonify(
+                        {
+                            "error": f"Rate limit exceeded. Maximum {rate_limit} requests per minute allowed."
+                        }
+                    ),
+                    429,
+                )
+
+            # Add current timestamp to the list
+            rate_limit_data[client_ip].append(current_time)
+
+        return f(*args, **kwargs)
+
+    return decorated_function
+
+
+@api_blueprint.route("/", methods=["GET"])
+@api_access_control
+def api_documentation():
+    """
+    Provide documentation on the available API endpoints.
+    """
+    api_docs = {
+        "api_version": "v1",
+        "description": "REST API for Local Deep Research",
+        "endpoints": [
+            {
+                "path": "/api/v1/quick_summary",
+                "method": "POST",
+                "description": "Generate a quick research summary",
+                "parameters": {
+                    "query": "Research query (required)",
+                    "search_tool": "Search engine to use (optional)",
+                    "iterations": "Number of search iterations (optional)",
+                    "temperature": "LLM temperature (optional)",
+                },
+            },
+            {
+                "path": "/api/v1/generate_report",
+                "method": "POST",
+                "description": "Generate a comprehensive research report",
+                "parameters": {
+                    "query": "Research query (required)",
+                    "output_file": "Path to save report (optional)",
+                    "searches_per_section": "Searches per report section (optional)",
+                    "model_name": "LLM model to use (optional)",
+                    "temperature": "LLM temperature (optional)",
+                },
+            },
+            {
+                "path": "/api/v1/analyze_documents",
+                "method": "POST",
+                "description": "Search and analyze documents in a local collection",
+                "parameters": {
+                    "query": "Search query (required)",
+                    "collection_name": "Local collection name (required)",
+                    "max_results": "Maximum results to return (optional)",
+                    "temperature": "LLM temperature (optional)",
+                    "force_reindex": "Force collection reindexing (optional)",
+                },
+            },
+        ],
+    }
+
+    return jsonify(api_docs)
+
+
+@api_blueprint.route("/health", methods=["GET"])
+def health_check():
+    """Simple health check endpoint."""
+    return jsonify(
+        {"status": "ok", "message": "API is running", "timestamp": time.time()}
+    )
+
+
+@api_blueprint.route("/quick_summary_test", methods=["POST"])
+@api_access_control
+def api_quick_summary_test():
+    """Test endpoint using programmatic access with minimal parameters for fast testing."""
+    data = request.json
+    if not data or "query" not in data:
+        return jsonify({"error": "Query parameter is required"}), 400
+
+    query = data.get("query")
+
+    try:
+        # Import here to avoid circular imports
+        from ..api.research_functions import quick_summary
+
+        logger.info(f"Processing quick_summary_test request: query='{query}'")
+
+        # Use minimal parameters for faster testing
+        result = quick_summary(
+            query=query,
+            search_tool="wikipedia",  # Use fast Wikipedia search for testing
+            iterations=1,  # Single iteration for speed
+            temperature=0.7,
+        )
+
+        return jsonify(result)
+    except Exception as e:
+        logger.error(
+            f"Error in quick_summary_test API: {str(e)}", exc_info=True
+        )
+        return (
+            jsonify(
+                {
+                    "error": "An internal error has occurred. Please try again later."
+                }
+            ),
+            500,
+        )
+
+
+@api_blueprint.route("/quick_summary", methods=["POST"])
+@api_access_control
+def api_quick_summary():
+    """
+    Generate a quick research summary via REST API.
+
+    POST /api/v1/quick_summary
+    {
+        "query": "Advances in fusion energy research",
+        "search_tool": "auto",  # Optional: search engine to use
+        "iterations": 2,        # Optional: number of search iterations
+        "temperature": 0.7      # Optional: LLM temperature
+    }
+    """
+    data = request.json
+    if not data or "query" not in data:
+        return jsonify({"error": "Query parameter is required"}), 400
+
+    # Extract query and optional parameters
+    query = data.get("query")
+    params = {k: v for k, v in data.items() if k != "query"}
+
+    try:
+        # Import here to avoid circular imports
+        from ..api.research_functions import quick_summary
+
+        logger.info(f"Processing quick_summary request: query='{query}'")
+
+        # Set reasonable defaults for API use
+        params.setdefault("temperature", 0.7)
+        params.setdefault("search_tool", "auto")
+        params.setdefault("iterations", 1)
+
+        # Call the actual research function
+        result = quick_summary(query, **params)
+
+        return jsonify(result)
+    except TimeoutError:
+        logger.error("Request timed out")
+        return (
+            jsonify(
+                {
+                    "error": "Request timed out. Please try with a simpler query or fewer iterations."
+                }
+            ),
+            504,
+        )
+    except Exception as e:
+        logger.error(f"Error in quick_summary API: {str(e)}", exc_info=True)
+        return (
+            jsonify(
+                {
+                    "error": "An internal error has occurred. Please try again later."
+                }
+            ),
+            500,
+        )
+
+
+@api_blueprint.route("/generate_report", methods=["POST"])
+@api_access_control
+def api_generate_report():
+    """
+    Generate a comprehensive research report via REST API.
+
+    POST /api/v1/generate_report
+    {
+        "query": "Impact of climate change on agriculture",
+        "output_file": "/path/to/save/report.md",  # Optional
+        "searches_per_section": 2,                 # Optional
+        "model_name": "gpt-4",                     # Optional
+        "temperature": 0.5                         # Optional
+    }
+    """
+    data = request.json
+    if not data or "query" not in data:
+        return jsonify({"error": "Query parameter is required"}), 400
+
+    query = data.get("query")
+    params = {k: v for k, v in data.items() if k != "query"}
+
+    try:
+        # Import here to avoid circular imports
+        from ..api.research_functions import generate_report
+
+        # Set reasonable defaults for API use
+        params.setdefault("searches_per_section", 1)
+        params.setdefault("temperature", 0.7)
+
+        logger.info(
+            f"Processing generate_report request: query='{query}', params={params}"
+        )
+
+        result = generate_report(query, **params)
+
+        # Don't return the full content for large reports
+        if (
+            result
+            and "content" in result
+            and isinstance(result["content"], str)
+            and len(result["content"]) > 10000
+        ):
+            # Include a summary of the report content
+            content_preview = (
+                result["content"][:2000] + "... [Content truncated]"
+            )
+            result["content"] = content_preview
+            result["content_truncated"] = True
+
+        return jsonify(result)
+    except TimeoutError:
+        logger.error("Request timed out")
+        return (
+            jsonify(
+                {"error": "Request timed out. Please try with a simpler query."}
+            ),
+            504,
+        )
+    except Exception as e:
+        logger.error(f"Error in generate_report API: {str(e)}", exc_info=True)
+        return (
+            jsonify(
+                {
+                    "error": "An internal error has occurred. Please try again later."
+                }
+            ),
+            500,
+        )
+
+
+@api_blueprint.route("/analyze_documents", methods=["POST"])
+@api_access_control
+def api_analyze_documents():
+    """
+    Search and analyze documents in a local collection via REST API.
+
+    POST /api/v1/analyze_documents
+    {
+        "query": "neural networks in medicine",
+        "collection_name": "research_papers",      # Required: local collection name
+        "max_results": 20,                         # Optional: max results to return
+        "temperature": 0.7,                        # Optional: LLM temperature
+        "force_reindex": false                     # Optional: force reindexing
+    }
+    """
+    data = request.json
+    if not data or "query" not in data or "collection_name" not in data:
+        return (
+            jsonify(
+                {
+                    "error": "Both query and collection_name parameters are required"
+                }
+            ),
+            400,
+        )
+
+    query = data.get("query")
+    collection_name = data.get("collection_name")
+    params = {
+        k: v for k, v in data.items() if k not in ["query", "collection_name"]
+    }
+
+    try:
+        result = analyze_documents(query, collection_name, **params)
+        return jsonify(result)
+    except Exception as e:
+        logger.error(f"Error in analyze_documents API: {str(e)}", exc_info=True)
+        return (
+            jsonify(
+                {
+                    "error": "An internal error has occurred. Please try again later."
+                }
+            ),
+            500,
+        )

local_deep_research/web/app.py

@@ -5,6 +5,7 @@ from loguru import logger
 
 from ..setup_data_dir import setup_data_dir
 from ..utilities.db_utils import get_db_setting
+from ..utilities.log_utils import config_logger
 from .app_factory import create_app
 from .models.database import (
     DB_PATH,
@@ -15,6 +16,9 @@ from .models.database import (
 # Ensure data directory exists
 setup_data_dir()
 
+# Configure logging with milestone level
+config_logger("ldr_web")
+
 # Run schema upgrades if database exists
 if os.path.exists(DB_PATH):
     try:
@@ -32,9 +36,9 @@ def check_migration_needed():
     """Check if database migration is needed, based on presence of legacy files and absence of new DB"""
     if not os.path.exists(DB_PATH):
         # The new database doesn't exist, check if legacy databases exist
-        legacy_files_exist = os.path.exists(LEGACY_DEEP_RESEARCH_DB) or os.path.exists(
-            LEGACY_RESEARCH_HISTORY_DB
-        )
+        legacy_files_exist = os.path.exists(
+            LEGACY_DEEP_RESEARCH_DB
+        ) or os.path.exists(LEGACY_RESEARCH_HISTORY_DB)
 
         if legacy_files_exist:
             logger.info(
@@ -45,16 +49,15 @@ def check_migration_needed():
     return False
 
 
-# Create the Flask app and SocketIO instance
-app, socketio = create_app()
-
-
-@logger.catch()
+@logger.catch
 def main():
     """
     Entry point for the web application when run as a command.
     This function is needed for the package's entry point to work properly.
     """
+    # Create the Flask app and SocketIO instance
+    app, socketio = create_app()
+
     # Check if migration is needed
     if check_migration_needed():
         logger.info(
@@ -93,15 +96,8 @@ def main():
     host = get_db_setting("web.host", "0.0.0.0")
     debug = get_db_setting("web.debug", True)
 
-    logger.info(f"Starting web server on {host}:{port} (debug: {debug})")
-    socketio.run(
-        app,
-        debug=debug,
-        host=host,
-        port=port,
-        allow_unsafe_werkzeug=True,
-        use_reloader=False,
-    )
+    with app.app_context():
+        socketio.run(host, port, debug=debug)
 
 
 if __name__ == "__main__":

local_deep_research/web/app_factory.py

@@ -6,17 +6,15 @@ from flask import (
     Flask,
     jsonify,
     make_response,
-    redirect,
     request,
     send_from_directory,
-    url_for,
 )
-from flask_socketio import SocketIO
 from flask_wtf.csrf import CSRFProtect
 from loguru import logger
 
 from ..utilities.log_utils import InterceptHandler
 from .models.database import DB_PATH, init_db
+from .services.socket_service import SocketIOService
 
 
 def create_app():
@@ -38,7 +36,9 @@ def create_app():
             TEMPLATE_DIR = (package_dir / "templates").as_posix()
 
         # Initialize Flask app with package directories
-        app = Flask(__name__, static_folder=STATIC_DIR, template_folder=TEMPLATE_DIR)
+        app = Flask(
+            __name__, static_folder=STATIC_DIR, template_folder=TEMPLATE_DIR
+        )
         logger.debug(f"Using package static path: {STATIC_DIR}")
         logger.debug(f"Using package template path: {TEMPLATE_DIR}")
     except Exception:
@@ -58,6 +58,14 @@ def create_app():
     # Exempt Socket.IO from CSRF protection
     csrf.exempt("research.socket_io")
 
+    # Disable CSRF for API routes
+    @app.before_request
+    def disable_csrf_for_api():
+        if request.path.startswith("/api/v1/") or request.path.startswith(
+            "/research/api/"
+        ):
+            csrf.protect = lambda: None
+
     # Database configuration - Use unified ldr.db from the database module
     db_path = DB_PATH
     app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{db_path}"
@@ -65,26 +73,12 @@ def create_app():
     app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
     app.config["SQLALCHEMY_ECHO"] = False
 
-    # Initialize extensions
-    socketio = SocketIO(
-        app,
-        cors_allowed_origins="*",
-        async_mode="threading",
-        path="/research/socket.io",
-        logger=False,
-        engineio_logger=False,
-        ping_timeout=20,
-        ping_interval=5,
-    )
-
     # Initialize the database
     create_database(app)
     init_db()
 
     # Register socket service
-    from .services.socket_service import set_socketio
-
-    set_socketio(socketio)
+    socket_service = SocketIOService(app=app)
 
     # Apply middleware
     apply_middleware(app)
@@ -95,10 +89,7 @@ def create_app():
     # Register error handlers
     register_error_handlers(app)
 
-    # Register socket event handlers
-    register_socket_events(socketio)
-
-    return app, socketio
+    return app, socket_service
 
 
 def apply_middleware(app):
@@ -123,12 +114,18 @@ def apply_middleware(app):
         response.headers["X-Content-Security-Policy"] = csp
 
         # Add CORS headers for API requests
-        if request.path.startswith("/api/"):
+        if request.path.startswith("/api/") or request.path.startswith(
+            "/research/api/"
+        ):
             response.headers["Access-Control-Allow-Origin"] = "*"
             response.headers["Access-Control-Allow-Methods"] = (
-                "GET, POST, DELETE, OPTIONS"
+                "GET, POST, PUT, DELETE, OPTIONS"
+            )
+            response.headers["Access-Control-Allow-Headers"] = (
+                "Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override"
             )
-            response.headers["Access-Control-Allow-Headers"] = "Content-Type"
+            response.headers["Access-Control-Allow-Credentials"] = "true"
+            response.headers["Access-Control-Max-Age"] = "3600"
 
         return response
 
@@ -144,28 +141,69 @@ def apply_middleware(app):
                 # Return empty response to prevent further processing
                 return "", 200
 
+    # Handle CORS preflight requests
+    @app.before_request
+    def handle_preflight():
+        if request.method == "OPTIONS":
+            if request.path.startswith("/api/") or request.path.startswith(
+                "/research/api/"
+            ):
+                response = app.make_default_options_response()
+                response.headers["Access-Control-Allow-Origin"] = "*"
+                response.headers["Access-Control-Allow-Methods"] = (
+                    "GET, POST, PUT, DELETE, OPTIONS"
+                )
+                response.headers["Access-Control-Allow-Headers"] = (
+                    "Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override"
+                )
+                response.headers["Access-Control-Allow-Credentials"] = "true"
+                response.headers["Access-Control-Max-Age"] = "3600"
+                return response
+
 
 def register_blueprints(app):
     """Register blueprints with the Flask app."""
 
     # Import blueprints
+    from .api import api_blueprint  # Import the API blueprint
     from .routes.api_routes import api_bp  # Import the API blueprint
     from .routes.history_routes import history_bp
+    from .routes.metrics_routes import metrics_bp
     from .routes.research_routes import research_bp
     from .routes.settings_routes import settings_bp
 
+    # Add root route
+    @app.route("/")
+    def index():
+        """Root route - redirect to research page"""
+        from flask import redirect, url_for
+
+        return redirect(url_for("research.index"))
+
     # Register blueprints
     app.register_blueprint(research_bp)
     app.register_blueprint(history_bp, url_prefix="/research/api")
+    app.register_blueprint(metrics_bp)
     app.register_blueprint(settings_bp)
     app.register_blueprint(
         api_bp, url_prefix="/research/api"
     )  # Register API blueprint with prefix
 
-    # Add root route redirect
-    @app.route("/")
-    def root_index():
-        return redirect(url_for("research.index"))
+    # Register API v1 blueprint
+    app.register_blueprint(api_blueprint)  # Already has url_prefix='/api/v1'
+
+    # After registration, update CSRF exemptions
+    if hasattr(app, "extensions") and "csrf" in app.extensions:
+        csrf = app.extensions["csrf"]
+        # Exempt the API blueprint routes by actual endpoints
+        csrf.exempt("api_v1")
+        csrf.exempt("api")
+        for rule in app.url_map.iter_rules():
+            if rule.endpoint and (
+                rule.endpoint.startswith("api_v1.")
+                or rule.endpoint.startswith("api.")
+            ):
+                csrf.exempt(rule.endpoint)
 
     # Add favicon route
     @app.route("/favicon.ico")
@@ -192,41 +230,6 @@ def register_error_handlers(app):
         return make_response(jsonify({"error": "Server error"}), 500)
 
 
-def register_socket_events(socketio):
-    """Register Socket.IO event handlers."""
-
-    from .routes.research_routes import get_globals
-    from .services.socket_service import (
-        handle_connect,
-        handle_default_error,
-        handle_disconnect,
-        handle_socket_error,
-        handle_subscribe,
-    )
-
-    @socketio.on("connect")
-    def on_connect():
-        handle_connect(request)
-
-    @socketio.on("disconnect")
-    def on_disconnect():
-        handle_disconnect(request)
-
-    @socketio.on("subscribe_to_research")
-    def on_subscribe(data):
-        globals_dict = get_globals()
-        active_research = globals_dict.get("active_research", {})
-        handle_subscribe(data, request, active_research)
-
-    @socketio.on_error
-    def on_error(e):
-        return handle_socket_error(e)
-
-    @socketio.on_error_default
-    def on_default_error(e):
-        return handle_default_error(e)
-
-
 def create_database(app):
     """
     Create the database and tables for the application.
@@ -250,7 +253,9 @@ def create_database(app):
     Base.metadata.create_all(engine)
 
     # Configure session factory
-    session_factory = sessionmaker(bind=engine, autocommit=False, autoflush=False)
+    session_factory = sessionmaker(
+        bind=engine, autocommit=False, autoflush=False
+    )
     app.db_session = scoped_session(session_factory)
 
     # Run migrations and setup predefined settings