local-deep-research 0.5.9__py3-none-any.whl → 0.6.1__py3-none-any.whl

local_deep_research/web_search_engines/search_engine_base.py

@@ -6,10 +6,28 @@ from typing import Any, Dict, List, Optional
 
 from langchain_core.language_models import BaseLLM
 from loguru import logger
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    retry_if_exception_type,
+    RetryError,
+)
+from tenacity.wait import wait_base
 
 from ..advanced_search_system.filters.base_filter import BaseFilter
 from ..metrics.search_tracker import get_search_tracker
 from ..utilities.db_utils import get_db_setting
+from .rate_limiting import RateLimitError, get_tracker
+
+
+class AdaptiveWait(wait_base):
+    """Custom wait strategy that uses adaptive rate limiting."""
+
+    def __init__(self, get_wait_func):
+        self.get_wait_func = get_wait_func
+
+    def __call__(self, retry_state):
+        return self.get_wait_func()
 
 
 class BaseSearchEngine(ABC):
@@ -59,6 +77,12 @@ class BaseSearchEngine(ABC):
             1, int(max_results)
         )  # Ensure it's a positive integer
 
+        # Rate limiting attributes
+        self.engine_type = self.__class__.__name__
+        self.rate_tracker = get_tracker()
+        self._last_wait_time = None
+        self._last_results_count = 0
+
     @property
     def max_filtered_results(self) -> int:
         """Get the maximum number of filtered results."""
@@ -84,7 +108,32 @@ class BaseSearchEngine(ABC):
             value = 10
         self._max_results = max(1, int(value))
 
-    def run(self, query: str) -> List[Dict[str, Any]]:
+    def _get_adaptive_wait(self) -> float:
+        """Get adaptive wait time from tracker."""
+        wait_time = self.rate_tracker.get_wait_time(self.engine_type)
+        self._last_wait_time = wait_time
+        logger.debug(
+            f"{self.engine_type} waiting {wait_time:.2f}s before retry"
+        )
+        return wait_time
+
+    def _record_retry_outcome(self, retry_state) -> None:
+        """Record outcome after retry completes."""
+        success = (
+            not retry_state.outcome.failed if retry_state.outcome else False
+        )
+        self.rate_tracker.record_outcome(
+            self.engine_type,
+            self._last_wait_time or 0,
+            success,
+            retry_state.attempt_number,
+            error_type="RateLimitError" if not success else None,
+            search_result_count=self._last_results_count if success else 0,
+        )
+
+    def run(
+        self, query: str, research_context: Dict[str, Any] = None
+    ) -> List[Dict[str, Any]]:
         """
         Run the search engine with a given query, retrieving and filtering results.
         This implements a two-phase retrieval approach:
@@ -100,6 +149,19 @@ class BaseSearchEngine(ABC):
         """
         # Track search call for metrics
         tracker = get_search_tracker()
+
+        # For thread-safe context propagation: if we have research_context parameter, use it
+        # Otherwise, try to inherit from current thread context (normal case)
+        # This allows strategies running in threads to explicitly pass context when needed
+        current_context = tracker._get_research_context()
+        if research_context:
+            # Explicit context provided - use it and set it for this thread
+            tracker.set_research_context(research_context)
+        elif not current_context.get("research_id"):
+            # No context in current thread and none provided - try to get from main thread
+            # This handles the case where we're in a worker thread without context
+            pass  # Will use empty context, research_id will be None
+
         engine_name = self.__class__.__name__.replace(
             "SearchEngine", ""
         ).lower()
@@ -109,58 +171,114 @@ class BaseSearchEngine(ABC):
         error_message = None
         results_count = 0
 
-        try:
-            # Step 1: Get preview information for items
-            previews = self._get_previews(query)
-            if not previews:
-                logger.info(
-                    f"Search engine {self.__class__.__name__} returned no preview results for query: {query}"
-                )
-                results_count = 0
-                return []
+        # Define the core search function with retry logic
+        if self.rate_tracker.enabled:
+            # Rate limiting enabled - use retry with adaptive wait
+            @retry(
+                stop=stop_after_attempt(3),
+                wait=AdaptiveWait(lambda: self._get_adaptive_wait()),
+                retry=retry_if_exception_type((RateLimitError,)),
+                after=self._record_retry_outcome,
+                reraise=True,
+            )
+            def _run_with_retry():
+                nonlocal success, error_message, results_count
+                return _execute_search()
+        else:
+            # Rate limiting disabled - run without retry
+            def _run_with_retry():
+                nonlocal success, error_message, results_count
+                return _execute_search()
+
+        def _execute_search():
+            nonlocal success, error_message, results_count
+
+            try:
+                # Step 1: Get preview information for items
+                previews = self._get_previews(query)
+                if not previews:
+                    logger.info(
+                        f"Search engine {self.__class__.__name__} returned no preview results for query: {query}"
+                    )
+                    results_count = 0
+                    return []
 
-            for preview_filter in self._preview_filters:
-                previews = preview_filter.filter_results(previews, query)
+                for preview_filter in self._preview_filters:
+                    previews = preview_filter.filter_results(previews, query)
 
-            # Step 2: Filter previews for relevance with LLM
-            # TEMPORARILY DISABLED: Skip LLM relevance filtering
-            filtered_items = previews
-            logger.info(
-                f"LLM relevance filtering disabled - returning all {len(previews)} previews"
-            )
+                # Step 2: Filter previews for relevance with LLM
+                # TEMPORARILY DISABLED: Skip LLM relevance filtering
+                filtered_items = previews
+                logger.info(
+                    f"LLM relevance filtering disabled - returning all {len(previews)} previews"
+                )
 
-            # # Original filtering code (disabled):
-            # filtered_items = self._filter_for_relevance(previews, query)
-            # if not filtered_items:
-            #     logger.info(
-            #         f"All preview results were filtered out as irrelevant for query: {query}"
-            #     )
-            #     # Do not fall back to previews, return empty list instead
-            #     results_count = 0
-            #     return []
-
-            # Step 3: Get full content for filtered items
-            # Import config inside the method to avoid circular import
-
-            if get_db_setting("search.snippets_only", True):
-                logger.info("Returning snippet-only results as per config")
-                results = filtered_items
-            else:
-                results = self._get_full_content(filtered_items)
+                # Step 3: Get full content for filtered items
+                if get_db_setting("search.snippets_only", True):
+                    logger.info("Returning snippet-only results as per config")
+                    results = filtered_items
+                else:
+                    results = self._get_full_content(filtered_items)
+
+                for content_filter in self._content_filters:
+                    results = content_filter.filter_results(results, query)
+
+                results_count = len(results)
+                self._last_results_count = results_count
+
+                # Record success if we get here and rate limiting is enabled
+                if (
+                    self.rate_tracker.enabled
+                    and self._last_wait_time is not None
+                ):
+                    self.rate_tracker.record_outcome(
+                        self.engine_type,
+                        self._last_wait_time,
+                        success=True,
+                        retry_count=1,  # First attempt succeeded
+                        search_result_count=results_count,
+                    )
 
-            for content_filter in self._content_filters:
-                results = content_filter.filter_results(results, query)
+                return results
 
-            results_count = len(results)
-            return results
+            except RateLimitError:
+                # Only re-raise if rate limiting is enabled
+                if self.rate_tracker.enabled:
+                    raise
+                else:
+                    # If rate limiting is disabled, treat as regular error
+                    success = False
+                    error_message = "Rate limit hit but rate limiting disabled"
+                    logger.warning(
+                        f"Rate limit hit on {self.__class__.__name__} but rate limiting is disabled"
+                    )
+                    results_count = 0
+                    return []
+            except Exception as e:
+                # Other errors - don't retry
+                success = False
+                error_message = str(e)
+                logger.exception(
+                    f"Search engine {self.__class__.__name__} failed"
+                )
+                results_count = 0
+                return []
 
+        try:
+            return _run_with_retry()
+        except RetryError as e:
+            # All retries exhausted
+            success = False
+            error_message = f"Rate limited after all retries: {e}"
+            logger.exception(
+                f"{self.__class__.__name__} failed after all retries"
+            )
+            return []
         except Exception as e:
             success = False
             error_message = str(e)
-            logger.error(f"Search engine {self.__class__.__name__} failed: {e}")
-            results_count = 0
+            logger.exception(f"Search engine {self.__class__.__name__} error")
             return []
-
         finally:
             # Record search metrics
             response_time_ms = int((time.time() - start_time) * 1000)

local_deep_research/web_search_engines/search_engine_factory.py

@@ -8,6 +8,7 @@ from loguru import logger
 from ..utilities.db_utils import get_db_setting
 from .search_engine_base import BaseSearchEngine
 from .search_engines_config import default_search_engine, search_config
+from .retriever_registry import retriever_registry
 
 
 def create_search_engine(
@@ -24,6 +25,18 @@ def create_search_engine(
     Returns:
         Initialized search engine instance or None if creation failed
     """
+    # Check if this is a registered retriever first
+    retriever = retriever_registry.get(engine_name)
+    if retriever:
+        logger.info(f"Using registered LangChain retriever: {engine_name}")
+        from .engines.search_engine_retriever import RetrieverSearchEngine
+
+        return RetrieverSearchEngine(
+            retriever=retriever,
+            name=engine_name,
+            max_results=kwargs.get("max_results", 10),
+        )
+
     # If engine name not found, use default
     if engine_name not in search_config():
         logger.warning(
@@ -313,6 +326,7 @@ def get_search(
     logger.info(
         f"Creating search engine for tool: {search_tool} with params: {params.keys()}"
     )
+
     engine = create_search_engine(search_tool, **params)
 
     # Add debugging to check if engine is None

local_deep_research/web_search_engines/search_engines_config.py

@@ -57,6 +57,26 @@ def search_config() -> Dict[str, Any]:
     search_engines = _extract_per_engine_config(config_data)
     search_engines["auto"] = get_db_setting("search.engine.auto", {})
 
+    # Add registered retrievers as available search engines
+    from .retriever_registry import retriever_registry
+
+    for name in retriever_registry.list_registered():
+        search_engines[name] = {
+            "module_path": ".engines.search_engine_retriever",
+            "class_name": "RetrieverSearchEngine",
+            "requires_api_key": False,
+            "requires_llm": False,
+            "description": f"LangChain retriever: {name}",
+            "strengths": [
+                "Domain-specific knowledge",
+                "No rate limits",
+                "Fast retrieval",
+            ],
+            "weaknesses": ["Limited to indexed content"],
+            "supports_full_search": True,
+            "is_retriever": True,  # Mark as retriever for identification
+        }
+
     logger.info(
         f"Loaded {len(search_engines)} search engines from configuration file"
     )

local_deep_research-0.6.1.dist-info/METADATA

@@ -0,0 +1,374 @@
+Metadata-Version: 2.1
+Name: local-deep-research
+Version: 0.6.1
+Summary: AI-powered research assistant with deep, iterative analysis using LLMs and web searches
+Author-Email: LearningCircuit <185559241+LearningCircuit@users.noreply.github.com>, HashedViking <6432677+HashedViking@users.noreply.github.com>, djpetti <djpetti@gmail.com>
+License: MIT License
+         
+         Copyright (c) 2025 LearningCircuit
+         
+         Permission is hereby granted, free of charge, to any person obtaining a copy
+         of this software and associated documentation files (the "Software"), to deal
+         in the Software without restriction, including without limitation the rights
+         to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+         copies of the Software, and to permit persons to whom the Software is
+         furnished to do so, subject to the following conditions:
+         
+         The above copyright notice and this permission notice shall be included in all
+         copies or substantial portions of the Software.
+         
+         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+         IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+         FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+         AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+         SOFTWARE.
+         
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Project-URL: Homepage, https://github.com/LearningCircuit/local-deep-research
+Project-URL: Bug Tracker, https://github.com/LearningCircuit/local-deep-research/issues
+Requires-Python: >=3.10
+Requires-Dist: langchain>=0.3.18
+Requires-Dist: langchain-community>=0.3.17
+Requires-Dist: langchain-core>=0.3.34
+Requires-Dist: langchain-ollama>=0.2.3
+Requires-Dist: langchain-openai>=0.3.5
+Requires-Dist: langchain-anthropic>=0.3.13
+Requires-Dist: duckduckgo_search>=7.3.2
+Requires-Dist: python-dateutil>=2.9.0
+Requires-Dist: typing_extensions>=4.12.2
+Requires-Dist: justext
+Requires-Dist: playwright
+Requires-Dist: beautifulsoup4
+Requires-Dist: flask>=3.1.0
+Requires-Dist: flask-cors>=3.0.10
+Requires-Dist: flask-socketio>=5.1.1
+Requires-Dist: sqlalchemy>=1.4.23
+Requires-Dist: wikipedia
+Requires-Dist: arxiv>=1.4.3
+Requires-Dist: pypdf
+Requires-Dist: sentence-transformers
+Requires-Dist: faiss-cpu
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: toml>=0.10.2
+Requires-Dist: platformdirs>=3.0.0
+Requires-Dist: dynaconf
+Requires-Dist: requests>=2.28.0
+Requires-Dist: tiktoken>=0.4.0
+Requires-Dist: xmltodict>=0.13.0
+Requires-Dist: lxml>=4.9.2
+Requires-Dist: pdfplumber>=0.9.0
+Requires-Dist: unstructured>=0.10.0
+Requires-Dist: google-search-results
+Requires-Dist: importlib-resources>=6.5.2
+Requires-Dist: setuptools>=78.1.0
+Requires-Dist: flask-wtf>=1.2.2
+Requires-Dist: optuna>=4.3.0
+Requires-Dist: elasticsearch==8.14.0
+Requires-Dist: methodtools>=0.4.7
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: cachetools>=5.5.2
+Requires-Dist: matplotlib>=3.10.3
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: plotly>=6.0.1
+Requires-Dist: kaleido==0.2.1
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: tenacity>=8.0.0
+Description-Content-Type: text/markdown
+
+# Local Deep Research
+
+<div align="center">
+
+[![GitHub stars](https://img.shields.io/github/stars/LearningCircuit/local-deep-research?style=for-the-badge)](https://github.com/LearningCircuit/local-deep-research/stargazers)
+[![Docker Pulls](https://img.shields.io/docker/pulls/localdeepresearch/local-deep-research?style=for-the-badge)](https://hub.docker.com/r/localdeepresearch/local-deep-research)
+[![PyPI Downloads](https://img.shields.io/pypi/dm/local-deep-research?style=for-the-badge)](https://pypi.org/project/local-deep-research/)
+
+[![Tests](https://img.shields.io/github/actions/workflow/status/LearningCircuit/local-deep-research/tests.yml?branch=main&style=for-the-badge&label=Tests)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/tests.yml)
+[![CodeQL](https://img.shields.io/github/actions/workflow/status/LearningCircuit/local-deep-research/codeql.yml?branch=main&style=for-the-badge&label=CodeQL)](https://github.com/LearningCircuit/local-deep-research/security/code-scanning)
+
+[![Discord](https://img.shields.io/discord/1352043059562680370?style=for-the-badge&logo=discord)](https://discord.gg/ttcqQeFcJ3)
+[![Reddit](https://img.shields.io/badge/Reddit-r/LocalDeepResearch-FF4500?style=for-the-badge&logo=reddit)](https://www.reddit.com/r/LocalDeepResearch/)
+
+
+**AI-powered research assistant for deep, iterative research**
+
+*Performs deep, iterative research using multiple LLMs and search engines with proper citations*
+</div>
+## 🚀 What is Local Deep Research?
+
+LDR is an AI research assistant that performs systematic research by:
+
+- **Breaking down complex questions** into focused sub-queries
+- **Searching multiple sources** in parallel (web, academic papers, local documents)
+- **Verifying information** across sources for accuracy
+- **Creating comprehensive reports** with proper citations
+
+It aims to help researchers, students, and professionals find accurate information quickly while maintaining transparency about sources.
+
+## 🎯 Why Choose LDR?
+
+- **Privacy-Focused**: Run entirely locally with Ollama + SearXNG
+- **Flexible**: Use any LLM, any search engine, any vector store
+- **Comprehensive**: Multiple research modes from quick summaries to detailed reports
+- **Transparent**: Track costs and performance with built-in analytics
+- **Open Source**: MIT licensed with an active community
+
+## ✨ Key Features
+
+### 🔍 Research Modes
+- **Quick Summary** - Get answers in 30 seconds to 3 minutes with citations
+- **Detailed Research** - Comprehensive analysis with structured findings
+- **Report Generation** - Professional reports with sections and table of contents
+- **Document Analysis** - Search your private documents with AI
+
+### 🛠️ Advanced Capabilities
+- **[LangChain Integration](docs/LANGCHAIN_RETRIEVER_INTEGRATION.md)** - Use any vector store as a search engine
+- **[REST API](docs/api-quickstart.md)** - Language-agnostic HTTP access
+- **[Benchmarking](docs/BENCHMARKING.md)** - Test and optimize your configuration
+- **[Analytics Dashboard](docs/analytics-dashboard.md)** - Track costs, performance, and usage metrics
+- **Real-time Updates** - WebSocket support for live research progress
+- **Export Options** - Download results as PDF or Markdown
+- **Research History** - Save, search, and revisit past research
+- **Adaptive Rate Limiting** - Intelligent retry system that learns optimal wait times
+- **Keyboard Shortcuts** - Navigate efficiently (ESC, Ctrl+Shift+1-5)
+
+### 🌐 Search Sources
+
+#### Free Search Engines
+- **Academic**: arXiv, PubMed, Semantic Scholar
+- **General**: Wikipedia, SearXNG, DuckDuckGo
+- **Technical**: GitHub, Elasticsearch
+- **Historical**: Wayback Machine
+- **News**: The Guardian
+
+#### Premium Search Engines
+- **Tavily** - AI-powered search
+- **Google** - Via SerpAPI or Programmable Search Engine
+- **Brave Search** - Privacy-focused web search
+
+#### Custom Sources
+- **Local Documents** - Search your files with AI
+- **LangChain Retrievers** - Any vector store or database
+- **Meta Search** - Combine multiple engines intelligently
+
+[Full Search Engines Guide →](docs/search-engines.md)
+
+## ⚡ Quick Start
+
+### Option 1: Docker (Quickstart no MAC/ARM)
+
+```bash
+# Step 1: Pull and run SearXNG for optimal search results
+docker run -d -p 8080:8080 --name searxng searxng/searxng
+
+# Step 2: Pull and run Local Deep Research (Please build your own docker on ARM)
+docker run -d -p 5000:5000 --name local-deep-research --volume 'deep-research:/install/.venv/lib/python3.13/site-packages/data/' localdeepresearch/local-deep-research
+```
+
+### Option 2: Docker Compose (Recommended)
+
+LDR uses Docker compose to bundle the web app and all it's dependencies so
+you can get up and running quickly.
+
+#### Option 2a: DIY docker-compose
+See [docker-compose.yml](./docker-compose.yml) for a docker-compose file with reasonable defaults to get up and running with ollama, searxng, and local deep research all running locally.
+
+Things you may want/need to configure:
+* Ollama GPU driver
+* Ollama context length (depends on available VRAM)
+* Ollama keep alive (duration model will stay loaded into VRAM and idle before getting unloaded automatically)
+* Deep Research model (depends on available VRAM and preference)
+
+#### Option 2b: Use Cookie Cutter to tailor a docker-compose to your needs:
+
+##### Prerequisites
+
+- [Docker](https://docs.docker.com/engine/install/)
+- [Docker Compose](https://docs.docker.com/compose/install/)
+- `cookiecutter`: Run `pip install --user cookiecutter`
+
+Clone the repository:
+
+```bash
+git clone https://github.com/LearningCircuit/local-deep-research.git
+cd local-deep-research
+```
+
+### Configuring with Docker Compose
+
+Cookiecutter will interactively guide you through the process of creating a
+`docker-compose` configuration that meets your specific needs. This is the
+recommended approach if you are not very familiar with Docker.
+
+In the LDR repository, run the following command
+to generate the compose file:
+
+```bash
+cookiecutter cookiecutter-docker/
+docker compose -f docker-compose.default.yml up
+```
+
+[Docker Compose Guide →](docs/docker-compose-guide.md)
+
+### Option 3: Python Package
+
+```bash
+# Step 1: Install the package
+pip install local-deep-research
+
+# Step 2: Setup SearXNG for best results
+docker pull searxng/searxng
+docker run -d -p 8080:8080 --name searxng searxng/searxng
+
+# Step 3: Install Ollama from https://ollama.ai
+
+# Step 4: Download a model
+ollama pull gemma3:12b
+
+# Step 5: Start the web interface
+python -m local_deep_research.web.app
+```
+
+[Full Installation Guide →](https://github.com/LearningCircuit/local-deep-research/wiki/Installation)
+
+## 💻 Usage Examples
+
+### Python API
+```python
+from local_deep_research.api import quick_summary
+
+# Simple usage
+result = quick_summary("What are the latest advances in quantum computing?")
+print(result["summary"])
+
+# Advanced usage with custom configuration
+result = quick_summary(
+    query="Impact of AI on healthcare",
+    search_tool="searxng",
+    search_strategy="focused-iteration",
+    iterations=2
+)
+```
+
+### HTTP API
+```bash
+curl -X POST http://localhost:5000/api/v1/quick_summary \
+  -H "Content-Type: application/json" \
+  -d '{"query": "Explain CRISPR gene editing"}'
+```
+
+[More Examples →](examples/api_usage/)
+
+### Command Line Tools
+
+```bash
+# Run benchmarks from CLI
+python -m local_deep_research.benchmarks --dataset simpleqa --examples 50
+
+# Manage rate limiting
+python -m local_deep_research.web_search_engines.rate_limiting status
+python -m local_deep_research.web_search_engines.rate_limiting reset
+```
+
+## 🔗 Enterprise Integration
+
+Connect LDR to your existing knowledge base:
+
+```python
+from local_deep_research.api import quick_summary
+
+# Use your existing LangChain retriever
+result = quick_summary(
+    query="What are our deployment procedures?",
+    retrievers={"company_kb": your_retriever},
+    search_tool="company_kb"
+)
+```
+
+Works with: FAISS, Chroma, Pinecone, Weaviate, Elasticsearch, and any LangChain-compatible retriever.
+
+[Integration Guide →](docs/LANGCHAIN_RETRIEVER_INTEGRATION.md)
+
+## 📊 Performance & Analytics
+
+### Benchmark Results
+Early experiments on small SimpleQA dataset samples:
+
+| Configuration | Accuracy | Notes |
+|--------------|----------|--------|
+| gpt-4.1-mini + SearXNG + focused_iteration | 90-95% | Limited sample size |
+| gpt-4.1-mini + Tavily + focused_iteration | 90-95% | Limited sample size |
+| gemini-2.0-flash-001 + SearXNG | 82% | Single test run |
+
+Note: These are preliminary results from initial testing. Performance varies significantly based on query types, model versions, and configurations. [Run your own benchmarks →](docs/BENCHMARKING.md)
+
+### Built-in Analytics Dashboard
+Track costs, performance, and usage with detailed metrics. [Learn more →](docs/analytics-dashboard.md)
+
+## 🤖 Supported LLMs
+
+### Local Models (via Ollama)
+- Llama 3, Mistral, Gemma, DeepSeek
+- LLM processing stays local (search queries still go to web)
+- No API costs
+
+### Cloud Models
+- OpenAI (GPT-4, GPT-3.5)
+- Anthropic (Claude 3)
+- Google (Gemini)
+- 100+ models via OpenRouter
+
+[Model Setup →](docs/env_configuration.md)
+
+## 📚 Documentation
+
+### Getting Started
+- [Installation Guide](https://github.com/LearningCircuit/local-deep-research/wiki/Installation)
+- [Frequently Asked Questions](docs/faq.md)
+- [API Quickstart](docs/api-quickstart.md)
+- [Configuration Guide](docs/env_configuration.md)
+
+### Core Features
+- [All Features Guide](docs/features.md)
+- [Search Engines Guide](docs/search-engines.md)
+- [Analytics Dashboard](docs/analytics-dashboard.md)
+
+### Advanced Features
+- [LangChain Integration](docs/LANGCHAIN_RETRIEVER_INTEGRATION.md)
+- [Benchmarking System](docs/BENCHMARKING.md)
+- [Elasticsearch Setup](docs/elasticsearch_search_engine.md)
+- [SearXNG Setup](docs/SearXNG-Setup.md)
+
+### Development
+- [Docker Compose Guide](docs/docker-compose-guide.md)
+- [Development Guide](docs/developing.md)
+- [Security Guide](docs/security/CODEQL_GUIDE.md)
+- [Release Guide](docs/RELEASE_GUIDE.md)
+
+### Examples & Tutorials
+- [API Examples](examples/api_usage/)
+- [Benchmark Examples](examples/benchmarks/)
+- [Optimization Examples](examples/optimization/)
+
+## 🤝 Community & Support
+
+- [Discord](https://discord.gg/ttcqQeFcJ3) - Get help and share research techniques
+- [Reddit](https://www.reddit.com/r/LocalDeepResearch/) - Updates and showcases
+- [GitHub Issues](https://github.com/LearningCircuit/local-deep-research/issues) - Bug reports
+
+## 🚀 Contributing
+
+We welcome contributions! See our [Contributing Guide](CONTRIBUTING.md) to get started.
+
+## 📄 License
+
+MIT License - see [LICENSE](LICENSE) file.
+
+Built with: [LangChain](https://github.com/hwchase17/langchain), [Ollama](https://ollama.ai), [SearXNG](https://searxng.org/), [FAISS](https://github.com/facebookresearch/faiss)
+
+> **Support Free Knowledge:** Consider donating to [Wikipedia](https://donate.wikimedia.org), [arXiv](https://arxiv.org/about/give), or [PubMed](https://www.nlm.nih.gov/pubs/donations/donations.html).