local-deep-research 0.1.16__py3-none-any.whl → 0.1.17__py3-none-any.whl

local_deep_research/web_search_engines/full_search.py

@@ -1,254 +0,0 @@
-import justext
-from langchain_community.document_loaders import AsyncChromiumLoader
-from langchain_community.document_transformers import BeautifulSoupTransformer
-from langchain_core.language_models import BaseLLM
-from typing import List, Dict, Any, Optional, Union
-import json
-import os
-from .utilties.search_utilities import remove_think_tags
-from datetime import datetime
-from local_deep_research import config
-
-class FullSearchResults:
-    """
-    Enhanced web content retrieval class that works with the BaseSearchEngine architecture.
-    Can be used as a wrapper around web-based search engines like DuckDuckGo and SerpAPI.
-    """
-    
-    def __init__(
-        self,
-        llm: BaseLLM,
-        web_search,        
-        output_format: str = "list",
-        language: str = "English",
-        max_results: int = 10,
-        region: str = "wt-wt",
-        time: str = "y",
-        safesearch: str = "Moderate"
-    ):
-        """
-        Initialize the full search results processor.
-        
-        Args:
-            llm: Language model instance for relevance filtering
-            web_search: Web search engine instance that provides initial results
-            output_format: Format of output ('list' or other formats)
-            language: Language for content processing
-            max_results: Maximum number of search results
-            region: Search region
-            time: Time period for search results
-            safesearch: Safe search setting
-        """
-        self.llm = llm
-        self.output_format = output_format
-        self.language = language
-        self.max_results = max_results
-        self.region = region
-        self.time = time
-        self.safesearch = safesearch
-        self.web_search = web_search
-        os.environ["USER_AGENT"] = "Local Deep Research/1.0"
-
-        self.bs_transformer = BeautifulSoupTransformer()
-        self.tags_to_extract = ["p", "div", "span"]
-    
-    def run(self, query: str) -> List[Dict[str, Any]]:
-        """
-        Legacy method that performs a full search in one step.
-        Respects config parameters:
-        - SEARCH_SNIPPETS_ONLY: If True, only returns snippets without full content
-        - SKIP_RELEVANCE_FILTER: If True, returns all results without filtering
-        
-        Args:
-            query: The search query
-            
-        Returns:
-            List of search results with full content (unless SEARCH_SNIPPETS_ONLY is True)
-        """
-        # Phase 1: Get search results from the web search engine
-        previews = self._get_previews(query)
-        if not previews:
-            return []
-            
-        # Phase 2: Filter URLs using LLM (unless SKIP_RELEVANCE_FILTER is True)
-        if hasattr(config, 'SKIP_RELEVANCE_FILTER') and config.SKIP_RELEVANCE_FILTER:
-            relevant_items = previews
-            print("Skipping relevance filtering as per config")
-        else:
-            relevant_items = self._filter_relevant_items(previews, query)
-            if not relevant_items:
-                return []
-            
-        # Phase 3: Get full content for relevant items (unless SEARCH_SNIPPETS_ONLY is True)
-        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
-            print("Returning snippet-only results as per config")
-            return relevant_items
-        else:
-            results = self._get_full_content(relevant_items)
-            return results
-    
-    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
-        """
-        Get preview information from the web search engine.
-        
-        Args:
-            query: The search query
-            
-        Returns:
-            List of preview dictionaries
-        """
-        try:
-            # Get search results from the web search engine
-            search_results = self.web_search.invoke(query)
-            
-            if not isinstance(search_results, list):
-                print("Error: Expected search results in list format")
-                return []
-            
-            # Return the results as previews
-            return search_results
-            
-        except Exception as e:
-            print(f"Error getting previews: {e}")
-            return []
-    
-    def _filter_relevant_items(self, previews: List[Dict[str, Any]], query: str) -> List[Dict[str, Any]]:
-        """
-        Filter previews for relevance using LLM.
-        
-        Args:
-            previews: List of preview dictionaries
-            query: The original search query
-            
-        Returns:
-            List of relevant preview dictionaries
-        """
-        # Skip filtering if disabled in config or no previews
-        if not config.QUALITY_CHECK_DDG_URLS or not previews:
-            return previews
-        
-        # Format for LLM evaluation
-        now = datetime.now()
-        current_time = now.strftime("%Y-%m-%d")
-        prompt = f"""ONLY Return a JSON array. The response contains no letters. Evaluate these URLs for:
-            1. Timeliness (today: {current_time})
-            2. Factual accuracy (cross-reference major claims)
-            3. Source reliability (prefer official company websites, established news outlets)
-            4. Direct relevance to query: {query}
-
-            URLs to evaluate:
-            {json.dumps(previews, indent=2)}
-
-            Return a JSON array of indices (0-based) for sources that meet ALL criteria.
-            ONLY Return a JSON array of indices (0-based) and nothing else. No letters. 
-            Example response: \n[0, 2, 4]\n\n"""
-        
-        try:
-            # Get LLM's evaluation
-            response = self.llm.invoke(prompt)
-            
-            # Extract JSON array from response
-            response_text = remove_think_tags(response.content)
-            # Clean up response to handle potential formatting issues
-            response_text = response_text.strip()
-            
-            # Find the first occurrence of '[' and the last occurrence of ']'
-            start_idx = response_text.find('[')
-            end_idx = response_text.rfind(']')
-            
-            if start_idx >= 0 and end_idx > start_idx:
-                array_text = response_text[start_idx:end_idx+1]
-                good_indices = json.loads(array_text)
-                
-                # Return only the results with good indices
-                return [r for i, r in enumerate(previews) if i in good_indices]
-            else:
-                print("Could not find JSON array in response, returning all previews")
-                return previews
-                
-        except Exception as e:
-            print(f"URL filtering error: {e}")
-            # Fall back to returning all previews on error
-            return previews
-    
-    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        """
-        Get full content for the relevant items by retrieving and processing web pages.
-        
-        Args:
-            relevant_items: List of relevant preview dictionaries
-            
-        Returns:
-            List of result dictionaries with full content
-        """
-        nr_full_text = 0
-        
-        # Extract URLs from relevant items
-        urls = [item.get("link") for item in relevant_items if item.get("link")]
-        
-        if not urls:
-            print("\n === NO VALID LINKS ===\n")
-            return relevant_items
-        
-        try:
-            # Download the full HTML pages for filtered URLs
-            loader = AsyncChromiumLoader(urls)
-            html_docs = loader.load()
-            
-            # Process the HTML using BeautifulSoupTransformer
-            full_docs = self.bs_transformer.transform_documents(
-                html_docs, tags_to_extract=self.tags_to_extract
-            )
-            
-            # Remove boilerplate from each document
-            url_to_content = {}
-            for doc in full_docs:
-                nr_full_text += 1
-                source = doc.metadata.get("source")
-                if source:
-                    cleaned_text = self._remove_boilerplate(doc.page_content)
-                    url_to_content[source] = cleaned_text
-            
-            # Attach the cleaned full content to each result
-            results = []
-            for item in relevant_items:
-                new_item = item.copy()
-                link = item.get("link")
-                new_item["full_content"] = url_to_content.get(link, None)
-                results.append(new_item)
-            
-            print(f"FULL SEARCH WITH FILTERED URLS - Full text retrieved: {nr_full_text}")
-            return results
-            
-        except Exception as e:
-            print(f"Error retrieving full content: {e}")
-            # Return original items if full content retrieval fails
-            return relevant_items
-    
-    def _remove_boilerplate(self, html: str) -> str:
-        """
-        Remove boilerplate content from HTML.
-        
-        Args:
-            html: HTML content
-            
-        Returns:
-            Cleaned text content
-        """
-        if not html or not html.strip():
-            return ""
-        try:
-            paragraphs = justext.justext(html, justext.get_stoplist(self.language))
-            cleaned = "\n".join([p.text for p in paragraphs if not p.is_boilerplate])
-            return cleaned
-        except Exception as e:
-            print(f"Error removing boilerplate: {e}")
-            return html
-    
-    def invoke(self, query: str) -> List[Dict[str, Any]]:
-        """Compatibility method for LangChain tools"""
-        return self.run(query)
-    
-    def __call__(self, query: str) -> List[Dict[str, Any]]:
-        """Make the class callable like a function"""
-        return self.invoke(query)

local_deep_research-0.1.16.dist-info/METADATA

@@ -1,346 +0,0 @@
-Metadata-Version: 2.4
-Name: local-deep-research
-Version: 0.1.16
-Summary: AI-powered research assistant with deep, iterative analysis using LLMs and web searches
-Author-email: LearningCircuit <185559241+LearningCircuit@users.noreply.github.com>, HashedViking <6432677+HashedViking@users.noreply.github.com>
-License: MIT License
-        
-        Copyright (c) 2025 LearningCircuit
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
-        
-Project-URL: Homepage, https://github.com/LearningCircuit/local-deep-research
-Project-URL: Bug Tracker, https://github.com/LearningCircuit/local-deep-research/issues
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: langchain>=0.3.18
-Requires-Dist: langchain-community>=0.3.17
-Requires-Dist: langchain-core>=0.3.34
-Requires-Dist: langchain-ollama>=0.2.3
-Requires-Dist: langchain-openai>=0.3.5
-Requires-Dist: langchain_anthropic>=0.3.7
-Requires-Dist: duckduckgo_search>=7.3.2
-Requires-Dist: python-dateutil>=2.9.0
-Requires-Dist: typing_extensions>=4.12.2
-Requires-Dist: justext
-Requires-Dist: playwright
-Requires-Dist: beautifulsoup4
-Requires-Dist: flask>=2.0.1
-Requires-Dist: flask-cors>=3.0.10
-Requires-Dist: flask-socketio>=5.1.1
-Requires-Dist: sqlalchemy>=1.4.23
-Requires-Dist: wikipedia
-Requires-Dist: arxiv>=1.4.3
-Requires-Dist: pypdf
-Requires-Dist: sentence-transformers
-Requires-Dist: faiss-cpu
-Requires-Dist: pydantic>=2.0.0
-Requires-Dist: pydantic-settings>=2.0.0
-Requires-Dist: toml>=0.10.2
-Requires-Dist: platformdirs>=3.0.0
-Requires-Dist: dynaconf
-Dynamic: license-file
-
-# Local Deep Research
-
-A powerful AI-powered research assistant that performs deep, iterative analysis using multiple LLMs and web searches. The system can be run locally for privacy or configured to use cloud-based LLMs for enhanced capabilities.
-
-## Features
-
-- 🔍 **Advanced Research Capabilities**
-  - Automated deep research with intelligent follow-up questions
-  - Citation tracking and source verification
-  - Multi-iteration analysis for comprehensive coverage
-  - Full webpage content analysis (not just snippets)
-
-- 🤖 **Flexible LLM Support**
-  - Local AI processing with Ollama models
-  - Cloud LLM support (Claude, GPT)
-  - Supports all Langchain models
-  - Configurable model selection based on needs
-
-- 📊 **Rich Output Options**
-  - Detailed research findings with citations
-  - Comprehensive research reports
-  - Quick summaries for rapid insights
-  - Source tracking and verification
-
-- 🔒 **Privacy-Focused**
-  - Runs entirely on your machine when using local models
-  - Configurable search settings
-  - Transparent data handling
-
-- 🌐 **Enhanced Search Integration**
-  - **Auto-selection of search sources**: The "auto" search engine intelligently analyzes your query and selects the most appropriate search engine based on the query content
-  - **SearXNG** integration for local web-search engine, great for privacy, no API key required (requires a searxng server)
-  - Wikipedia integration for factual knowledge
-  - arXiv integration for scientific papers and academic research
-  - PubMed integration for biomedical literature and medical research
-  - DuckDuckGo integration for web searches (may experience rate limiting)
-  - SerpAPI integration for Google search results (requires API key)
-  - Google Programmable Search Engine integration for custom search experiences (requires API key)
-  - The Guardian integration for news articles and journalism (requires API key)
-  - **Local RAG search for private documents** - search your own documents with vector embeddings
-  - Full webpage content retrieval
-  - Source filtering and validation
-  - Configurable search parameters
-
-- 📑 **Local Document Search (RAG)**
-  - Vector embedding-based search of your local documents
-  - Create custom document collections for different topics
-  - Privacy-preserving - your documents stay on your machine
-  - Intelligent chunking and retrieval 
-  - Compatible with various document formats (PDF, text, markdown, etc.)
-  - Automatic integration with meta-search for unified queries
-
-## Example Research: Fusion Energy Developments
-
-The repository includes complete research examples demonstrating the tool's capabilities. For instance, our [fusion energy research analysis](https://github.com/LearningCircuit/local-deep-research/blob/main/examples/fusion-energy-research-developments.md) provides a comprehensive overview of:
-
-- Latest scientific breakthroughs in fusion research (2022-2025)
-- Private sector funding developments exceeding $6 billion
-- Expert projections for commercial fusion energy timelines
-- Regulatory frameworks being developed for fusion deployment
-- Technical challenges that must be overcome for commercial viability
-
-This example showcases the system's ability to perform multiple research iterations, follow evidence trails across scientific and commercial domains, and synthesize information from diverse sources while maintaining proper citation.
-
-## Installation
-
-1. Clone the repository:
-```bash
-git clone https://github.com/LearningCircuit/local-deep-research.git
-cd local-deep-research
-```
-(experimental pip install with new features (but not so well tested yet): **pip install local-deep-research** )
-2. Install dependencies:
-```bash
-pip install -r requirements.txt
-playwright install
-```
-
-3. Install Ollama (for local models):
-```bash
-# Install Ollama from https://ollama.ai
-ollama pull mistral  # Default model - many work really well choose best for your hardware (fits in GPU)
-```
-
-4. Configure environment variables:
-```bash
-# Copy the template
-cp .env.template .env
-```
-
-## Experimental install
-```bash
-#experimental pip install with new features (but not so well tested yet):
-pip install local-deep-research
-playwright install
-ollama pull mistral 
-```
-## Community & Support
-
-We've just launched our [Discord server](https://discord.gg/2E6gYU2Z) for this project!
-
-Our Discord server can help to exchange ideas about research approaches, discuss advanced usage patterns, and share other ideas.
-
-# Edit .env with your API keys (if using cloud LLMs)
-ANTHROPIC_API_KEY=your-api-key-here  # For Claude
-OPENAI_API_KEY=your-openai-key-here  # For GPT models
-GUARDIAN_API_KEY=your-guardian-api-key-here  # For The Guardian search
-```
-
-## Usage
-Terminal usage (not recommended):
-```bash
-python main.py
-```
-
-### Web Interface
-
-The project includes a web interface for a more user-friendly experience:
-
-```bash
-python app.py
-```
-
-This will start a local web server, accessible at `http://127.0.0.1:5000` in your browser.
-
-#### Web Interface Features:
-
-- **Dashboard**: Intuitive interface for starting and managing research queries
-- **Real-time Updates**: Track research progress with live updates
-- **Research History**: Access and manage past research queries
-- **PDF Export**: Download completed research reports as PDF documents
-- **Research Management**: Terminate ongoing research processes or delete past records
-
-![Web Interface](./web1.png)
-![Web Interface](./web2.png)
-### Configuration
-**Please report your best settings in issues so we can improve the default settings.**
-
-Key settings in `config.py`:
-```python
-# LLM Configuration
-DEFAULT_MODEL = "mistral"  # Change based on your needs
-DEFAULT_TEMPERATURE = 0.7
-MAX_TOKENS = 8000
-
-# Search Configuration
-MAX_SEARCH_RESULTS = 40
-SEARCH_REGION = "us-en"
-TIME_PERIOD = "y"
-SAFE_SEARCH = True
-SEARCH_SNIPPETS_ONLY = False
-
-# Choose search tool: "wiki", "arxiv", "duckduckgo", "guardian", "serp", "local_all", or "auto"
-search_tool = "auto"  # "auto" will intelligently select the best search engine for your query
-```
-
-## Local Document Search (RAG)
-
-The system includes powerful local document search capabilities using Retrieval-Augmented Generation (RAG). This allows you to search and retrieve content from your own document collections.
-
-### Setting Up Local Collections
-
-Create a file named `local_collections.py` in the project root directory:
-
-```python
-# local_collections.py
-import os
-from typing import Dict, Any
-
-# Registry of local document collections
-LOCAL_COLLECTIONS = {
-    # Research Papers Collection
-    "research_papers": {
-        "name": "Research Papers",
-        "description": "Academic research papers and articles",
-        "paths": [os.path.abspath("local_search_files/research_papers")],  # Use absolute paths
-        "enabled": True,
-        "embedding_model": "all-MiniLM-L6-v2",
-        "embedding_device": "cpu",
-        "embedding_model_type": "sentence_transformers",
-        "max_results": 20,
-        "max_filtered_results": 5,
-        "chunk_size": 800,  # Smaller chunks for academic content
-        "chunk_overlap": 150,
-        "cache_dir": ".cache/local_search/research_papers"
-    },
-    
-    # Personal Notes Collection
-    "personal_notes": {
-        "name": "Personal Notes",
-        "description": "Personal notes and documents",
-        "paths": [os.path.abspath("local_search_files/personal_notes")],  # Use absolute paths
-        "enabled": True,
-        "embedding_model": "all-MiniLM-L6-v2",
-        "embedding_device": "cpu",
-        "embedding_model_type": "sentence_transformers",
-        "max_results": 30,
-        "max_filtered_results": 10,
-        "chunk_size": 500,  # Smaller chunks for notes
-        "chunk_overlap": 100,
-        "cache_dir": ".cache/local_search/personal_notes"
-    }
-}
-```
-
-Create directories for your collections:
-
-```bash
-mkdir -p local_search_files/research_papers
-mkdir -p local_search_files/personal_notes
-```
-
-Add your documents to these folders, and the system will automatically index them and make them available for searching.
-
-### Using Local Search
-
-You can use local search in several ways:
-
-1. **Auto-selection**: Set `search_tool = "auto"` in `config.py` and the system will automatically use your local collections when appropriate for the query.
-
-2. **Explicit Selection**: Set `search_tool = "research_papers"` to search only that specific collection.
-
-3. **Search All Local Collections**: Set `search_tool = "local_all"` to search across all your local document collections.
-
-4. **Query Syntax**: Use `collection:collection_name your query` to target a specific collection within a query.
-
-### Search Engine Options
-
-The system supports multiple search engines that can be selected by changing the `search_tool` variable in `config.py`:
-
-- **Auto** (`auto`): Intelligent search engine selector that analyzes your query and chooses the most appropriate source (Wikipedia, arXiv, local collections, etc.)
-- **SearXNG** (`searxng`): Local web-search engine, great for privacy, no API key required (requires a searxng server)
-- **Wikipedia** (`wiki`): Best for general knowledge, facts, and overview information
-- **arXiv** (`arxiv`): Great for scientific and academic research, accessing preprints and papers
-- **PubMed** (`pubmed`): Excellent for biomedical literature, medical research, and health information
-- **DuckDuckGo** (`duckduckgo`): General web search that doesn't require an API key
-- **The Guardian** (`guardian`): Quality journalism and news articles (requires an API key)
-- **SerpAPI** (`serp`): Google search results (requires an API key)
-- **Google Programmable Search Engine** (`google_pse`): Custom search experiences with control over search scope and domains (requires API key and search engine ID)
-- **Local Collections**: Any collections defined in your `local_collections.py` file
-
-> **Note:** The "auto" option will intelligently select the best search engine based on your query. For example, if you ask about physics research papers, it might select arXiv or your research_papers collection, while if you ask about current events, it might select The Guardian or DuckDuckGo.
-
-> **Support Free Knowledge:** If you frequently use the search engines in this tool, please consider making a donation to these organizations. They provide valuable services and rely on user support to maintain their operations:
-> - [Donate to Wikipedia](https://donate.wikimedia.org)
-> - [Support The Guardian](https://support.theguardian.com)
-> - [Support arXiv](https://arxiv.org/about/give)
-> - [Donate to DuckDuckGo](https://duckduckgo.com/donations)
-> - [Support PubMed/NCBI](https://www.nlm.nih.gov/pubs/donations/donations.html)
-
-## License
-
-This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
-
-## Acknowledgments
-- Built with [Ollama](https://ollama.ai) for local AI processing
-- Search powered by multiple sources:
-  - [Wikipedia](https://www.wikipedia.org/) for factual knowledge (default search engine)
-  - [arXiv](https://arxiv.org/) for scientific papers
-  - [PubMed](https://pubmed.ncbi.nlm.nih.gov/) for biomedical literature
-  - [DuckDuckGo](https://duckduckgo.com) for web search
-  - [The Guardian](https://www.theguardian.com/) for quality journalism
-  - [SerpAPI](https://serpapi.com) for Google search results (requires API key)
-  - [SearXNG](https://searxng.org/) for local web-search engine 
-- Built on [LangChain](https://github.com/hwchase17/langchain) framework
-- Uses [justext](https://github.com/miso-belica/justext) for content extraction
-- [Playwright](https://playwright.dev) for web content retrieval
-- Uses [FAISS](https://github.com/facebookresearch/faiss) for vector similarity search
-- Uses [sentence-transformers](https://github.com/UKPLab/sentence-transformers) for embeddings
-
-## Contributing
-
-Contributions are welcome! Please feel free to submit a Pull Request.
-
-1. Fork the repository
-2. Create your feature branch (`git checkout -b feature/AmazingFeature`)
-3. Commit your changes (`git commit -m 'Add some AmazingFeature'`)
-4. Push to the branch (`git push origin feature/AmazingFeature`)
-5. Open a Pull Request
-
-## Star History
- 
- [![Star History Chart](https://api.star-history.com/svg?repos=LearningCircuit/local-deep-research&type=Date)](https://www.star-history.com/#LearningCircuit/local-deep-research&Date)