local-deep-research 0.1.0__py3-none-any.whl

local_deep_research/__init__.py

@@ -0,0 +1,24 @@
+"""
+Local Deep Research - AI-powered research assistant
+
+A powerful AI research system with iterative analysis capabilities
+and multiple search engines integration.
+"""
+
+__version__ = "0.1.0"
+
+# Initialize configuration on module import
+from .utilties.setup_utils import setup_user_directories
+
+# Import main components
+from .search_system import AdvancedSearchSystem
+from .report_generator import IntegratedReportGenerator
+from .config import get_llm, get_search
+
+# Export it
+__all__ = [
+    "AdvancedSearchSystem", 
+    "IntegratedReportGenerator",
+    "get_llm",
+    "get_search"
+]

local_deep_research/citation_handler.py

@@ -0,0 +1,113 @@
+# citation_handler.py
+
+from langchain_core.documents import Document
+from typing import Dict, List, Union, Any
+import re
+from .utilties.search_utilities import remove_think_tags
+from .config import settings
+
+class CitationHandler:
+    def __init__(self, llm):
+        self.llm = llm
+
+    def _create_documents(
+        self, search_results: Union[str, List[Dict]], nr_of_links: int = 0
+    ) -> List[Document]:
+        """Convert search results to LangChain documents format and add index to original search results."""
+        documents = []
+        if isinstance(search_results, str):
+            return documents
+
+        for i, result in enumerate(search_results):
+            if isinstance(result, dict):
+                # Add index to the original search result dictionary
+                result["index"] = str(i + nr_of_links + 1)
+                
+                content = result.get("full_content", result.get("snippet", ""))
+                documents.append(
+                    Document(
+                        page_content=content,
+                        metadata={
+                            "source": result.get("link", f"source_{i+1}"),
+                            "title": result.get("title", f"Source {i+1}"),
+                            "index": i + nr_of_links + 1,
+                        },
+                    )
+                )
+        return documents
+
+    def _format_sources(self, documents: List[Document]) -> str:
+        """Format sources with numbers for citation."""
+        sources = []
+        for doc in documents:
+            source_id = doc.metadata["index"]
+            sources.append(f"[{source_id}] {doc.page_content}")
+        return "\n\n".join(sources)
+
+    def analyze_initial(
+        self, query: str, search_results: Union[str, List[Dict]]
+    ) -> Dict[str, Any]:
+
+        documents = self._create_documents(search_results)
+        formatted_sources = self._format_sources(documents)
+        print(formatted_sources)
+        prompt = f"""Analyze the following information concerning the question and include citations using numbers in square brackets [1], [2], etc. When citing, use the source number provided at the start of each source.
+
+Question: {query}
+
+Sources:
+{formatted_sources}
+
+Provide a detailed analysis with citations and always keep URLS. Never make up sources. Example format: "According to the research [1], ..."
+"""
+
+        response = self.llm.invoke(prompt)
+
+        return {"content": remove_think_tags(response.content), "documents": documents}
+
+    def analyze_followup(
+        self,
+        question: str,
+        search_results: Union[str, List[Dict]],
+        previous_knowledge: str,
+        nr_of_links : int
+    ) -> Dict[str, Any]:
+        """Process follow-up analysis with citations."""
+        documents = self._create_documents(search_results, nr_of_links=nr_of_links)
+        formatted_sources = self._format_sources(documents)
+        print(formatted_sources)
+        # Add fact-checking step
+        fact_check_prompt = f"""Analyze these sources for factual consistency:
+        1. Cross-reference major claims between sources
+        2. Identify and flag any contradictions
+        3. Verify basic facts (dates, company names, ownership)
+        4. Note when sources disagree
+        
+        Previous Knowledge:
+        {previous_knowledge}
+
+        New Sources:
+        {formatted_sources}
+
+        Return any inconsistencies or conflicts found."""
+        if settings.GENERAL.ENABLE_FACT_CHECKING:
+            fact_check_response = remove_think_tags(self.llm.invoke(fact_check_prompt).content)
+        else:
+            fact_check_response = ""
+
+        prompt = f"""Using the previous knowledge and new sources, answer the question. Include citations using numbers in square brackets [1], [2], etc. When citing, use the source number provided at the start of each source. Reflect information from sources critically.
+
+            Previous Knowledge:
+            {previous_knowledge}
+
+            Question: {question}
+
+            New Sources:
+            {formatted_sources}
+            Reflect information from sources critically based on: {fact_check_response}. Never invent sources.
+            Provide a detailed answer with citations.  Example format: "According to [1], ..."
+            """
+
+        response = self.llm.invoke(prompt)
+
+        return {"content": remove_think_tags(response.content), "documents": documents}

local_deep_research/config.py

@@ -0,0 +1,166 @@
+# local_deep_research/config.py
+from dynaconf import Dynaconf
+from pathlib import Path
+import logging
+from platformdirs import user_documents_dir
+import os
+# Setup logging
+logger = logging.getLogger(__name__)
+
+# Get config directory
+def get_config_dir():
+    from platformdirs import user_config_dir
+    config_dir = Path(user_config_dir("local_deep_research", "LearningCircuit"))
+    print(f"Looking for config in: {config_dir}")
+    return config_dir
+
+# Define config paths
+CONFIG_DIR = get_config_dir() / "config"
+CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+SETTINGS_FILE = CONFIG_DIR / "settings.toml"
+SECRETS_FILE = CONFIG_DIR / ".secrets.toml"
+LLM_CONFIG_FILE = CONFIG_DIR / "llm_config.py"
+SEARCH_ENGINES_FILE = CONFIG_DIR / "search_engines.toml"
+
+LOCAL_COLLECTIONS_FILE = CONFIG_DIR / "local_collections.toml"
+print("CONFIGDIR:", CONFIG_DIR)
+print("SECRETS_FILE:", SECRETS_FILE)
+print("SETTINGS_FILE:", SETTINGS_FILE)
+
+
+# Set environment variable for Dynaconf to use
+docs_base = Path(user_documents_dir()) / "local_deep_research"
+os.environ["DOCS_DIR"] = str(docs_base)
+
+
+
+
+
+
+
+
+# Expose get_llm function
+def get_llm(*args, **kwargs):
+    """
+    Helper function to get LLM from llm_config.py
+    """
+    # Import here to avoid circular imports
+    import importlib.util
+    import sys
+    
+    llm_config_path = CONFIG_DIR / "llm_config.py"
+    
+    # If llm_config.py exists, use it
+    if llm_config_path.exists():
+        if str(CONFIG_DIR) not in sys.path:
+            sys.path.insert(0, str(CONFIG_DIR))
+            
+        spec = importlib.util.spec_from_file_location("llm_config", llm_config_path)
+        llm_config = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(llm_config)
+        
+        if hasattr(llm_config, "get_llm"):
+            return llm_config.get_llm(*args, **kwargs)
+    
+    # Fallback to utility function
+    from .utilties.llm_utils import get_model
+    return get_model(*args, **kwargs)
+
+# Expose get_search function
+def get_search(search_tool=None):
+    """
+    Helper function to get search engine
+    """
+    
+    # Use specified tool or default from settings
+    tool = search_tool or settings.search.tool
+    logger.info(f"Search tool is: {tool}")
+    
+    # Import here to avoid circular imports
+    from .web_search_engines.search_engine_factory import get_search as factory_get_search
+    
+    # Get search parameters
+    params = {
+        "search_tool": tool,
+        "llm_instance": get_llm(),
+        "max_results": settings.get("max_results"),
+        "region": settings.get("region"),
+        "time_period": settings.get("time_period"),
+        "safe_search": settings.get("safe_search"),
+        "search_snippets_only": settings.get("snippets_only"),
+        "search_language": settings.get("search_language"),
+        "max_filtered_results": settings.get("max_filtered_results")
+    }
+    
+    # Create and return search engine
+    return factory_get_search(**params)
+
+def init_config_files():
+    """Initialize config files if they don't exist"""
+    import shutil
+    from importlib.resources import files
+    
+    # Get default files path
+    try:
+        defaults_dir = files('local_deep_research.defaults')
+    except ImportError:
+        # Fallback for older Python versions
+        from pkg_resources import resource_filename
+        defaults_dir = Path(resource_filename('local_deep_research', 'defaults'))
+    
+    # Create settings.toml if it doesn't exist
+    settings_file = CONFIG_DIR / "settings.toml"
+    if not settings_file.exists():
+        shutil.copy(defaults_dir / "main.toml", settings_file)
+        logger.info(f"Created settings.toml at {settings_file}")
+    
+    # Create llm_config.py if it doesn't exist
+    llm_config_file = CONFIG_DIR / "llm_config.py"
+    if not llm_config_file.exists():
+        shutil.copy(defaults_dir / "llm_config.py", llm_config_file)
+        logger.info(f"Created llm_config.py at {llm_config_file}")
+        
+    # Create local_collections.toml if it doesn't exist
+    collections_file = CONFIG_DIR / "local_collections.toml"
+    if not collections_file.exists():
+        shutil.copy(defaults_dir / "local_collections.toml", collections_file)
+        logger.info(f"Created local_collections.toml at {collections_file}")
+    
+    # Create search_engines.toml if it doesn't exist
+    search_engines_file = CONFIG_DIR / "search_engines.toml"
+    if not search_engines_file.exists():
+        shutil.copy(defaults_dir / "search_engines.toml", search_engines_file)
+        logger.info(f"Created search_engines.toml at {search_engines_file}")
+        
+    secrets_file = CONFIG_DIR / ".secrets.toml"
+    if not secrets_file.exists():
+        with open(secrets_file, "w") as f:
+            f.write("""
+# ANTHROPIC_API_KEY = "your-api-key-here"
+# OPENAI_API_KEY = "your-openai-key-here"
+# GOOGLE_API_KEY = "your-google-key-here"
+# SERP_API_KEY = "your-api-key-here"
+# GUARDIAN_API_KEY = "your-api-key-here"
+# GOOGLE_PSE_API_KEY = "your-google-api-key-here"
+# GOOGLE_PSE_ENGINE_ID = "your-programmable-search-engine-id-here"
+""")
+        
+# Initialize config files on import
+init_config_files()
+
+# Use an absolute path to your .secrets.toml for testing
+secrets_file = Path(SECRETS_FILE)
+
+settings = Dynaconf(
+    settings_files=[
+        str(SETTINGS_FILE),
+        str(LOCAL_COLLECTIONS_FILE),
+        str(SEARCH_ENGINES_FILE),  
+    ],
+    secrets=str(SECRETS_FILE),
+    env_prefix="LDR",
+    load_dotenv=True,
+    envvar_prefix="LDR",
+    env_file=str(CONFIG_DIR / ".env"),
+)
+

local_deep_research/defaults/__init__.py

@@ -0,0 +1,44 @@
+"""
+Default configuration module for Local Deep Research.
+
+This module is responsible for loading and initializing default
+configuration files and resources used throughout the application.
+"""
+import os
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Define the path to the package's defaults directory
+DEFAULTS_DIR = Path(__file__).parent
+
+# Default files available in this package
+DEFAULT_FILES = {
+    "main.toml": DEFAULTS_DIR / "main.toml",
+    "local_collections.toml": DEFAULTS_DIR / "local_collections.toml",
+    "search_engines.toml": DEFAULTS_DIR / "search_engines.toml",
+    "llm_config.py": DEFAULTS_DIR / "llm_config.py"
+}
+
+def get_default_file_path(filename):
+    """Get the path to a default configuration file."""
+    if filename in DEFAULT_FILES:
+        return DEFAULT_FILES[filename]
+    return None
+
+def list_default_files():
+    """List all available default configuration files."""
+    return list(DEFAULT_FILES.keys())
+
+def ensure_defaults_exist():
+    """Verify that all expected default files exist in the package."""
+    missing = []
+    for filename, filepath in DEFAULT_FILES.items():
+        if not filepath.exists():
+            missing.append(filename)
+    
+    if missing:
+        logger.warning(f"The following default files are missing from the package: {', '.join(missing)}")
+        return False
+    return True

local_deep_research/defaults/llm_config.py

@@ -0,0 +1,269 @@
+"""
+LLM configuration for Local Deep Research.
+
+This file controls which language models are available and how they're configured.
+You can customize model selection, parameters, and fallbacks here.
+"""
+
+from langchain_anthropic import ChatAnthropic
+from langchain_openai import ChatOpenAI
+from langchain_ollama import ChatOllama
+from langchain_community.llms import VLLM
+from local_deep_research.config import settings
+import os
+import logging
+from enum import Enum, auto
+
+# Initialize environment
+logger = logging.getLogger(__name__)
+
+# Provider enum
+class ModelProvider(Enum):
+    OLLAMA = auto()
+    OPENAI = auto()
+    ANTHROPIC = auto()
+    VLLM = auto()
+    OPENAI_ENDPOINT = auto()
+    NONE = auto()
+
+# ================================
+# USER CONFIGURATION SECTION
+# ================================
+
+# Set your preferred model provider here
+DEFAULT_PROVIDER = ModelProvider.OLLAMA  # Change this to your preferred provider
+
+# Set your default model name here
+DEFAULT_MODEL = "mistral"  # Your default model
+
+# Set default model parameters
+DEFAULT_TEMPERATURE = 0.7
+MAX_TOKENS = 30000
+
+# Server URLs
+OPENAI_ENDPOINT_URL = "https://openrouter.ai/api/v1"  # For OpenRouter or compatible services
+OLLAMA_BASE_URL = "http://localhost:11434"  # URL for Ollama server
+
+
+
+
+# ================================
+# LLM FUNCTIONS
+# ================================
+
+
+
+    
+
+def get_llm(model_name=None, temperature=None, provider=None):
+    """
+    Get LLM instance based on model name and provider.
+    
+    Args:
+        model_name: Name of the model to use (if None, uses DEFAULT_MODEL)
+        temperature: Model temperature (if None, uses DEFAULT_TEMPERATURE)
+        provider: Provider to use (if None, uses DEFAULT_PROVIDER)
+    
+    Returns:
+        A LangChain LLM instance
+    """
+    if model_name is None:
+        model_name = DEFAULT_MODEL
+    
+    if temperature is None:
+        temperature = DEFAULT_TEMPERATURE
+        
+    if provider is None:
+        provider = DEFAULT_PROVIDER
+    
+    # Common parameters for all models
+    common_params = {
+        "temperature": temperature,
+        "max_tokens": MAX_TOKENS,
+    }
+    
+    # Handle different providers
+    if provider == ModelProvider.ANTHROPIC:
+        api_key = settings.get('ANTHROPIC_API_KEY', '')
+        if not api_key:
+            logger.warning("ANTHROPIC_API_KEY not found. Falling back to default model.")
+            return get_fallback_model(temperature)
+        
+        return ChatAnthropic(
+            model=model_name, anthropic_api_key=api_key, **common_params
+        )
+    
+    elif provider == ModelProvider.OPENAI:
+        api_key = settings.get('OPENAI_API_KEY', '')
+        if not api_key:
+            logger.warning("OPENAI_API_KEY not found. Falling back to default model.")
+            return get_fallback_model(temperature)
+        
+        return ChatOpenAI(model=model_name, api_key=api_key, **common_params)
+    
+    elif provider == ModelProvider.OPENAI_ENDPOINT:
+        api_key = settings.OPENAI_ENDPOINT_API_KEY
+
+        if not api_key:
+            logger.warning("OPENAI_ENDPOINT_API_KEY not found. Falling back to default model.")
+            return get_fallback_model(temperature)
+        
+        return ChatOpenAI(
+            model=model_name, 
+            api_key=api_key,
+            openai_api_base=OPENAI_ENDPOINT_URL, 
+            **common_params
+        )
+    
+    elif provider == ModelProvider.VLLM:
+        try:
+            return VLLM(
+                model=model_name,
+                trust_remote_code=True,
+                max_new_tokens=128,
+                top_k=10,
+                top_p=0.95,
+                temperature=temperature,
+            )
+        except Exception as e:
+            logger.error(f"Error loading VLLM model: {e}")
+            logger.warning("Falling back.")
+            return get_fallback_model(temperature)
+    
+    elif provider == ModelProvider.OLLAMA:
+        try:
+            # Use the configurable Ollama base URL
+            base_url = settings.get('OLLAMA_BASE_URL', OLLAMA_BASE_URL)
+            return ChatOllama(model=model_name, base_url=base_url, **common_params)
+        except Exception as e:
+            logger.error(f"Error loading Ollama model: {e}")
+            return get_fallback_model(temperature)
+    
+    else:
+        return get_fallback_model(temperature)
+
+def get_fallback_model(temperature=DEFAULT_TEMPERATURE):
+    """Create a dummy model for when no providers are available"""
+    from langchain_community.llms.fake import FakeListLLM
+    return FakeListLLM(
+        responses=["No language models are available. Please install Ollama or set up API keys."]
+    )
+
+# ================================
+# COMPATIBILITY FUNCTIONS
+# ================================
+
+def get_available_provider_types():
+    """Return available model providers"""
+    providers = {}
+    
+    if is_ollama_available():
+        providers["ollama"] = "Ollama (local models)"
+    
+    if is_openai_available():
+        providers["openai"] = "OpenAI API"
+    
+    if is_anthropic_available():
+        providers["anthropic"] = "Anthropic API"
+    
+    if is_openai_endpoint_available():
+        providers["openai_endpoint"] = "OpenAI-compatible Endpoint"
+    
+    # Check for VLLM capability
+    try:
+        import torch
+        import transformers
+        providers["vllm"] = "VLLM (local models)"
+    except ImportError:
+        pass
+    
+    # Default fallback
+    if not providers:
+        providers["none"] = "No model providers available"
+    
+    return providers
+
+
+
+
+# ================================
+# HELPER FUNCTIONS
+# ================================
+
+def is_openai_available():
+    """Check if OpenAI is available"""
+    try:
+        api_key = settings.api_keys.get('OPENAI_API_KEY', '')
+        return bool(api_key)
+    except:
+        return False
+
+def is_anthropic_available():
+    """Check if Anthropic is available"""
+    try:
+        api_key = settings.api_keys.get('ANTHROPIC_API_KEY', '')
+        return bool(api_key)
+    except:
+        return False
+
+
+
+def is_openai_endpoint_available():
+    """Check if OpenAI endpoint is available"""
+    print(os.getenv("OPENAI_ENDPOINT_API_KEY"))
+    try:
+        api_key = settings.OPENAI_ENDPOINT_API_KEY
+        return bool(api_key) 
+    except:
+        return False
+
+def is_ollama_available():
+    """Check if Ollama is running"""
+    try:
+        import requests
+        base_url = settings.get('OLLAMA_BASE_URL', OLLAMA_BASE_URL)
+        response = requests.get(f"{base_url}/api/tags", timeout=1.0)
+        return response.status_code == 200
+    except:
+        return False
+
+def is_vllm_available():
+    """Check if VLLM capability is available"""
+    try:
+        import torch
+        import transformers
+        return True
+    except ImportError:
+        return False
+
+def get_available_providers():
+    """Get dictionary of available providers"""
+    providers = {}
+    
+    if is_ollama_available():
+        providers[ModelProvider.OLLAMA] = "Ollama (local models)"
+    
+    if is_openai_available():
+        providers[ModelProvider.OPENAI] = "OpenAI API"
+    
+    if is_anthropic_available():
+        providers[ModelProvider.ANTHROPIC] = "Anthropic API"
+    
+    if is_openai_endpoint_available():
+        providers[ModelProvider.OPENAI_ENDPOINT] = "OpenAI-compatible Endpoint"
+    
+    if is_vllm_available():
+        providers[ModelProvider.VLLM] = "VLLM (local models)"
+    
+    if not providers:
+        providers[ModelProvider.NONE] = "No model providers available"
+    
+    return providers
+
+# Log which providers are available
+AVAILABLE_PROVIDERS = get_available_providers()
+logger.info(f"Available providers: {[p.name for p in AVAILABLE_PROVIDERS.keys()]}")
+
+# Check if selected provider is available
+if DEFAULT_PROVIDER not in AVAILABLE_PROVIDERS and DEFAULT_PROVIDER != ModelProvider.NONE:
+    logger.warning(f"Selected provider {DEFAULT_PROVIDER.name} is not available.")

local_deep_research/defaults/local_collections.toml

@@ -0,0 +1,47 @@
+# Default local document collections configuration
+# Each collection functions as an independent search engine
+
+# Project Documents Collection
+[project_docs]
+name = "Project Documents"
+description = "Project documentation and specifications"
+paths = ["@format ${DOCS_DIR}/project_documents"]
+enabled = true
+embedding_model = "all-MiniLM-L6-v2"
+embedding_device = "cpu"
+embedding_model_type = "sentence_transformers"
+max_results = 20
+max_filtered_results = 5
+chunk_size = 1000
+chunk_overlap = 200
+cache_dir = "__CACHE_DIR__/local_search/project_docs"
+
+# Research Papers Collection
+[research_papers]
+name = "Research Papers"
+description = "Academic research papers and articles"
+paths = ["@format ${DOCS_DIR}/research_papers"]
+enabled = true
+embedding_model = "all-MiniLM-L6-v2"
+embedding_device = "cpu"
+embedding_model_type = "sentence_transformers"
+max_results = 20
+max_filtered_results = 5
+chunk_size = 800
+chunk_overlap = 150
+cache_dir = "__CACHE_DIR__/local_search/research_papers"
+
+# Personal Notes Collection
+[personal_notes]
+name = "Personal Notes"
+description = "Personal notes and documents"
+paths = ["@format ${DOCS_DIR}/personal_notes"]
+enabled = true
+embedding_model = "all-MiniLM-L6-v2"
+embedding_device = "cpu"
+embedding_model_type = "sentence_transformers"
+max_results = 30
+max_filtered_results = 10
+chunk_size = 500
+chunk_overlap = 100
+cache_dir = "__CACHE_DIR__/local_search/personal_notes"