local-deep-research 0.4.4__py3-none-any.whl → 0.5.0__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_local.py

@@ -56,7 +56,9 @@ def _get_file_loader(file_path: str) -> Optional[BaseLoader]:
             return UnstructuredExcelLoader(str(file_path))
         else:
             # Try the text loader as a fallback for unknown extensions
-            logger.warning(f"Unknown file extension for {file_path}, trying TextLoader")
+            logger.warning(
+                f"Unknown file extension for {file_path}, trying TextLoader"
+            )
             return TextLoader(str(file_path), encoding="utf-8")
     except Exception:
         logger.exception(f"Error creating loader for {file_path}")
@@ -208,7 +210,9 @@ class LocalEmbeddingManager:
 
         # Check if vector store exists and is up to date
         if vector_store_path.exists() and not self._check_folders_modified():
-            logger.info(f"Loading existing vector store from {vector_store_path}")
+            logger.info(
+                f"Loading existing vector store from {vector_store_path}"
+            )
             try:
                 vector_store = FAISS.load_local(
                     str(vector_store_path),
@@ -306,9 +310,13 @@ class LocalEmbeddingManager:
             last_indexed = 0
             indexed_files = set()
         else:
-            last_indexed = self.indexed_folders[folder_hash].get("last_indexed", 0)
+            last_indexed = self.indexed_folders[folder_hash].get(
+                "last_indexed", 0
+            )
             indexed_files = (
-                self.indexed_folders[folder_hash].get("indexed_files", {}).keys()
+                self.indexed_folders[folder_hash]
+                .get("indexed_files", {})
+                .keys()
             )
 
         # Check if any file in the folder has been modified since last indexing
@@ -345,11 +353,15 @@ class LocalEmbeddingManager:
             self.chunk_overlap,
             self.embedding_model,
         ):
-            logger.info("Embedding configuration has changed, re-indexing folder.")
+            logger.info(
+                "Embedding configuration has changed, re-indexing folder."
+            )
             return True
         return False
 
-    def index_folder(self, folder_path: str, force_reindex: bool = False) -> bool:
+    def index_folder(
+        self, folder_path: str, force_reindex: bool = False
+    ) -> bool:
         """
         Index all documents in a folder for vector search.
 
@@ -434,7 +446,9 @@ class LocalEmbeddingManager:
         # Split documents into chunks
         logger.info(f"Splitting {len(all_docs)} documents into chunks")
         splits = self.text_splitter.split_documents(all_docs)
-        logger.info(f"Created {len(splits)} chunks from {len(modified_files)} files")
+        logger.info(
+            f"Created {len(splits)} chunks from {len(modified_files)} files"
+        )
 
         # Create vector store
         ids = []
@@ -448,10 +462,14 @@ class LocalEmbeddingManager:
         indexed_files = {}
         if folder_hash in self.indexed_folders:
             indexed_files = (
-                self.indexed_folders[folder_hash].get("indexed_files", {}).copy()
+                self.indexed_folders[folder_hash]
+                .get("indexed_files", {})
+                .copy()
             )
         for split_id, split in zip(ids, splits):
-            split_source = str(Path(split.metadata["source"]).relative_to(folder_path))
+            split_source = str(
+                Path(split.metadata["source"]).relative_to(folder_path)
+            )
             id_list = indexed_files.setdefault(split_source, [])
             id_list.append(split_id)
 
@@ -465,7 +483,8 @@ class LocalEmbeddingManager:
                 delete_paths.append(relative_path)
         if delete_ids:
             logger.info(
-                f"Deleting {len(delete_paths)} non-existent files from the " f"index."
+                f"Deleting {len(delete_paths)} non-existent files from the "
+                f"index."
             )
             self.vector_stores[folder_hash].delete(delete_ids)
         for path in delete_paths:
@@ -543,7 +562,9 @@ class LocalEmbeddingManager:
             if path.exists() and path.is_dir():
                 valid_folder_paths.append(path)
             else:
-                logger.warning(f"Skipping non-existent folder in search: {path}")
+                logger.warning(
+                    f"Skipping non-existent folder in search: {path}"
+                )
 
         # If no valid folders, return empty results
         if not valid_folder_paths:
@@ -578,8 +599,10 @@ class LocalEmbeddingManager:
             vector_store = self.vector_stores[folder_hash]
 
             try:
-                docs_with_scores = vector_store.similarity_search_with_relevance_scores(
-                    query, k=limit
+                docs_with_scores = (
+                    vector_store.similarity_search_with_relevance_scores(
+                        query, k=limit
+                    )
                 )
 
                 for doc, similarity in docs_with_scores:
@@ -685,7 +708,9 @@ class LocalSearchEngine(BaseSearchEngine):
             if os.path.exists(path) and os.path.isdir(path):
                 self.valid_folder_paths.append(path)
             else:
-                logger.warning(f"Folder not found or is not a directory: {path}")
+                logger.warning(
+                    f"Folder not found or is not a directory: {path}"
+                )
 
         # If no valid folders, log a clear message
         if not self.valid_folder_paths and paths:
@@ -768,7 +793,9 @@ class LocalSearchEngine(BaseSearchEngine):
                 if name in self.collections
             }
             if not collections_to_search:
-                logger.warning(f"No valid collections found among: {collection_names}")
+                logger.warning(
+                    f"No valid collections found among: {collection_names}"
+                )
                 return []
         else:
             # Search in all collections
@@ -811,13 +838,13 @@ class LocalSearchEngine(BaseSearchEngine):
         previews = []
         for i, result in enumerate(raw_results):
             # Create a unique ID
-            result_id = (
-                f"local-{i}-{hashlib.md5(result['content'][:50].encode()).hexdigest()}"
-            )
+            result_id = f"local-{i}-{hashlib.md5(result['content'][:50].encode()).hexdigest()}"
 
             # Extract filename and path
             source_path = result["metadata"].get("source", "Unknown")
-            filename = result["metadata"].get("filename", os.path.basename(source_path))
+            filename = result["metadata"].get(
+                "filename", os.path.basename(source_path)
+            )
 
             # Create preview snippet (first ~200 chars of content)
             snippet = (
@@ -845,10 +872,12 @@ class LocalSearchEngine(BaseSearchEngine):
                 "similarity": result["similarity"],
                 "folder": folder_path.as_posix(),
                 "collection": collection_name,
-                "collection_description": self.collections.get(collection_name, {}).get(
-                    "description", ""
-                ),
-                "_full_content": result["content"],  # Store full content for later
+                "collection_description": self.collections.get(
+                    collection_name, {}
+                ).get("description", ""),
+                "_full_content": result[
+                    "content"
+                ],  # Store full content for later
                 "_metadata": result["metadata"],  # Store metadata for later
             }
 
@@ -933,7 +962,9 @@ class LocalSearchEngine(BaseSearchEngine):
                 if collection_name in self.collections:
                     specified_collections.append(collection_name)
                     # Remove this part from the query
-                    remaining_query = remaining_query.replace(part, "", 1).strip()
+                    remaining_query = remaining_query.replace(
+                        part, "", 1
+                    ).strip()
 
         # If collections were specified in the query, they override the parameter
         if specified_collections:
@@ -1047,7 +1078,9 @@ class LocalSearchEngine(BaseSearchEngine):
         success = True
 
         for path in paths:
-            if not self.embedding_manager.index_folder(path, force_reindex=True):
+            if not self.embedding_manager.index_folder(
+                path, force_reindex=True
+            ):
                 success = False
 
         return success

local_deep_research/web_search_engines/engines/search_engine_local_all.py

@@ -37,7 +37,9 @@ class LocalAllSearchEngine(BaseSearchEngine):
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
 
         # Find all local collection search engines
@@ -76,7 +78,9 @@ class LocalAllSearchEngine(BaseSearchEngine):
         Returns:
             List of preview dictionaries
         """
-        logger.info(f"Searching across all local collections for query: {query}")
+        logger.info(
+            f"Searching across all local collections for query: {query}"
+        )
 
         all_previews = []
 
@@ -91,18 +95,24 @@ class LocalAllSearchEngine(BaseSearchEngine):
                 for preview in previews:
                     preview["collection_id"] = collection_id
                     preview["collection_name"] = engine_info["name"]
-                    preview["collection_description"] = engine_info["description"]
+                    preview["collection_description"] = engine_info[
+                        "description"
+                    ]
 
                 all_previews.extend(previews)
             except Exception:
-                logger.exception(f"Error searching collection '{collection_id}'")
+                logger.exception(
+                    f"Error searching collection '{collection_id}'"
+                )
 
         if not all_previews:
             logger.info(f"No local documents found for query: {query}")
             return []
 
         # Sort by similarity score if available
-        all_previews.sort(key=lambda x: float(x.get("similarity", 0)), reverse=True)
+        all_previews.sort(
+            key=lambda x: float(x.get("similarity", 0)), reverse=True
+        )
 
         # Limit to max_results
         return all_previews[: self.max_results]

local_deep_research/web_search_engines/engines/search_engine_pubmed.py

@@ -49,7 +49,9 @@ class PubMedSearchEngine(BaseSearchEngine):
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
         self.max_results = max(self.max_results, 25)
         self.api_key = api_key
@@ -100,7 +102,9 @@ class PubMedSearchEngine(BaseSearchEngine):
             data = response.json()
             count = int(data["esearchresult"]["count"])
 
-            logger.info("Query '%s' has %s total results in PubMed", query, count)
+            logger.info(
+                "Query '%s' has %s total results in PubMed", query, count
+            )
             return count
 
         except Exception as e:
@@ -119,10 +123,14 @@ class PubMedSearchEngine(BaseSearchEngine):
         """
         # Remove field specifications and operators
         simplified = re.sub(r"\[\w+\]", "", query)  # Remove [Field] tags
-        simplified = re.sub(r"\b(AND|OR|NOT)\b", "", simplified)  # Remove operators
+        simplified = re.sub(
+            r"\b(AND|OR|NOT)\b", "", simplified
+        )  # Remove operators
 
         # Remove quotes and parentheses
-        simplified = simplified.replace('"', "").replace("(", "").replace(")", "")
+        simplified = (
+            simplified.replace('"', "").replace("(", "").replace(")", "")
+        )
 
         # Split by whitespace and join terms with 4+ chars (likely meaningful)
         terms = [term for term in simplified.split() if len(term) >= 4]
@@ -220,7 +228,9 @@ Return ONLY the search query without any explanations.
                 optimized_query = cleaned_lines[0]
 
                 # Remove any quotes that wrap the entire query
-                if optimized_query.startswith('"') and optimized_query.endswith('"'):
+                if optimized_query.startswith('"') and optimized_query.endswith(
+                    '"'
+                ):
                     optimized_query = optimized_query[1:-1]
 
                 # Remove any explanation phrases that might be at the beginning
@@ -235,7 +245,9 @@ Return ONLY the search query without any explanations.
                         # Find the actual query part - typically after a colon
                         colon_pos = optimized_query.find(":")
                         if colon_pos > 0:
-                            optimized_query = optimized_query[colon_pos + 1 :].strip()
+                            optimized_query = optimized_query[
+                                colon_pos + 1 :
+                            ].strip()
 
                 # Check if the query still seems to contain explanations
                 if (
@@ -262,12 +274,16 @@ Return ONLY the search query without any explanations.
                             optimized_query = " ".join(query_parts)
             else:
                 # Fall back to original query if cleaning fails
-                logger.warning("Failed to extract a clean query from LLM response")
+                logger.warning(
+                    "Failed to extract a clean query from LLM response"
+                )
                 optimized_query = query
 
             # Final safety check - if query looks too much like an explanation, use original
             if len(optimized_query.split()) > 30:
-                logger.warning("Query too verbose, falling back to simpler form")
+                logger.warning(
+                    "Query too verbose, falling back to simpler form"
+                )
                 # Create a simple query from the original
                 words = [
                     w
@@ -389,7 +405,9 @@ Return ONLY the search query without any explanations.
             historical_years = [str(year) for year in range(1900, 2020)]
 
             query_lower = query.lower()
-            has_historical_term = any(term in query_lower for term in historical_terms)
+            has_historical_term = any(
+                term in query_lower for term in historical_terms
+            )
             has_past_year = any(year in query for year in historical_years)
 
             return has_historical_term or has_past_year
@@ -504,7 +522,9 @@ The default assumption should be that medical and scientific queries want RECENT
                 strategy = "no_time_filter"
         else:
             # Historical query - run without time filter
-            logger.info("Using historical search strategy without date filtering")
+            logger.info(
+                "Using historical search strategy without date filtering"
+            )
             results = self._search_pubmed(query)
 
         return results, strategy
@@ -546,14 +566,18 @@ The default assumption should be that medical and scientific queries want RECENT
             data = response.json()
             id_list = data["esearchresult"]["idlist"]
 
-            logger.info(f"PubMed search for '{query}' found {len(id_list)} results")
+            logger.info(
+                f"PubMed search for '{query}' found {len(id_list)} results"
+            )
             return id_list
 
         except Exception as e:
             logger.error(f"Error searching PubMed: {e}")
             return []
 
-    def _get_article_summaries(self, id_list: List[str]) -> List[Dict[str, Any]]:
+    def _get_article_summaries(
+        self, id_list: List[str]
+    ) -> List[Dict[str, Any]]:
         """
         Get summaries for a list of PubMed article IDs.
 
@@ -594,7 +618,9 @@ The default assumption should be that medical and scientific queries want RECENT
                     # Extract authors (if available)
                     authors = []
                     if "authors" in article:
-                        authors = [author["name"] for author in article["authors"]]
+                        authors = [
+                            author["name"] for author in article["authors"]
+                        ]
 
                     # Create summary dictionary
                     summary = {
@@ -742,7 +768,9 @@ The default assumption should be that medical and scientific queries want RECENT
                         if pmcids:
                             pmid_to_pmcid[str(pmid)] = f"PMC{pmcids[0]}"
 
-            logger.info(f"Found {len(pmid_to_pmcid)} PMC IDs for full-text access")
+            logger.info(
+                f"Found {len(pmid_to_pmcid)} PMC IDs for full-text access"
+            )
             return pmid_to_pmcid
 
         except Exception as e:
@@ -761,7 +789,12 @@ The default assumption should be that medical and scientific queries want RECENT
         """
         try:
             # Prepare parameters
-            params = {"db": "pmc", "id": pmcid, "retmode": "xml", "rettype": "full"}
+            params = {
+                "db": "pmc",
+                "id": pmcid,
+                "retmode": "xml",
+                "rettype": "full",
+            }
 
             # Add API key if available
             if self.api_key:
@@ -832,13 +865,17 @@ The default assumption should be that medical and scientific queries want RECENT
 
         # If no results, try a simplified query
         if not pmid_list:
-            logger.warning(f"No PubMed results found using strategy: {strategy}")
+            logger.warning(
+                f"No PubMed results found using strategy: {strategy}"
+            )
             simplified_query = self._simplify_query(optimized_query)
             if simplified_query != optimized_query:
                 logger.info(f"Trying with simplified query: {simplified_query}")
                 pmid_list, strategy = self._adaptive_search(simplified_query)
                 if pmid_list:
-                    logger.info(f"Simplified query found {len(pmid_list)} results")
+                    logger.info(
+                        f"Simplified query found {len(pmid_list)} results"
+                    )
 
         if not pmid_list:
             logger.warning("No PubMed results found after query simplification")
@@ -876,7 +913,9 @@ The default assumption should be that medical and scientific queries want RECENT
 
             previews.append(preview)
 
-        logger.info(f"Found {len(previews)} PubMed previews using strategy: {strategy}")
+        logger.info(
+            f"Found {len(previews)} PubMed previews using strategy: {strategy}"
+        )
         return previews
 
     def _get_full_content(
@@ -900,7 +939,9 @@ The default assumption should be that medical and scientific queries want RECENT
             logger.info("Snippet-only mode, skipping full content retrieval")
             return relevant_items
 
-        logger.info(f"Getting content for {len(relevant_items)} PubMed articles")
+        logger.info(
+            f"Getting content for {len(relevant_items)} PubMed articles"
+        )
 
         # Collect all PMIDs for relevant items
         pmids = []
@@ -938,10 +979,11 @@ The default assumption should be that medical and scientific queries want RECENT
             if (
                 pmid in pmid_to_pmcid
                 and self.get_full_text
-                and len([r for r in results if r.get("content_type") == "full_text"])
+                and len(
+                    [r for r in results if r.get("content_type") == "full_text"]
+                )
                 < self.full_text_limit
             ):
-
                 # Get full text content
                 pmcid = pmid_to_pmcid[pmid]
                 full_text = self._get_pmc_full_text(pmcid)

local_deep_research/web_search_engines/engines/search_engine_searxng.py

@@ -63,12 +63,16 @@ class SearXNGSearchEngine(BaseSearchEngine):
 
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
 
         # Validate and normalize the instance URL if provided
         self.instance_url = instance_url.rstrip("/")
-        logger.info(f"SearXNG initialized with instance URL: {self.instance_url}")
+        logger.info(
+            f"SearXNG initialized with instance URL: {self.instance_url}"
+        )
         try:
             # Make sure it's accessible.
             response = requests.get(self.instance_url, timeout=5)
@@ -97,8 +101,14 @@ class SearXNGSearchEngine(BaseSearchEngine):
         self.engines = engines
         self.language = language
         try:
-            self.safe_search = SafeSearchSetting[safe_search]
-        except KeyError:
+            # Handle both string names and integer values
+            if isinstance(safe_search, int) or (
+                isinstance(safe_search, str) and str(safe_search).isdigit()
+            ):
+                self.safe_search = SafeSearchSetting(int(safe_search))
+            else:
+                self.safe_search = SafeSearchSetting[safe_search]
+        except (ValueError, KeyError):
             logger.error(
                 "'{}' is not a valid safe search setting. Disabling safe search",
                 safe_search,
@@ -207,7 +217,9 @@ class SearXNGSearchEngine(BaseSearchEngine):
                 "Upgrade-Insecure-Requests": "1",
             }
 
-            logger.info(f"Sending request to SearXNG instance at {self.instance_url}")
+            logger.info(
+                f"Sending request to SearXNG instance at {self.instance_url}"
+            )
             response = requests.get(
                 self.search_url,
                 params=params,
@@ -237,7 +249,9 @@ class SearXNGSearchEngine(BaseSearchEngine):
                         )
                         result_elements = soup.select('div[id^="result"]')
 
-                    logger.info(f"Found {len(result_elements)} search result elements")
+                    logger.info(
+                        f"Found {len(result_elements)} search result elements"
+                    )
 
                     for idx, result_element in enumerate(result_elements):
                         if idx >= self.max_results:
@@ -264,7 +278,9 @@ class SearXNGSearchEngine(BaseSearchEngine):
                         )
 
                         title = (
-                            title_element.get_text(strip=True) if title_element else ""
+                            title_element.get_text(strip=True)
+                            if title_element
+                            else ""
                         )
 
                         url = ""
@@ -279,7 +295,11 @@ class SearXNGSearchEngine(BaseSearchEngine):
                             else ""
                         )
 
-                        if not url and title_element and title_element.has_attr("href"):
+                        if (
+                            not url
+                            and title_element
+                            and title_element.has_attr("href")
+                        ):
                             url = title_element["href"]
 
                         logger.debug(
@@ -310,7 +330,9 @@ class SearXNGSearchEngine(BaseSearchEngine):
                     logger.exception("Error parsing HTML results")
                     return []
             else:
-                logger.error(f"SearXNG returned status code {response.status_code}")
+                logger.error(
+                    f"SearXNG returned status code {response.status_code}"
+                )
                 return []
 
         except Exception:
@@ -328,7 +350,9 @@ class SearXNGSearchEngine(BaseSearchEngine):
             List of preview dictionaries
         """
         if not self.is_available:
-            logger.warning("SearXNG engine is disabled (no instance URL provided)")
+            logger.warning(
+                "SearXNG engine is disabled (no instance URL provided)"
+            )
             return []
 
         logger.info(f"Getting SearXNG previews for query: {query}")
@@ -383,7 +407,9 @@ class SearXNGSearchEngine(BaseSearchEngine):
         logger.info("Retrieving full webpage content")
 
         try:
-            results_with_content = self.full_search._get_full_content(relevant_items)
+            results_with_content = self.full_search._get_full_content(
+                relevant_items
+            )
             return results_with_content
 
         except Exception:

local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py

@@ -66,7 +66,9 @@ class SemanticScholarSearchEngine(BaseSearchEngine):
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
         )
 
         self.api_key = api_key
@@ -157,7 +159,9 @@ class SemanticScholarSearchEngine(BaseSearchEngine):
             if method.upper() == "GET":
                 response = self.session.get(url, params=params, timeout=30)
             elif method.upper() == "POST":
-                response = self.session.post(url, params=params, json=data, timeout=30)
+                response = self.session.post(
+                    url, params=params, json=data, timeout=30
+                )
             else:
                 raise ValueError(f"Unsupported HTTP method: {method}")
 
@@ -165,7 +169,9 @@ class SemanticScholarSearchEngine(BaseSearchEngine):
             if response.status_code == 429:
                 logger.warning("Rate limit exceeded, waiting and retrying...")
                 time.sleep(2.0)  # Wait longer on rate limit
-                self.rate_limit_wait *= 1.5  # Increase wait time for future requests
+                self.rate_limit_wait *= (
+                    1.5  # Increase wait time for future requests
+                )
                 return self._make_request(url, params, data, method)  # Retry
 
             response.raise_for_status()
@@ -258,7 +264,9 @@ Return ONLY the optimized search query with no explanation.
 
             params = {
                 "query": query,
-                "limit": min(self.max_results, 100),  # API limit is 100 per request
+                "limit": min(
+                    self.max_results, 100
+                ),  # API limit is 100 per request
                 "fields": ",".join(fields),
             }
 
@@ -351,15 +359,21 @@ Format each query on a new line with no numbering or explanation. Keep each quer
                     ):  # Handle various LLM response formats
                         content = response.content
                         alt_queries = [
-                            q.strip() for q in content.strip().split("\n") if q.strip()
+                            q.strip()
+                            for q in content.strip().split("\n")
+                            if q.strip()
                         ]
                     elif isinstance(response, str):
                         alt_queries = [
-                            q.strip() for q in response.strip().split("\n") if q.strip()
+                            q.strip()
+                            for q in response.strip().split("\n")
+                            if q.strip()
                         ]
 
                     # Try each alternative query
-                    for alt_query in alt_queries[:3]:  # Limit to first 3 alternatives
+                    for alt_query in alt_queries[
+                        :3
+                    ]:  # Limit to first 3 alternatives
                         logger.info("Trying LLM-suggested query: %s", alt_query)
                         alt_papers = self._direct_search(alt_query)
 
@@ -495,7 +509,9 @@ Format each query on a new line with no numbering or explanation. Keep each quer
                 snippet = ""
                 if abstract:
                     snippet = (
-                        abstract[:250] + "..." if len(abstract) > 250 else abstract
+                        abstract[:250] + "..."
+                        if len(abstract) > 250
+                        else abstract
                     )
 
                 venue = paper.get("venue", "")
@@ -597,7 +613,9 @@ Format each query on a new line with no numbering or explanation. Keep each quer
 
                     # Add fields of study
                     if "fieldsOfStudy" in paper_details:
-                        result["fields_of_study"] = paper_details["fieldsOfStudy"]
+                        result["fields_of_study"] = paper_details[
+                            "fieldsOfStudy"
+                        ]
 
             # Remove temporary fields
             if "_paper_id" in result: