async-web-search 0.1.0__tar.gz

async_web_search-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+__pycache__
+.venv/
+venv/
+*.pyc
+
+__ruff_cache__/

async_web_search-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Your Name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

async_web_search-0.1.0/PKG-INFO

@@ -0,0 +1,59 @@
+Metadata-Version: 2.3
+Name: async-web-search
+Version: 0.1.0
+Summary: Async web search library supporting Google, Wikipedia, and arXiv
+Project-URL: Homepage, https://github.com/nwaughachukwuma/web-search
+Project-URL: Bug Tracker, https://github.com/nwaughachukwuma/web-search/issues
+Author: Chukwuma
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: asyncio
+Requires-Dist: beautifulsoup4
+Requires-Dist: httpx
+Requires-Dist: lxml
+Requires-Dist: wikipedia
+Description-Content-Type: text/markdown
+
+# Web Search
+
+Async web search library supporting Google Custom Search, Wikipedia, and arXiv APIs.
+
+## Installation
+
+```bash
+pip install web-search
+```
+
+## Usage
+
+```python
+from web_search import WebSearch, WebSearchConfig, GoogleSearchConfig
+
+config = WebSearchConfig(
+    sources=["google", "wikipedia"],
+    google_config=GoogleSearchConfig(
+        api_key="your_api_key",
+        cse_id="your_cse_id"
+    )
+)
+
+search = WebSearch(config)
+results = await search.search("quantum computing")
+print(results)
+```
+
+## Features
+
+- Async/concurrent searching
+- Multiple source support (Google, Wikipedia, arXiv)
+- Content extraction and cleaning
+- Configurable search parameters
+
+## License
+
+MIT

async_web_search-0.1.0/README.md

@@ -0,0 +1,38 @@
+# Web Search
+
+Async web search library supporting Google Custom Search, Wikipedia, and arXiv APIs.
+
+## Installation
+
+```bash
+pip install web-search
+```
+
+## Usage
+
+```python
+from web_search import WebSearch, WebSearchConfig, GoogleSearchConfig
+
+config = WebSearchConfig(
+    sources=["google", "wikipedia"],
+    google_config=GoogleSearchConfig(
+        api_key="your_api_key",
+        cse_id="your_cse_id"
+    )
+)
+
+search = WebSearch(config)
+results = await search.search("quantum computing")
+print(results)
+```
+
+## Features
+
+- Async/concurrent searching
+- Multiple source support (Google, Wikipedia, arXiv)
+- Content extraction and cleaning
+- Configurable search parameters
+
+## License
+
+MIT

async_web_search-0.1.0/pyproject.toml

@@ -0,0 +1,64 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "async-web-search"
+requires-python = ">=3.9"
+version = "0.1.0"
+description = "Async web search library supporting Google, Wikipedia, and arXiv"
+
+readme = "README.md"
+license = "MIT"
+authors = [
+  { name="Chukwuma" },
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
+    "Topic :: Software Development :: Libraries :: Python Modules"
+]
+dependencies = [
+    "asyncio",
+    "httpx",
+    "beautifulsoup4",
+    "wikipedia",
+    "lxml"
+]
+
+[project.urls]
+"Homepage" = "https://github.com/nwaughachukwuma/web-search"
+"Bug Tracker" = "https://github.com/nwaughachukwuma/web-search/issues"
+
+[tool.hatch.build]
+include = [
+    "src/**/*.py",
+    "LICENSE",
+    "README.md"
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/web_search"]
+
+[tool.ruff]
+line-length = 120
+include = ["src/**/*.(py|pyi)", "tests/**/*.(py|pyi)", "pyproject.toml"]
+
+[tool.ruff.lint]
+select = ["D102", "F401"]
+
+[tool.ruff.lint.isort]
+case-sensitive = true
+relative-imports-order = "closest-to-furthest"
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.pylint]
+max-line-length = 120
+
+[tool.mypy]
+ignore_missing_imports = false

async_web_search-0.1.0/src/web_search/__init__.py

@@ -0,0 +1,17 @@
+from .search import WebSearch
+from .config import (
+    WebSearchConfig,
+    GoogleSearchConfig,
+    KnowledgeSearchConfig,
+    SearchSources,
+    SearchResult,
+)
+
+__all__ = [
+    "WebSearch",
+    "WebSearchConfig",
+    "GoogleSearchConfig",
+    "KnowledgeSearchConfig",
+    "SearchSources",
+    "SearchResult",
+]

async_web_search-0.1.0/src/web_search/config.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass, field
+from typing import Literal
+
+SearchSources = Literal["google", "wikipedia", "arxiv"]
+
+
+@dataclass
+class GoogleSearchConfig:
+    api_key: str
+    cse_id: str
+    max_results: int = 3
+    app_domain: str | None = None
+
+
+@dataclass
+class KnowledgeSearchConfig:
+    max_results: int = 3
+    max_sources: int = 10
+    max_preview_chars: int = 1024
+
+
+@dataclass
+class WebSearchConfig:
+    sources: list[SearchSources] = field(default_factory=lambda: ["google"])
+    google_config: GoogleSearchConfig | None = None
+    knowledge_config: KnowledgeSearchConfig | None = None
+
+
+@dataclass
+class SearchResult:
+    url: str
+    title: str
+    preview: str
+
+    def __str__(self):
+        return f"Title: {self.title}\nPreview: {self.preview}"

async_web_search-0.1.0/src/web_search/google.py

@@ -0,0 +1,143 @@
+import asyncio
+from typing import Any, Coroutine, Dict, List
+from urllib.parse import unquote
+
+import httpx
+from bs4 import BeautifulSoup
+
+from .config import GoogleSearchConfig, SearchResult
+
+GOOGLE_SEARCH_URL = "https://www.googleapis.com/customsearch/v1"
+
+
+class GoogleSearch:
+    config: GoogleSearchConfig
+
+    def __init__(self, config: GoogleSearchConfig | None = None):
+        self.config = config if config else GoogleSearchConfig()
+
+    async def _compile_google_search(self, query: str):
+        results = await self._google_search(query)
+        return "\n\n".join(str(item) for item in results if item.preview)
+
+    async def _google_search(self, query: str, **kwargs):
+        """
+        Perform a Google search using the Custom Search Engine API
+        """
+        params = {
+            "q": unquote(query),
+            "key": self.config.api_key,
+            "cx": self.config.cse_id,
+            "num": 5,
+        }
+        params.update(kwargs)
+        headers = {"Referer": self.config.app_domain}
+
+        async with httpx.AsyncClient() as client:
+            response = await client.get(
+                GOOGLE_SEARCH_URL, params=params, headers=headers
+            )
+            response.raise_for_status()
+
+            json_data = response.json()
+
+        items = json_data.get("items", [])[: self.config.max_results]
+        result = await self.extract_relevant_items(items)
+        return result
+
+    async def extract_relevant_items(
+        self, search_results: List[Dict[str, Any]]
+    ) -> List[SearchResult]:
+        """
+        Extract relevant items from the search results
+        """
+        tasks: list[Coroutine[Any, Any, SearchResult | None]] = []
+
+        for item in search_results:
+            url = item.get("link")
+            if url and self._is_valid_url(url):
+                tasks.append(self._process_search_item(url, item))
+
+        if not tasks:
+            return []
+
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        return [item for item in results if isinstance(item, SearchResult)]
+
+    def _is_valid_url(self, url: str) -> bool:
+        invalid_extensions = (
+            ".pdf",
+            ".doc",
+            ".docx",
+            ".ppt",
+            ".pptx",
+            ".xls",
+            ".xlsx",
+            ".zip",
+            ".rar",
+        )
+        invalid_domains = ("youtube.com", "vimeo.com", "facebook.com", "twitter.com")
+        return not (
+            url.endswith(invalid_extensions)
+            or any(domain in url for domain in invalid_domains)
+        )
+
+    async def _process_search_item(
+        self, url: str, item: Dict, char_limit=2000
+    ) -> SearchResult | None:
+        """
+        Process and fetch the result of a single search item url
+        """
+        try:
+            content = await self._scrape_page_content(url)
+            return SearchResult(
+                url=url, title=item.get("title", ""), preview=content[:char_limit]
+            )
+        except Exception:
+            return None
+
+    async def _scrape_page_content(self, url: str) -> str:
+        """
+        Fetch and extract content from a webpage
+        """
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.get(url)
+                response.raise_for_status()
+
+            soup = BeautifulSoup(response.text, "lxml")
+            # Remove unwanted elements
+            for element in soup.find_all(
+                ["script", "style", "nav", "header", "footer", "ads"]
+            ):
+                element.decompose()
+
+            content_elements = soup.find_all(
+                ["article", "main", "div"],
+                class_=["content", "article", "post", "entry", "main-content"],
+            )
+
+            if not content_elements:
+                # Fallback to paragraph extraction if no main content container found
+                content_elements = soup.find_all("p")
+
+            # Extract text from found elements
+            content = "\n".join(
+                element.get_text(strip=True)
+                for element in content_elements
+                if element.get_text(strip=True)
+            )
+
+            # If still no content, try getting all text
+            if not content:
+                content = soup.get_text(strip=True)
+
+            return self._clean_content(content)
+        except Exception:
+            return ""
+
+    def _clean_content(self, content: str) -> str:
+        """Remove very short lines (likely navigation/menu items)"""
+        content = " ".join(content.split())
+        lines = [line for line in content.split("\n") if len(line) > 30]
+        return "\n".join(lines)

async_web_search-0.1.0/src/web_search/knowledge.py

@@ -0,0 +1,137 @@
+import asyncio
+
+import httpx
+import wikipedia
+from bs4 import BeautifulSoup
+from .config import KnowledgeSearchConfig, SearchResult
+
+
+class KnowledgeSearch:
+    config: KnowledgeSearchConfig
+
+    def __init__(self, config: KnowledgeSearchConfig | None = None):
+        self.config = config if config else KnowledgeSearchConfig()
+
+    async def fetch_knowledge(self, query: str):
+        """
+        Fetch knowledge from multiple sources concurrently,
+        including Wikipedia, arXiv, and other scientific sources
+        """
+        # listed in order of importance
+        tasks = [
+            self._search_wikipedia(query),
+            self._search_arxiv_papers(query),
+            # add more knowledge sources here
+        ]
+
+        sources: list[SearchResult] = []
+
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        for result in results:
+            if isinstance(result, list):
+                sources.extend(result)
+
+        sources = sources[: self.config.max_sources]
+        return "\n\n".join(str(source) for source in sources if source.preview)
+
+    async def _compile_wikipedia(self, query: str) -> str:
+        results = await self._search_wikipedia(query)
+        return "\n\n".join(str(item) for item in results)
+
+    async def _compile_arxiv_papers(self, query: str) -> str:
+        results = await self._search_arxiv_papers(query)
+        return "\n\n".join(str(item) for item in results)
+
+    async def _search_wikipedia(self, query: str) -> list[SearchResult]:
+        """
+        Fetch relevant Wikipedia articles
+        """
+        try:
+            sources: list[SearchResult] = []
+            search_results = wikipedia.search(query, results=self.config.max_results)
+
+            for title in search_results:
+                try:
+                    page = wikipedia.page(title)
+                    if not page.content:
+                        continue
+
+                    preview = self._extract_relevant_wiki_sections(page.content)
+                    if not preview:
+                        continue
+
+                    sources.append(
+                        SearchResult(url=page.url, title=page.title, preview=preview)
+                    )
+                except wikipedia.exceptions.DisambiguationError:
+                    continue
+                except wikipedia.exceptions.PageError:
+                    continue
+
+            return sources
+        except Exception:
+            return []
+
+    async def _search_arxiv_papers(self, query: str) -> list[SearchResult]:
+        """
+        Fetch papers from arXiv and other scientific sources
+        """
+        ARXIV_URL = "http://export.arxiv.org/api/query"
+        try:
+            params = {
+                "search_query": f"all:{query}",
+                "start": 0,
+                "max_results": self.config.max_results,
+                "sortBy": "relevance",
+                "sortOrder": "descending",
+            }
+            async with httpx.AsyncClient(timeout=20) as client:
+                response = await client.get(ARXIV_URL, params=params)
+                response.raise_for_status()
+
+            soup = BeautifulSoup(response.text, "lxml-xml")
+            entries = soup.find_all("entry")
+
+            sources: list[SearchResult] = []
+            for entry in entries:
+                title = entry.title.text.strip()
+                url = entry.id.text.strip()
+                preview = entry.summary.text.strip()
+
+                if not preview:
+                    continue
+
+                sources.append(SearchResult(url=url, title=title, preview=preview))
+
+            return sources
+        except Exception:
+            return []
+
+    def _extract_relevant_wiki_sections(self, content: str) -> str:
+        """
+        Extract the most relevant sections from Wikipedia content
+        """
+        paragraphs = content.split("\n\n")
+        # Remove references and other metadata
+        cleaned_paragraphs = [
+            p
+            for p in paragraphs
+            if not any(
+                marker in p.lower()
+                for marker in [
+                    "references",
+                    "external links",
+                    "see also",
+                    "== notes ==",
+                ]
+            )
+        ]
+
+        result = ""
+        for p in cleaned_paragraphs:
+            if len(result + p) <= self.config.max_preview_chars:
+                result += p + "\n\n"
+            else:
+                break
+
+        return result.strip()

async_web_search-0.1.0/src/web_search/search.py

@@ -0,0 +1,31 @@
+import asyncio
+from typing import Any, Coroutine, List
+
+from .google import GoogleSearch
+from .knowledge import KnowledgeSearch
+from .config import WebSearchConfig
+
+
+class WebSearch(GoogleSearch, KnowledgeSearch):
+    def __init__(self, config: WebSearchConfig | None = None):
+        self.config = config if config else WebSearchConfig()
+
+        self.sources = self.config.sources
+        GoogleSearch.__init__(self, self.config.google_config)
+        KnowledgeSearch.__init__(self, config=self.config.knowledge_config)
+
+    async def search(self, query: str):
+        """
+        Search the web for relevant content
+        """
+        tasks: List[Coroutine[Any, Any, str]] = []
+
+        if "google" in self.sources:
+            tasks.append(self._compile_google_search(query))
+        if "wikipedia" in self.sources:
+            tasks.append(self._compile_wikipedia(query))
+        if "arxiv" in self.sources:
+            tasks.append(self._compile_arxiv_papers(query))
+
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        return "\n\n".join(item for item in results if isinstance(item, str))