better-websearch-cli 0.0.1__py3-none-any.whl

better_websearch_cli-0.0.1.dist-info/METADATA

@@ -0,0 +1,148 @@
+Metadata-Version: 2.4
+Name: better-websearch-cli
+Version: 0.0.1
+Summary: CLI tool to fetch URLs and search the web
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer[all]>=0.12
+Requires-Dist: httpx>=0.25
+Requires-Dist: selectolax>=0.3
+Requires-Dist: markdownify>=0.12
+Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Provides-Extra: spa
+Requires-Dist: playwright>=1.40; extra == "spa"
+Dynamic: license-file
+
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="public/banner.jpg">
+    <source media="(prefers-color-scheme: light)" srcset="public/banner.jpg">
+    <img src="public/banner.jpg" alt="Websearch CLI Logo" width="100%">
+  </picture>
+</p>
+
+<h1 align="center">Websearch CLI</h1>
+
+<p align="center">
+  Fetch URLs and search the web from your terminal. Fast, simple, and extensible.
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/websearch/">
+    <img src="https://img.shields.io/pypi/v/websearch" alt="PyPI Version">
+  </a>
+  <a href="https://github.com/AliiiBenn/websearch">
+    <img src="https://img.shields.io/github/license/AliiiBenn/websearch" alt="License">
+  </a>
+  <a href="https://github.com/AliiiBenn/websearch/actions">
+    <img src="https://img.shields.io/github/actions/workflow/status/AliiiBenn/websearch/test" alt="Tests">
+  </a>
+  <a href="https://www.python.org/">
+    <img src="https://img.shields.io/badge/python-3.14+-blue" alt="Python">
+  </a>
+</p>
+
+Fetch web pages and search the internet - all from your command line.
+
+## Why Websearch CLI?
+
+- **Fast** - Async HTTP powered by httpx
+- **Smart caching** - Local cache with TTL and automatic eviction
+- **Clean output** - HTML to Markdown conversion with XSS protection
+- **Type-safe** - 100% type-annotated Python
+
+## Features
+
+- Fetch URLs and convert to Markdown
+- Web search via Brave Search API
+- Local caching with TTL and size limits
+- XSS protection and HTML sanitization
+- SPA (Single Page App) detection
+- Retry with exponential backoff
+
+## Quick Start
+
+```bash
+# Install
+uv add websearch
+
+# Fetch a URL as Markdown
+websearch fetch https://example.com
+
+# Search the web
+websearch search "python async tutorial"
+
+# Fetch with options
+websearch fetch https://example.com --no-cache --verbose
+websearch search "python" -n 20 -t news
+```
+
+## Command Overview
+
+| Command | Description |
+|---------|-------------|
+| `websearch fetch <url>` | Fetch URL and convert to Markdown |
+| `websearch search <query>` | Search the web |
+| `websearch ping` | Check if CLI is working |
+
+## Fetch Options
+
+```
+--refresh, -r       Skip cache and force fresh fetch
+--no-cache          Disable caching
+--no-verify         Skip SSL certificate verification
+--output, -o PATH    Output file path
+--verbose, -v        Show verbose output
+```
+
+## Search Options
+
+```
+--count, -n <n>      Number of results (1-50, default: 10)
+--type, -t <type>   Result type: web, news, images, videos
+--output, -o PATH    Output file path
+--json               Output raw JSON response
+--no-cache           Disable caching
+```
+
+## Configuration
+
+Set your Brave API key:
+
+```bash
+export BRAVE_API_KEY=your_api_key_here
+```
+
+Get your API key at https://brave.com/search/api/
+
+## Development
+
+```bash
+# Clone and install
+git clone https://github.com/AliiiBenn/websearch.git
+cd websearch
+uv sync --dev
+
+# Run tests
+uv run pytest
+
+# Lint and type-check
+uv run ruff check websearch/
+uv run mypy websearch/
+
+# Try it out
+uv run websearch fetch https://example.com
+```
+
+## Contributing
+
+Contributions are welcome! Feel free to open issues or submit PRs.
+
+## License
+
+MIT - See [LICENSE](LICENSE) for details.

better_websearch_cli-0.0.1.dist-info/RECORD

@@ -0,0 +1,30 @@
+better_websearch_cli-0.0.1.dist-info/licenses/LICENSE,sha256=S69ieINMLlsFsxDUi1x2dDdYTJ5pUgg7bFV3UFDra3M,1070
+websearch/__main__.py,sha256=jJZn0xqgiVJv8ZRITXEltYjcVpHCWwWJgoGmC1Db6lc,40
+websearch/main.py,sha256=Y6oqX_rSsjXGvx0dxjI4SyXCdnzT9wWpyINRl2XM2c0,5652
+websearch/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+websearch/core/cache/__init__.py,sha256=L6JdU6OL9P1jWBQV89ZmlH9TcupXr5T-_Dqa3ppJHsk,102
+websearch/core/cache/cache.py,sha256=yZxzZpLKy8to9z9ZOp8QZGBQJnmLoD98tK7y2zg7DNo,6985
+websearch/core/cache/key.py,sha256=uOWX4hNdYoryS0WePo1FfhJPfXrn_3SJg7NFKUuX4Fo,2353
+websearch/core/cache/storage.py,sha256=PU1DPTXF_iBVlc3REtmr0kEzhTrmhq6HEsHEZJFXOLI,5873
+websearch/core/cache/ttl.py,sha256=ACZEPsgDlwaVnsXxBTrbPvxiM_icrBDfmMAOqpkdkyo,1614
+websearch/core/converter/__init__.py,sha256=HRyFronkvSTIfv3Ey9Ia8VY33JKWxGzIsIhtO9c2M2E,137
+websearch/core/converter/converter.py,sha256=eBMOYU2C-B8nQpSZYPYdiHG5YY8kDA9aCMFQnPMDjyM,3110
+websearch/core/converter/encoding.py,sha256=_vbQ6iYNJEmFmBLmxwqNPBW_DnvFiCiGudbmx1_gc1o,451
+websearch/core/converter/security.py,sha256=FjYkEtedEKAcuhS6xbPibVA7z02sQeMBGEFxtKTweGE,763
+websearch/core/fetcher/__init__.py,sha256=SWrxKOnqCz3eP8JvOeF6XibFFe8pfhNYRtcl3sRkGzs,927
+websearch/core/fetcher/backoff.py,sha256=jdERTOLFVXGHrVDXx1nxWlubg_zL_Nj_EvzyfCKB2Hg,582
+websearch/core/fetcher/detection.py,sha256=7p4SUFZI71AMG7qKRFMT2yyDiV8fygzDKeMiaIGneBk,1058
+websearch/core/fetcher/errors.py,sha256=o5Cj-cB1Je6GjPjAb14fApQwHqnN6elzYWNfYjeAUKY,1605
+websearch/core/fetcher/fetcher.py,sha256=Lor6j9iUHUxd15JF40Yqg49ad61_doawNIm5xyajkM8,7643
+websearch/core/search/__init__.py,sha256=wflO_nwTh_P7kx7GTnNHmTfXrDWJZfJd2PWgq_ekCxo,503
+websearch/core/search/client.py,sha256=B3zlaJvEF3ykAOcgFM-bxCUNACSQrZFucCJI16rttSI,4685
+websearch/core/search/search.py,sha256=CReu47M5rHMuLolru2HLJ02seTht8Uu66by_GMm0Dqw,5677
+websearch/core/search/types.py,sha256=DX4aXOSFqunwtacFsE-NFyfq1y3QgwGx5RkxOeZgHXc,1102
+websearch/core/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+websearch/core/types/maybe.py,sha256=eKJEg3S9W5CKn9fwjf6VHkqwI_kiGfKSdnxnJ6ZNDz4,5090
+websearch/core/types/result.py,sha256=DPJgrjh05EuTn5ZgEx6qlG6Ozr9qXoXxkKxqj4uzMuw,5754
+better_websearch_cli-0.0.1.dist-info/METADATA,sha256=g1dnZNT0-ugJ0q6_8L2Qy11DsWKcWHQ8tml3CtyROnE,3758
+better_websearch_cli-0.0.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+better_websearch_cli-0.0.1.dist-info/entry_points.txt,sha256=gDCbSYPCmbUYRzHhI7dpvO18VWq2Vnbq54VdNDOXczI,50
+better_websearch_cli-0.0.1.dist-info/top_level.txt,sha256=D1fdg-HoBORVAOYW9PRiEIq8wqg7uUQMyhmDJv6X3S4,10
+better_websearch_cli-0.0.1.dist-info/RECORD,,

better_websearch_cli-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

better_websearch_cli-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+websearch = websearch.main:main

better_websearch_cli-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Websearch CLI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

better_websearch_cli-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+websearch

websearch/__main__.py

@@ -0,0 +1,3 @@
+from websearch.main import main
+
+main()

websearch/core/cache/__init__.py

@@ -0,0 +1,5 @@
+"""URL and search result cache."""
+
+from websearch.core.cache.cache import Cache
+
+__all__ = ["Cache"]

websearch/core/cache/cache.py

@@ -0,0 +1,254 @@
+"""Main cache interface."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from websearch.core.cache.storage import CacheStorage
+from websearch.core.cache.ttl import (
+    DEFAULT_SEARCH_TTL,
+    DEFAULT_URL_TTL,
+    get_search_ttl,
+    get_url_ttl,
+    is_expired,
+)
+from websearch.core.types.maybe import Just, Maybe, Nothing
+
+
+class Cache:
+    """URL and search result cache."""
+
+    def __init__(
+        self,
+        cache_dir: Path | None = None,
+        enabled: bool = True,
+        max_size: int = 500 * 1024 * 1024,  # 500MB
+    ):
+        """Initialize cache.
+
+        Args:
+            cache_dir: Optional custom cache directory
+            enabled: Whether caching is enabled
+            max_size: Maximum cache size in bytes
+        """
+        self.storage = CacheStorage(cache_dir)
+        self.enabled = enabled
+        self.max_size = max_size
+
+    def get_url(self, url: str) -> Maybe[tuple[bytes, dict[str, Any]]]:
+        """Get cached URL content and metadata.
+
+        Args:
+            url: URL to retrieve
+
+        Returns:
+            Just((content, metadata)) on cache hit,
+            Nothing on cache miss or expired
+        """
+        if not self.enabled:
+            return Nothing()
+
+        content, metadata = self.storage.get_url(url)
+
+        if content is None or metadata is None:
+            return Nothing()
+
+        # Check expiration
+        cached_at = metadata.get("cached_at")
+        ttl = metadata.get("ttl", DEFAULT_URL_TTL)
+
+        if cached_at is None:
+            return Nothing()
+
+        from datetime import datetime
+
+        if isinstance(cached_at, str):
+            cached_at = datetime.fromisoformat(cached_at.replace("Z", "+00:00"))
+
+        if is_expired(cached_at, ttl):
+            return Nothing()
+
+        return Just((content, metadata))
+
+    def set_url(
+        self,
+        url: str,
+        content: bytes,
+        metadata: dict[str, Any] | None = None,
+        ttl: float | None = None,
+    ) -> None:
+        """Cache URL content.
+
+        Args:
+            url: URL being cached
+            content: HTML content
+            metadata: Optional additional metadata
+            ttl: Optional TTL override
+        """
+        if not self.enabled:
+            return
+
+        # Check size and evict if needed
+        self._evict_if_needed()
+
+        meta = metadata.copy() if metadata else {}
+        meta["url"] = url
+        meta["ttl"] = ttl or get_url_ttl()
+
+        self.storage.set_url(url, content, meta)
+
+    def get_search(
+        self, query: str, count: int, result_type: str = "web"
+    ) -> Maybe[dict[str, Any]]:
+        """Get cached search results.
+
+        Args:
+            query: Search query
+            count: Number of results
+            result_type: Type of results
+
+        Returns:
+            Just(results) on cache hit, Nothing otherwise
+        """
+        if not self.enabled:
+            return Nothing()
+
+        data = self.storage.get_search(query, count, result_type)
+
+        if data is None:
+            return Nothing()
+
+        metadata = data.get("metadata", {})
+        cached_at = metadata.get("cached_at")
+        ttl = metadata.get("ttl", DEFAULT_SEARCH_TTL)
+
+        if cached_at is None:
+            return Nothing()
+
+        from datetime import datetime
+
+        if isinstance(cached_at, str):
+            cached_at = datetime.fromisoformat(cached_at.replace("Z", "+00:00"))
+
+        if is_expired(cached_at, ttl):
+            return Nothing()
+
+        return Just(data.get("results", {}))
+
+    def set_search(
+        self,
+        query: str,
+        count: int,
+        result_type: str,
+        results: dict[str, Any],
+    ) -> None:
+        """Cache search results.
+
+        Args:
+            query: Search query
+            count: Number of results
+            result_type: Type of results
+            results: Results data
+        """
+        if not self.enabled:
+            return
+
+        ttl = get_search_ttl()
+        self.storage.set_search(query, count, result_type, results, ttl)
+
+    def is_fresh(self, url: str) -> bool:
+        """Check if URL is cached and fresh.
+
+        Args:
+            url: URL to check
+
+        Returns:
+            True if cached and not expired
+        """
+        return self.get_url(url).is_just()
+
+    def invalidate(self, url: str) -> bool:
+        """Invalidate cached URL.
+
+        Args:
+            url: URL to invalidate
+
+        Returns:
+            True if something was deleted
+        """
+        return self.storage.delete(url)
+
+    def clear(self) -> None:
+        """Clear all cache."""
+        self.storage.clear()
+
+    def stats(self) -> dict[str, Any]:
+        """Get cache statistics.
+
+        Returns:
+            Dict with size, url_count, search_count
+        """
+        size = self.storage.get_size()
+
+        url_count = 0
+        search_count = 0
+
+        storage = self.storage
+        if storage.url_dir.exists():
+            url_count = len(list(storage.url_dir.rglob("*.html")))
+
+        if storage.search_dir.exists():
+            search_count = len(list(storage.search_dir.rglob("*.json")))
+
+        return {
+            "size_bytes": size,
+            "size_mb": round(size / (1024 * 1024), 2),
+            "url_count": url_count,
+            "search_count": search_count,
+            "enabled": self.enabled,
+            "max_size_mb": self.max_size / (1024 * 1024),
+        }
+
+    def _evict_if_needed(self) -> None:
+        """Evict LRU entries if cache exceeds max size."""
+        size = self.storage.get_size()
+
+        if size < self.max_size:
+            return
+
+        # Collect all cached files with their last access time
+        files_with_time = []
+        if self.storage.url_dir.exists():
+            for path in self.storage.url_dir.rglob("*"):
+                if path.is_file():
+                    files_with_time.append((path.stat().st_atime, path))
+
+        if self.storage.search_dir.exists():
+            for path in self.storage.search_dir.rglob("*.json"):
+                if path.is_file():
+                    files_with_time.append((path.stat().st_atime, path))
+
+        # Sort by access time (oldest first)
+        files_with_time.sort(key=lambda x: x[0])
+
+        # Delete oldest files until under limit
+        current_size = size
+        for _, path in files_with_time:
+            if current_size < self.max_size * 0.9:  # Stop at 90% to avoid frequent eviction
+                break
+            try:
+                file_size = path.stat().st_size
+                path.unlink()
+                current_size -= file_size
+
+                # Try to delete parent dirs if empty
+                parent = path.parent
+                while parent != self.storage.cache_dir and parent.exists():
+                    try:
+                        parent.rmdir()
+                    except OSError:
+                        break
+                    parent = parent.parent
+            except OSError:
+                continue

websearch/core/cache/key.py

@@ -0,0 +1,99 @@
+"""Cache key generation and URL normalization."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from urllib.parse import unquote, urlparse
+
+
+def normalize_url(url: str) -> str:
+    """Normalize URL for consistent caching.
+
+    Args:
+        url: Raw URL
+
+    Returns:
+        Normalized URL with lowercase scheme/domain, decoded path, sorted query
+    """
+    parsed = urlparse(url)
+
+    # Lowercase scheme and netloc
+    scheme = parsed.scheme.lower()
+    netloc = parsed.netloc.lower()
+
+    # Remove default ports
+    if (scheme == "http" and netloc.endswith(":80")) or (
+        scheme == "https" and netloc.endswith(":443")
+    ):
+        netloc = netloc.rsplit(":", 1)[0]
+
+    # Decode path and lowercase it for case-insensitive matching
+    path = unquote(parsed.path).lower()
+    if not path:
+        path = "/"
+
+    # Normalize path (remove trailing slash except for root)
+    if path != "/" and path.endswith("/"):
+        path = path.rstrip("/")
+
+    # Build query with sorted params
+    query = parsed.query
+
+    # Rebuild URL
+    normalized = f"{scheme}://{netloc}{path}"
+    if query:
+        normalized += f"?{query}"
+
+    return normalized
+
+
+def get_cache_key(url: str) -> Path:
+    """Get filesystem path for URL cache.
+
+    Args:
+        url: Normalized URL
+
+    Returns:
+        Path relative to cache directory
+    """
+    normalized = normalize_url(url)
+    parsed = urlparse(normalized)
+
+    domain = parsed.netloc
+    path = parsed.path.lstrip("/")
+
+    if not path:
+        path = "index.html"
+    elif not path.endswith(".html"):
+        path = path + "/index.html"
+
+    return Path(domain) / path
+
+
+def get_url_hash(url: str) -> str:
+    """Get SHA256 hash of URL for search cache keys.
+
+    Args:
+        url: URL to hash
+
+    Returns:
+        Short hex hash (8 characters)
+    """
+    normalized = normalize_url(url)
+    return hashlib.sha256(normalized.encode()).hexdigest()[:8]
+
+
+def get_search_key(query: str, count: int, result_type: str = "web") -> str:
+    """Get cache filename for search results.
+
+    Args:
+        query: Search query
+        count: Number of results
+        result_type: Type of results (web, news, etc.)
+
+    Returns:
+        Cache filename like "dc9a8f5_10_web.json"
+    """
+    query_hash = get_url_hash(query)
+    return f"{query_hash}_{count}_{result_type}.json"