bits-bie 1.2.0__py3-none-any.whl

bie/discovery.py

@@ -0,0 +1,214 @@
+"""
+Free, no-API-key, real-time web search discovery for BIE.
+
+This module answers the question: *"given a query, what URLs on the
+internet are relevant right now?"* — without requiring any paid API,
+subscription, or API key.
+
+It tries multiple lightweight, no-JS public search endpoints in order,
+falling back automatically if one is blocked, rate-limited, or returns
+no results:
+
+  1. DuckDuckGo HTML  (``https://html.duckduckgo.com/html/``)
+  2. DuckDuckGo Lite  (``https://lite.duckduckgo.com/lite/``)
+  3. Bing HTML        (``https://www.bing.com/search``)
+
+This is the **discovery** step. BIE then crawls the discovered URLs with
+Bitscrape and ranks the extracted content with its hybrid BM25+vector
+index — giving a genuine "type a query, get a real-time answer from the
+internet" experience with zero configuration and zero cost.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from urllib.parse import parse_qs, unquote, urlparse
+
+import httpx
+
+logger = logging.getLogger("bie.discovery")
+
+_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+)
+
+# DuckDuckGo wraps result links as //duckduckgo.com/l/?uddg=<encoded-url>&...
+_DDG_REDIRECT_RE = re.compile(r"^(?:https?:)?//duckduckgo\.com/l/\?")
+
+# Tracking/redirector domains we never want to treat as a "real" result
+_BLOCKED_HOST_FRAGMENTS = (
+    "duckduckgo.com",
+    "bing.com/search",
+    "go.microsoft.com",
+    "r.search.yahoo.com",
+)
+
+
+def discover_urls(query: str, max_results: int = 5, timeout: float = 15.0) -> list[str]:
+    """Return up to ``max_results`` candidate URLs for ``query`` from the
+    live web — no API key required.
+
+    Tries DuckDuckGo (HTML, then Lite) and falls back to Bing HTML search
+    if both fail or return nothing.
+
+    Args:
+        query: The natural-language search query.
+        max_results: Maximum number of URLs to return.
+        timeout: HTTP request timeout in seconds (per attempt).
+
+    Returns:
+        A list of absolute, deduplicated URLs in result order. Returns an
+        empty list only if every backend fails — callers should treat this
+        as "try again" rather than "no results exist".
+    """
+    headers = {
+        "User-Agent": _USER_AGENT,
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    attempts = [
+        ("ddg_html", _fetch_ddg_html, query),
+        ("ddg_lite", _fetch_ddg_lite, query),
+        ("bing_html", _fetch_bing_html, query),
+    ]
+
+    for name, fetch, q in attempts:
+        try:
+            with httpx.Client(timeout=timeout, headers=headers, follow_redirects=True) as client:
+                html = fetch(client, q)
+        except httpx.HTTPError as exc:
+            logger.warning("Discovery backend %s failed: %s", name, exc)
+            continue
+
+        if not html:
+            continue
+
+        urls = _parse_result_urls(html, max_results)
+        if urls:
+            logger.debug("Discovery backend %s returned %d url(s)", name, len(urls))
+            return urls
+
+    logger.warning("All discovery backends failed or returned no results for query=%r", query)
+    return []
+
+
+def discover_urls_multi(
+    queries: list[str], max_results_per_query: int = 5, max_total: int = 15, timeout: float = 15.0
+) -> list[str]:
+    """Run :func:`discover_urls` for several query variants and merge the
+    results, ranked by how many variants surfaced each URL.
+
+    This implements simple **query fan-out**: searching multiple phrasings
+    of the same question (e.g. the original query plus a couple of
+    rewordings) surfaces a broader, more relevant set of candidate pages
+    than a single query alone — particularly for ambiguous or
+    multi-faceted questions.
+
+    Args:
+        queries: Query variants to search, in priority order. The first
+            is treated as the primary query.
+        max_results_per_query: How many URLs to fetch per query variant.
+        max_total: Maximum number of URLs to return overall.
+        timeout: Per-request timeout in seconds.
+
+    Returns:
+        Deduplicated URLs, ordered by (number of variants that returned
+        them, then first-seen order). URLs found by multiple query
+        variants are considered more likely relevant.
+    """
+    url_votes: dict[str, int] = {}
+    url_order: dict[str, int] = {}
+    order_counter = 0
+
+    for query in queries:
+        for url in discover_urls(query, max_results=max_results_per_query, timeout=timeout):
+            if url not in url_votes:
+                url_votes[url] = 0
+                url_order[url] = order_counter
+                order_counter += 1
+            url_votes[url] += 1
+
+    ranked = sorted(url_votes.keys(), key=lambda u: (-url_votes[u], url_order[u]))
+    return ranked[:max_total]
+
+
+def _fetch_ddg_html(client: httpx.Client, query: str) -> str:
+    resp = client.post("https://html.duckduckgo.com/html/", data={"q": query})
+    resp.raise_for_status()
+    return resp.text
+
+
+def _fetch_ddg_lite(client: httpx.Client, query: str) -> str:
+    resp = client.post("https://lite.duckduckgo.com/lite/", data={"q": query})
+    resp.raise_for_status()
+    return resp.text
+
+
+def _fetch_bing_html(client: httpx.Client, query: str) -> str:
+    resp = client.get("https://www.bing.com/search", params={"q": query, "form": "QBLH"})
+    resp.raise_for_status()
+    return resp.text
+
+
+def _parse_result_urls(html: str, max_results: int) -> list[str]:
+    """Extract organic result URLs from a search results page.
+
+    Handles:
+      - DuckDuckGo HTML:  ``<a class="result__a" href="...">``
+      - DuckDuckGo Lite:  ``<a class="result-link" href="...">``
+      - Bing:             ``<li class="b_algo">...<a href="...">``
+    """
+    hrefs = re.findall(r'class="result__a"[^>]*href="([^"]+)"', html)
+    if not hrefs:
+        hrefs = re.findall(r'class="result-link"[^>]*href="([^"]+)"', html)
+    if not hrefs:
+        hrefs = _extract_bing_hrefs(html)
+
+    urls: list[str] = []
+    seen: set[str] = set()
+    for href in hrefs:
+        url = _resolve_redirect(href)
+        if not url or not url.startswith("http"):
+            continue
+        if _is_blocked_host(url):
+            continue
+        if url in seen:
+            continue
+        seen.add(url)
+        urls.append(url)
+        if len(urls) >= max_results:
+            break
+
+    return urls
+
+
+def _extract_bing_hrefs(html: str) -> list[str]:
+    """Extract result links from Bing's organic result blocks
+    (``<li class="b_algo">``)."""
+    hrefs: list[str] = []
+    for block in re.findall(r'<li class="b_algo".*?</li>', html, flags=re.S):
+        m = re.search(r'<h2[^>]*>\s*<a[^>]*href="([^"]+)"', block)
+        if m:
+            hrefs.append(m.group(1))
+    return hrefs
+
+
+def _resolve_redirect(href: str) -> str | None:
+    """Unwrap DuckDuckGo's ``//duckduckgo.com/l/?uddg=<url-encoded-target>``
+    redirect links to get the real target URL. Other links pass through
+    unchanged."""
+    href = href.strip().replace("&amp;", "&")
+
+    if _DDG_REDIRECT_RE.match(href):
+        parsed = urlparse(href if href.startswith("http") else f"https:{href}")
+        qs = parse_qs(parsed.query)
+        target = qs.get("uddg", [None])[0]
+        return unquote(target) if target else None
+
+    return href
+
+
+def _is_blocked_host(url: str) -> bool:
+    return any(fragment in url for fragment in _BLOCKED_HOST_FRAGMENTS)

bie/engine.py

@@ -0,0 +1,151 @@
+"""
+The ``BIE`` class — the main entry point of the BitSearch Intelligence
+Engine Python API.
+
+.. code-block:: python
+
+    import bie
+
+    engine = bie.BIE()
+    engine.crawl(["https://example.com/news"])
+    results = engine.search("what happened today")
+    for r in results:
+        print(r)
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+
+from bie.chunker import chunk_document
+from bie.config import BIESettings
+from bie.crawler import Crawler
+from bie.index import HybridIndex
+from bie.models import Document, SearchResponse, SearchResult
+
+logger = logging.getLogger("bie")
+
+
+class BIE:
+    """The BitSearch Intelligence Engine — crawl, index, and search the web.
+
+    Args:
+        settings: Optional :class:`bie.config.BIESettings`. If omitted,
+            settings are loaded from environment variables / ``.env``.
+
+    Example::
+
+        engine = bie.BIE()
+        engine.crawl(["https://www.bbc.com/news"])
+        for hit in engine.search("global markets"):
+            print(hit.title, hit.url, hit.score)
+    """
+
+    def __init__(self, settings: BIESettings | None = None) -> None:
+        self.settings = settings or BIESettings()
+        self.index = HybridIndex(self.settings)
+        self.crawler = Crawler(self.settings)
+
+    # ------------------------------------------------------------------
+    # Ingestion
+    # ------------------------------------------------------------------
+
+    def crawl(
+        self,
+        urls: list[str],
+        allowed_domains: list[str] | None = None,
+        instruction: str = "",
+    ) -> int:
+        """Crawl ``urls`` (and linked pages, bounded by settings) and add
+        the extracted documents to the index.
+
+        Args:
+            urls: Seed URLs to crawl.
+            allowed_domains: Restrict link-following to these domains
+                (defaults to the domains of ``urls``).
+            instruction: Optional short natural-language description of
+                what to look for (e.g. "pricing and plans pages"). When
+                set, outgoing links are ranked by keyword overlap with the
+                instruction and only the most relevant are followed. This
+                is a keyword-relevance heuristic, not full semantic
+                understanding — see :mod:`bie.spiders.generic` for
+                details.
+
+        Returns the number of documents added.
+        """
+        documents = self.crawler.crawl(
+            urls, allowed_domains=allowed_domains, instruction=instruction
+        )
+        for doc in documents:
+            self.add_document(doc)
+        return len(documents)
+
+    def add_document(self, doc: Document) -> None:
+        """Add a single pre-fetched :class:`~bie.models.Document` to the index."""
+        chunks = chunk_document(
+            doc, chunk_size=self.settings.chunk_size, overlap=self.settings.chunk_overlap
+        )
+        if not chunks:
+            return
+        self.index.add_document(doc, chunks)
+
+    def add_text(
+        self,
+        url: str,
+        text: str,
+        title: str = "",
+        trust_score: float = 0.5,
+        **metadata,
+    ) -> None:
+        """Add raw text directly (no crawling) — useful for indexing local
+        documents, PDFs you've already extracted, API responses, etc."""
+        doc = Document(
+            url=url,
+            title=title or url,
+            text=text,
+            trust_score=trust_score,
+            metadata=metadata,
+        )
+        self.add_document(doc)
+
+    # ------------------------------------------------------------------
+    # Retrieval
+    # ------------------------------------------------------------------
+
+    def search(self, query: str, top_k: int = 10) -> list[SearchResult]:
+        """Run a hybrid (BM25 + vector) search over the indexed documents."""
+        return self.index.search(query, top_k=top_k)
+
+    def search_full(self, query: str, top_k: int = 10) -> SearchResponse:
+        """Like :meth:`search`, but returns a full :class:`SearchResponse`
+        with timing and index-size metadata (matches the ``/search`` API)."""
+        start = time.perf_counter()
+        results = self.search(query, top_k=top_k)
+        took_ms = (time.perf_counter() - start) * 1000
+        return SearchResponse(
+            query=query,
+            results=results,
+            took_ms=round(took_ms, 2),
+            total_indexed_documents=len(self.index),
+        )
+
+    # ------------------------------------------------------------------
+    # Convenience
+    # ------------------------------------------------------------------
+
+    def search_web(self, query: str, urls: list[str], top_k: int = 10) -> list[SearchResult]:
+        """One-shot: crawl ``urls``, then immediately search the freshly
+        indexed content. Equivalent to ``engine.crawl(urls)`` followed by
+        ``engine.search(query)``."""
+        self.crawl(urls)
+        return self.search(query, top_k=top_k)
+
+    def __len__(self) -> int:
+        return len(self.index)
+
+    def __repr__(self) -> str:  # pragma: no cover
+        return (
+            f"<BIE documents={len(self.index)} "
+            f"vector_search={'on' if self.index.vector_enabled else 'off'}>"
+        )

bie/extract.py

@@ -0,0 +1,218 @@
+"""
+``bie.extract()`` — retrieve clean, readable Markdown from a single URL.
+
+This is BIE's "give me this page as clean text" primitive: fetch a URL,
+strip navigation/ads/scripts/styling noise, and convert the main content
+to Markdown — the format LLMs work with best.
+
+For static HTML pages, this uses a direct HTTP fetch (fast, no browser).
+For JavaScript-rendered pages (SPAs, sites that return a near-empty
+``<body>`` until JS runs), pass ``render_js=True`` to fall back to a
+headless Playwright browser — requires the optional ``bie[render]``
+extra (``pip install "bits-bie[render]"`` plus ``playwright install
+chromium`` once).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+import httpx
+from markdownify import markdownify
+
+from bie.security import scan_for_prompt_injection
+
+if TYPE_CHECKING:
+    from bie.security import SecurityReport
+
+logger = logging.getLogger("bie.extract")
+
+_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0 Safari/537.36 BIE/0.5"
+)
+
+# Tags whose content is never useful for an LLM and should be stripped
+# before markdown conversion.
+_STRIP_TAGS = (
+    "script",
+    "style",
+    "noscript",
+    "nav",
+    "footer",
+    "header",
+    "form",
+    "iframe",
+    "svg",
+    "button",
+    "aside",
+)
+
+# A page is considered "JS-required" if, after stripping noise tags, the
+# remaining visible text is suspiciously short.
+_JS_REQUIRED_TEXT_THRESHOLD = 80
+
+
+class ExtractError(RuntimeError):
+    """Raised when a page can't be fetched or extracted."""
+
+
+@dataclass
+class ExtractResult:
+    """Result of :func:`bie.extract.extract`."""
+
+    url: str
+    title: str
+    markdown: str
+    text: str
+    word_count: int
+    rendered_with_js: bool = False
+    security: "SecurityReport | None" = field(default=None)
+
+    def __str__(self) -> str:  # pragma: no cover - convenience only
+        flag = " [JS-rendered]" if self.rendered_with_js else ""
+        return f"<ExtractResult {self.url!r} title={self.title!r} words={self.word_count}{flag}>"
+
+
+def extract(
+    url: str,
+    render_js: bool = False,
+    timeout: float = 20.0,
+    scan_security: bool = True,
+) -> ExtractResult:
+    """Fetch ``url`` and return its content as clean Markdown.
+
+    Args:
+        url: The page to fetch.
+        render_js: If True, render the page with a headless browser
+            (requires ``pip install "bits-bie[render]"`` and
+            ``playwright install chromium``). If False (default), BIE
+            still auto-detects JS-only pages and raises a helpful
+            :class:`ExtractError` suggesting ``render_js=True`` rather
+            than silently returning near-empty content.
+        timeout: Request timeout in seconds.
+        scan_security: If True (default), scan the extracted text for
+            prompt-injection patterns and attach a
+            :class:`bie.security.SecurityReport` to ``result.security``.
+            This does **not** remove or alter content, only flags it.
+
+    Returns:
+        An :class:`ExtractResult` with ``markdown``, plain ``text``,
+        ``title``, and ``word_count``.
+
+    Raises:
+        ExtractError: if the page can't be fetched, or appears to require
+            JavaScript and ``render_js=False``.
+    """
+    if render_js:
+        html = _fetch_with_playwright(url, timeout)
+        rendered_with_js = True
+    else:
+        html = _fetch_static(url, timeout)
+        rendered_with_js = False
+
+        if _looks_js_only(html):
+            raise ExtractError(
+                f"{url} appears to require JavaScript to render its content "
+                f"(static fetch returned very little text). Retry with "
+                f"extract(url, render_js=True) — requires "
+                f'\'pip install "bits-bie[render]"\' and '
+                f"'playwright install chromium' once."
+            )
+
+    title, markdown, text = _to_markdown(html)
+
+    result = ExtractResult(
+        url=url,
+        title=title,
+        markdown=markdown,
+        text=text,
+        word_count=len(text.split()),
+        rendered_with_js=rendered_with_js,
+    )
+
+    if scan_security:
+        result.security = scan_for_prompt_injection(text)
+
+    return result
+
+
+def _fetch_static(url: str, timeout: float) -> str:
+    headers = {"User-Agent": _USER_AGENT}
+    try:
+        with httpx.Client(timeout=timeout, headers=headers, follow_redirects=True) as client:
+            resp = client.get(url)
+            resp.raise_for_status()
+            return resp.text
+    except httpx.HTTPError as exc:
+        raise ExtractError(f"Failed to fetch {url}: {exc}") from exc
+
+
+def _fetch_with_playwright(url: str, timeout: float) -> str:
+    try:
+        from playwright.async_api import async_playwright
+    except ImportError as exc:
+        raise ExtractError(
+            "render_js=True requires the 'playwright' package. Install with: "
+            'pip install "bits-bie[render]" && playwright install chromium'
+        ) from exc
+
+    async def _run() -> str:
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch()
+            try:
+                page = await browser.new_page(user_agent=_USER_AGENT)
+                await page.goto(url, timeout=timeout * 1000, wait_until="networkidle")
+                return await page.content()
+            finally:
+                await browser.close()
+
+    try:
+        return asyncio.run(_run())
+    except Exception as exc:
+        raise ExtractError(f"Failed to render {url} with Playwright: {exc}") from exc
+
+
+def _looks_js_only(html: str) -> bool:
+    """Heuristic: after stripping script/style/nav/etc, is there
+    suspiciously little visible text? Typical of SPA shells that render
+    everything client-side (e.g. ``<div id="root"></div>``)."""
+    _, _, text = _to_markdown(html)
+    return len(text.strip()) < _JS_REQUIRED_TEXT_THRESHOLD
+
+
+def _to_markdown(html: str) -> tuple[str, str, str]:
+    """Strip noise tags and convert to (title, markdown, plain_text)."""
+    from selectolax.parser import HTMLParser
+
+    tree = HTMLParser(html)
+
+    title_node = tree.css_first("title")
+    title = _clean_text(title_node.text()) if title_node else ""
+
+    for tag in _STRIP_TAGS:
+        for node in tree.css(tag):
+            node.decompose()
+
+    body = tree.css_first("body") or tree
+    body_html = body.html or ""
+
+    markdown = markdownify(body_html, heading_style="ATX", strip=["a"]).strip()
+    markdown = _collapse_blank_lines(markdown)
+
+    text_node = tree.css_first("body") or tree
+    text = _clean_text(text_node.text(separator=" ", deep=True))
+
+    return title, markdown, text
+
+
+def _collapse_blank_lines(markdown: str) -> str:
+    return re.sub(r"\n{3,}", "\n\n", markdown)
+
+
+def _clean_text(text: str) -> str:
+    return re.sub(r"\s+", " ", text or "").strip()