bits-bie 1.2.0__py3-none-any.whl

bie/__init__.py

@@ -0,0 +1,85 @@
+"""
+BIE — BitSearch Intelligence Engine
+=====================================
+
+A real-time web search engine and crawling toolkit for AI applications —
+built on top of **Bitscrape** (https://pypi.org/project/bitscrape/). No
+API keys, no subscriptions, no third-party search services.
+
+Core primitives
+----------------
+
+- :func:`websearch` — search the live internet for a query (no URLs needed)
+- :func:`search` — crawl + rank specific URLs against a query
+- :func:`extract` — get clean Markdown from a single URL
+- :func:`map_site` — discover a site's sitemap before crawling
+- :func:`crawl_site` — crawl a site guided by a natural-language instruction
+- :class:`BIE` — build a persistent, queryable index
+
+Quick start
+-----------
+
+.. code-block:: python
+
+    import bie
+
+    # Search the live internet — no URLs, no API key, no subscription
+    results = bie.websearch("who won the latest F1 race")
+    for r in results:
+        print(r.title, r.url)
+        print(r.snippet)
+
+    # Get clean markdown from a specific page
+    page = bie.extract("https://example.com/article")
+    print(page.markdown)
+
+    # Discover a site's structure before crawling
+    sitemap = bie.map_site("https://example.com")
+    print(sitemap.urls[:10])
+
+    # Crawl a site guided by an instruction
+    engine, results = bie.crawl_site(
+        ["https://docs.example.com"],
+        instruction="authentication and rate limits",
+    )
+
+Run as a server::
+
+    bie serve --port 8000
+
+Run as an MCP tool (for Claude Desktop, Claude Code, etc.)::
+
+    bie mcp
+"""
+
+from __future__ import annotations
+
+from bie.config import BIESettings
+from bie.engine import BIE
+from bie.extract import ExtractError, ExtractResult, extract
+from bie.models import Document, SearchResult
+from bie.quicksearch import search, websearch
+from bie.security import SecurityFinding, SecurityReport, scan_for_prompt_injection
+from bie.sitecrawl import crawl_site
+from bie.sitemap import SiteMap, map_site
+
+__version__ = "0.5.0"
+
+__all__ = [
+    "BIE",
+    "BIESettings",
+    "Document",
+    "SearchResult",
+    "search",
+    "websearch",
+    "extract",
+    "ExtractResult",
+    "ExtractError",
+    "map_site",
+    "SiteMap",
+    "crawl_site",
+    "scan_for_prompt_injection",
+    "SecurityReport",
+    "SecurityFinding",
+    "__version__",
+]

bie/chunker.py

@@ -0,0 +1,83 @@
+"""
+Lightweight text chunker — splits cleaned document text into
+paragraph/section-sized chunks for indexing (PRD Module 8: Context Builder).
+
+No heavy NLP deps; sentence/paragraph aware, with overlap.
+"""
+
+from __future__ import annotations
+
+import re
+
+from bie.models import Chunk, Document
+
+_PARA_SPLIT = re.compile(r"\n\s*\n+")
+_SENT_SPLIT = re.compile(r"(?<=[.!?])\s+")
+
+
+def chunk_document(doc: Document, chunk_size: int = 800, overlap: int = 100) -> list[Chunk]:
+    """Split a document's text into overlapping chunks.
+
+    Strategy:
+      1. Split on paragraph boundaries.
+      2. Greedily pack paragraphs into chunks up to ``chunk_size`` chars.
+      3. If a single paragraph exceeds ``chunk_size``, split it by sentence.
+      4. Apply a small character-overlap between consecutive chunks so
+         retrieval doesn't lose context at boundaries.
+    """
+    text = (doc.text or "").strip()
+    if not text:
+        return []
+
+    paragraphs = [p.strip() for p in _PARA_SPLIT.split(text) if p.strip()]
+    if not paragraphs:
+        paragraphs = [text]
+
+    units: list[str] = []
+    for para in paragraphs:
+        if len(para) <= chunk_size:
+            units.append(para)
+        else:
+            sentences = _SENT_SPLIT.split(para)
+            buf = ""
+            for sent in sentences:
+                if len(buf) + len(sent) + 1 <= chunk_size:
+                    buf = f"{buf} {sent}".strip()
+                else:
+                    if buf:
+                        units.append(buf)
+                    buf = sent
+            if buf:
+                units.append(buf)
+
+    chunks: list[Chunk] = []
+    buf = ""
+    offset = 0
+    for unit in units:
+        candidate = f"{buf}\n\n{unit}".strip() if buf else unit
+        if len(candidate) <= chunk_size:
+            buf = candidate
+            continue
+
+        if buf:
+            chunks.append(_make_chunk(doc, buf, offset))
+            offset += max(len(buf) - overlap, 0)
+            tail = buf[-overlap:] if overlap else ""
+            buf = f"{tail}\n\n{unit}".strip() if tail else unit
+        else:
+            buf = unit
+
+    if buf:
+        chunks.append(_make_chunk(doc, buf, offset))
+
+    return chunks
+
+
+def _make_chunk(doc: Document, text: str, start_offset: int) -> Chunk:
+    return Chunk(
+        doc_id=doc.doc_id,
+        text=text,
+        start_offset=start_offset,
+        end_offset=start_offset + len(text),
+        metadata={"site": doc.site, "title": doc.title},
+    )

bie/cli.py

@@ -0,0 +1,275 @@
+"""
+BIE command-line interface.
+
+Examples::
+
+    bie search "AI regulation 2026" --url https://example.com/news
+    bie search-live "who won the latest F1 race"
+    bie extract https://example.com/article
+    bie map https://example.com
+    bie crawl https://example.com --max-pages 20 --instruction "pricing pages"
+    bie serve --port 8000
+    bie mcp
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+
+import click
+
+from bie import __version__
+from bie.config import BIESettings
+from bie.engine import BIE
+
+
+@click.group()
+@click.version_option(__version__, prog_name="bie")
+def cli() -> None:
+    """BIE — BitSearch Intelligence Engine. Real-time web search & extraction for AI apps."""
+
+
+@cli.command()
+@click.argument("query")
+@click.option("--url", "urls", multiple=True, required=True, help="Seed URL(s) to crawl & search")
+@click.option("--top-k", default=10, show_default=True, help="Number of results to return")
+@click.option("--max-pages", default=10, show_default=True, help="Max pages to crawl per seed URL")
+@click.option("--no-embeddings", is_flag=True, help="Disable semantic/vector search (BM25 only)")
+@click.option("--json", "as_json", is_flag=True, help="Output raw JSON")
+def search(query: str, urls: tuple[str, ...], top_k: int, max_pages: int, no_embeddings: bool, as_json: bool) -> None:
+    """Crawl URL(s) and search the freshly indexed content for QUERY."""
+    settings = BIESettings(max_pages=max_pages, use_embeddings=not no_embeddings)
+    engine = BIE(settings)
+    click.echo(f"Crawling {len(urls)} source(s)...", err=True)
+    n = engine.crawl(list(urls))
+    click.echo(f"Indexed {n} document(s). Searching...", err=True)
+
+    response = engine.search_full(query, top_k=top_k)
+
+    if as_json:
+        click.echo(response.model_dump_json(indent=2))
+        return
+
+    if not response.results:
+        click.echo("No results found.")
+        return
+
+    for i, r in enumerate(response.results, 1):
+        click.echo(f"\n{i}. {r.title}")
+        click.echo(f"   {r.url}")
+        click.echo(f"   score={r.score:.4f}  trust={r.trust_score:.2f}")
+        click.echo(f"   {r.snippet}")
+    click.echo(f"\n({response.took_ms} ms, {response.total_indexed_documents} docs indexed)")
+
+
+@cli.command(name="search-live")
+@click.argument("query")
+@click.option("--top-k", default=10, show_default=True, help="Number of results to return")
+@click.option("--discovery-results", default=8, show_default=True, help="Candidate URLs to discover")
+@click.option("--no-deep", is_flag=True, help="Skip crawling; return raw discovery order without snippets")
+@click.option("--no-embeddings", is_flag=True, help="Disable semantic/vector re-ranking (BM25 only)")
+@click.option("--json", "as_json", is_flag=True, help="Output raw JSON")
+def search_live(
+    query: str,
+    top_k: int,
+    discovery_results: int,
+    no_deep: bool,
+    no_embeddings: bool,
+    as_json: bool,
+) -> None:
+    """Search the live internet for QUERY — no seed URLs, no API key, no subscription.
+
+    Discovers relevant URLs via free public search endpoints (DuckDuckGo,
+    with a Bing fallback), crawls them with Bitscrape, and ranks the
+    extracted content against QUERY using BIE's hybrid BM25+vector index.
+    """
+    import bie
+
+    results = bie.websearch(
+        query,
+        top_k=top_k,
+        discovery_results=discovery_results,
+        deep=not no_deep,
+        use_embeddings=not no_embeddings,
+    )
+
+    if as_json:
+        click.echo(json.dumps([r.model_dump() for r in results], indent=2))
+        return
+
+    if not results:
+        click.echo(
+            "No results found. The free search backends may be temporarily "
+            "rate-limiting — try again in a moment."
+        )
+        return
+
+    for i, r in enumerate(results, 1):
+        click.echo(f"\n{i}. {r.title}")
+        click.echo(f"   {r.url}")
+        click.echo(f"   score={r.score:.4f}")
+        if r.snippet:
+            click.echo(f"   {r.snippet}")
+
+
+@cli.command()
+@click.argument("urls", nargs=-1, required=True)
+@click.option("--max-pages", default=40, show_default=True)
+@click.option("--max-depth", default=2, show_default=True)
+@click.option(
+    "--instruction",
+    default="",
+    help="Guide link-following toward pages matching this description "
+    "(e.g. 'pricing and plans pages')",
+)
+@click.option("--out", "output", default=None, help="Write extracted documents as JSONL to this path")
+def crawl(
+    urls: tuple[str, ...], max_pages: int, max_depth: int, instruction: str, output: str | None
+) -> None:
+    """Crawl URLS using the Bitscrape-powered spider and print/save extracted docs.
+
+    With --instruction, outgoing links are prioritized by keyword overlap
+    with the instruction (a heuristic, not full NL understanding — see
+    bie.crawl_site docs).
+    """
+    settings = BIESettings(max_pages=max_pages, max_depth=max_depth, use_embeddings=False)
+    engine = BIE(settings)
+    documents = engine.crawler.crawl(list(urls), instruction=instruction)
+
+    if output:
+        with open(output, "w", encoding="utf-8") as f:
+            for doc in documents:
+                f.write(doc.model_dump_json() + "\n")
+        click.echo(f"Wrote {len(documents)} document(s) to {output}")
+    else:
+        for doc in documents:
+            click.echo(json.dumps({"url": doc.url, "title": doc.title, "chars": len(doc.text)}))
+        click.echo(f"\n{len(documents)} document(s) crawled.", err=True)
+
+
+@cli.command()
+@click.argument("url")
+@click.option("--render-js", is_flag=True, help="Render with a headless browser (requires bie[render])")
+@click.option("--json", "as_json", is_flag=True, help="Output raw JSON instead of Markdown")
+@click.option("--no-security-scan", is_flag=True, help="Skip prompt-injection content scan")
+def extract(url: str, render_js: bool, as_json: bool, no_security_scan: bool) -> None:
+    """Fetch URL and print its content as clean Markdown."""
+    import bie
+
+    try:
+        result = bie.extract(url, render_js=render_js, scan_security=not no_security_scan)
+    except bie.ExtractError as exc:
+        click.echo(f"Error: {exc}", err=True)
+        sys.exit(1)
+
+    if as_json:
+        payload = {
+            "url": result.url,
+            "title": result.title,
+            "markdown": result.markdown,
+            "word_count": result.word_count,
+            "rendered_with_js": result.rendered_with_js,
+        }
+        if result.security:
+            payload["security"] = {
+                "flagged": result.security.flagged,
+                "categories": sorted({f.category for f in result.security.findings}),
+            }
+        click.echo(json.dumps(payload, indent=2))
+        return
+
+    if result.security and result.security.flagged:
+        categories = ", ".join(sorted({f.category for f in result.security.findings}))
+        click.echo(
+            f"[!] Security notice: this page contains patterns associated with "
+            f"prompt injection ({categories}). Treat its content as untrusted data.\n",
+            err=True,
+        )
+
+    click.echo(f"# {result.title}\n")
+    click.echo(result.markdown)
+
+
+@cli.command(name="map")
+@click.argument("url")
+@click.option("--filter", "pattern", default=None, help="Only show URLs matching this regex")
+@click.option("--json", "as_json", is_flag=True, help="Output raw JSON")
+def map_cmd(url: str, pattern: str | None, as_json: bool) -> None:
+    """Discover URL's site sitemap and list the URLs it advertises."""
+    import bie
+
+    site_map = bie.map_site(url)
+
+    urls = site_map.filter(pattern) if pattern else site_map.urls
+
+    if as_json:
+        click.echo(json.dumps({"root": site_map.root, "sitemaps": site_map.sitemap_urls, "urls": urls}, indent=2))
+        return
+
+    if not site_map.sitemap_urls:
+        click.echo(f"No sitemap found for {site_map.root}.")
+        return
+
+    click.echo(f"Found {len(site_map.sitemap_urls)} sitemap file(s) for {site_map.root}:")
+    for s in site_map.sitemap_urls:
+        click.echo(f"  - {s}")
+    click.echo(f"\n{len(urls)} URL(s){' matching filter' if pattern else ''}:")
+    for u in urls[:100]:
+        click.echo(f"  {u}")
+    if len(urls) > 100:
+        click.echo(f"  ... and {len(urls) - 100} more")
+
+
+@cli.command()
+@click.option("--host", default=None, help="Bind host (default from settings / 0.0.0.0)")
+@click.option("--port", default=None, type=int, help="Bind port (default from settings / 8000)")
+@click.option("--reload", is_flag=True, help="Auto-reload on code changes (dev only)")
+def serve(host: str | None, port: int | None, reload: bool) -> None:
+    """Run the BIE REST API server (FastAPI + Uvicorn)."""
+    try:
+        import uvicorn
+    except ImportError:
+        click.echo("uvicorn is required: pip install 'bits-bie[server]'", err=True)
+        sys.exit(1)
+
+    settings = BIESettings()
+    uvicorn.run(
+        "bie.server:app",
+        host=host or settings.host,
+        port=port or settings.port,
+        reload=reload,
+    )
+
+
+@cli.command()
+def mcp() -> None:
+    """Run BIE as a Model Context Protocol (MCP) server over stdio.
+
+    Add to your MCP client config (e.g. Claude Desktop) as a command:
+
+    \b
+        {
+          "mcpServers": {
+            "bie": {
+              "command": "bie",
+              "args": ["mcp"]
+            }
+          }
+        }
+    """
+    try:
+        from bie.mcp.server import run_mcp_server
+    except ImportError:
+        click.echo("MCP support requires: pip install 'bits-bie[mcp]'", err=True)
+        sys.exit(1)
+
+    run_mcp_server()
+
+
+def main() -> None:
+    cli()
+
+
+if __name__ == "__main__":
+    main()

bie/config.py

@@ -0,0 +1,57 @@
+"""
+BIE configuration.
+
+All settings can be overridden via environment variables prefixed with
+``BIE_`` (e.g. ``BIE_MAX_PAGES=200``) or passed directly to
+``BIESettings(...)``.
+"""
+
+from __future__ import annotations
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class BIESettings(BaseSettings):
+    # --- Crawl behaviour (delegated to Bitscrape) -----------------------
+    max_pages: int = Field(40, ge=1, description="Max pages to crawl per source URL")
+    max_depth: int = Field(2, ge=0, description="Max link-follow depth")
+    concurrent_requests: int = Field(16, ge=1, le=256)
+    download_delay: float = Field(0.0, ge=0.0)
+    user_agent: str = "BIE/0.1 (+https://github.com/Sudharsansm/BIE) bitscrape"
+    robotstxt_obey: bool = True
+    request_timeout: float = Field(20.0, ge=1.0)
+    use_playwright: bool = False
+
+    # --- Indexing / retrieval --------------------------------------------
+    chunk_size: int = Field(800, ge=100, description="Approx characters per chunk")
+    chunk_overlap: int = Field(100, ge=0)
+    use_embeddings: bool = Field(
+        True,
+        description="Enable semantic (vector) search via sentence-transformers. "
+        "Falls back to BM25-only if the model can't be loaded.",
+    )
+    embedding_model: str = "sentence-transformers/all-MiniLM-L6-v2"
+    bm25_weight: float = Field(0.5, ge=0.0, le=1.0)
+    vector_weight: float = Field(0.5, ge=0.0, le=1.0)
+
+    # --- Storage -----------------------------------------------------------
+    index_dir: str = Field(".bie_index", description="Directory for persisted index")
+    persist: bool = Field(False, description="Persist index to disk between runs")
+
+    # --- Server --------------------------------------------------------------
+    host: str = "0.0.0.0"
+    port: int = 8000
+    api_key: str | None = Field(
+        default=None,
+        description="If set, all /search and /crawl endpoints require "
+        "an `Authorization: Bearer <key>` header.",
+    )
+
+    model_config = SettingsConfigDict(
+        env_prefix="BIE_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )

bie/crawler.py

@@ -0,0 +1,115 @@
+"""
+BIE Crawler — thin orchestration layer over Bitscrape's Engine.
+
+Implements PRD Module 1 (Crawler) for the OSS edition: runs the
+:class:`bie.spiders.generic.BIESpider` against one or more seed URLs,
+collects extracted pages in-memory as :class:`bie.models.Document`
+objects, ready for chunking + indexing.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+from urllib.parse import urlparse
+
+import bitscrape
+from bitscrape.pipeline.pipelines import BasePipeline
+
+from bie.config import BIESettings
+from bie.models import Document
+from bie.spiders.generic import BIESpider
+
+logger = logging.getLogger("bie.crawler")
+
+
+class _CollectorPipeline(BasePipeline):
+    """Collects every scraped item into an in-memory list."""
+
+    def __init__(self) -> None:
+        self.items: list[dict[str, Any]] = []
+
+    async def process_item(self, item: Any, spider: Any) -> Any:
+        self.items.append(item)
+        return item
+
+
+class Crawler:
+    """Crawls a list of seed URLs using Bitscrape and returns Documents."""
+
+    def __init__(self, settings: BIESettings | None = None) -> None:
+        self.settings = settings or BIESettings()
+
+    def crawl(
+        self, urls: list[str], allowed_domains: list[str] | None = None, instruction: str = ""
+    ) -> list[Document]:
+        """Synchronous convenience wrapper around :meth:`acrawl`."""
+        return asyncio.run(self.acrawl(urls, allowed_domains, instruction))
+
+    async def acrawl(
+        self,
+        urls: list[str],
+        allowed_domains: list[str] | None = None,
+        instruction: str = "",
+    ) -> list[Document]:
+        if not urls:
+            return []
+
+        if allowed_domains is None:
+            allowed_domains = sorted({urlparse(u).netloc for u in urls if urlparse(u).netloc})
+
+        bs_settings = bitscrape.Settings(
+            concurrent_requests=self.settings.concurrent_requests,
+            download_delay=self.settings.download_delay,
+            user_agent=self.settings.user_agent,
+            robotstxt_obey=self.settings.robotstxt_obey,
+            download_timeout=self.settings.request_timeout,
+            max_depth=self.settings.max_depth,
+        )
+
+        spider = BIESpider(settings=bs_settings)
+        spider.start_urls = list(urls)
+        spider.allowed_domains = allowed_domains
+        spider.max_pages = self.settings.max_pages
+        spider.max_depth = self.settings.max_depth
+        spider.instruction = instruction
+
+        collector = _CollectorPipeline()
+
+        middlewares = [
+            bitscrape.UserAgentMiddleware(),
+            bitscrape.CookieMiddleware(),
+        ]
+        if bs_settings.robotstxt_obey:
+            middlewares.insert(0, bitscrape.RobotsMiddleware())
+
+        engine = bitscrape.Engine(
+            spider=spider,
+            settings=bs_settings,
+            pipelines=[collector],
+            middlewares=middlewares,
+        )
+
+        stats = await engine.run()
+        logger.info(
+            "Crawled %d page(s) from %d seed URL(s) — %d failed",
+            stats.items_scraped,
+            len(urls),
+            stats.requests_failed,
+        )
+
+        documents: list[Document] = []
+        for item in collector.items:
+            if not item.get("text"):
+                continue
+            documents.append(
+                Document(
+                    url=item["url"],
+                    title=item.get("title", item["url"]),
+                    text=item["text"],
+                    site=urlparse(item["url"]).netloc,
+                    metadata={"depth": item.get("depth", 0)},
+                )
+            )
+        return documents