doculayer 0.1.0__py3-none-any.whl

doculayer/__init__.py

@@ -0,0 +1,142 @@
+"""DocuLayer: live documentation access layer for AI agents.
+
+Zero hallucination — content is fetched verbatim from source, never generated.
+No disk storage  — ephemeral in-memory TTL cache only.
+Quick access     — llms.txt-first with BM25 section search.
+
+Quick start
+-----------
+    import asyncio
+    from doculayer import search, fetch
+
+    results = asyncio.run(search("dependency injection", "fastapi"))
+    for r in results:
+        print(r.section.source_url, r.score)
+        print(r.section.trimmed(200))
+
+    content = asyncio.run(fetch("httpx", section="AsyncClient"))
+    print(content)
+"""
+
+from __future__ import annotations
+
+from .config import DocuLayerConfig, get_config
+from .core.citation import attr_header
+from .core.fetcher import DocFetcher, FetchResult
+from .core.parser import DocParser, DocSection
+from .core.search import DocSearcher, SearchResult
+from .sources.generic import GenericDocSource
+from .sources.registry import resolve_identifier
+
+__version__ = "0.1.0"
+__all__ = [
+    "DocuLayerConfig",
+    "get_config",
+    "DocFetcher",
+    "FetchResult",
+    "DocParser",
+    "DocSection",
+    "DocSearcher",
+    "SearchResult",
+    "GenericDocSource",
+    "resolve_identifier",
+    "search",
+    "fetch",
+]
+
+# ── module-level singleton fetcher ────────────────────────────────────────────
+
+_fetcher: DocFetcher | None = None
+
+
+def _get_fetcher() -> DocFetcher:
+    global _fetcher
+    if _fetcher is None:
+        cfg = get_config()
+        _fetcher = DocFetcher(
+            ttl=cfg.cache_ttl,
+            max_size=cfg.max_cache_size,
+            timeout=cfg.fetch_timeout,
+            max_bytes=cfg.max_fetch_bytes,
+        )
+    return _fetcher
+
+
+# ── high-level async API ──────────────────────────────────────────────────────
+
+async def search(
+    query: str,
+    identifier: str,
+    top_k: int = 5,
+) -> list[SearchResult]:
+    """Search live documentation for *query*.
+
+    Args:
+        query:      What to find — e.g. ``"dependency injection"``.
+        identifier: Package name or URL — ``"fastapi"``, ``"pypi:httpx"``,
+                    ``"npm:react"``, ``"https://docs.example.com"``.
+        top_k:      Maximum number of sections to return.
+
+    Returns:
+        List of :class:`SearchResult` objects.  Every section is verbatim
+        text from the source — ``section.source_url`` identifies the origin.
+
+    Raises:
+        ValueError: if *identifier* cannot be resolved to a URL.
+    """
+    url = await resolve_identifier(identifier)
+    if not url:
+        raise ValueError(f"Cannot resolve documentation source: {identifier!r}")
+    return await GenericDocSource(url, _get_fetcher(), label=identifier).search(
+        query, top_k
+    )
+
+
+async def fetch(
+    identifier: str,
+    section: str | None = None,
+) -> str:
+    """Fetch documentation for *identifier*.
+
+    Args:
+        identifier: Package name or URL.
+        section:    Optional heading to extract — e.g. ``"Installation"``.
+
+    Returns:
+        Markdown string with a source attribution header.
+        Content is verbatim from the fetched page.
+
+    Raises:
+        ValueError: if *identifier* cannot be resolved.
+    """
+    url = await resolve_identifier(identifier)
+    if not url:
+        raise ValueError(f"Cannot resolve documentation source: {identifier!r}")
+
+    src = GenericDocSource(url, _get_fetcher(), label=identifier)
+    cfg = get_config()
+
+    if section:
+        s = await src.fetch_section(section)
+        if s:
+            return s.cited_content
+        return f"Section '{section}' not found in {identifier} docs."
+
+    sections = await src.fetch_page()
+    if not sections:
+        return f"No content found at {url}"
+
+    s0 = sections[0]
+    header = attr_header(s0.source_url, s0.fetched_at, s0.from_cache) + "\n\n"
+    limit = cfg.max_section_words * 4
+    parts: list[str] = []
+    total = 0
+    for s in sections:
+        wc = len(s.content.split())
+        if total + wc > limit:
+            parts.append("*(truncated — use `section=` for targeted access)*")
+            break
+        parts.append(s.content)
+        total += wc
+
+    return header + "\n\n".join(parts)

doculayer/cli.py

@@ -0,0 +1,132 @@
+"""DocuLayer CLI.
+
+Usage
+-----
+  doculayer setup                      # auto-detect IDE and install MCP server
+  doculayer search "dependency injection" --source fastapi
+  doculayer fetch httpx --section AsyncClient
+  doculayer resolve pydantic
+  doculayer sources
+  doculayer mcp          # start MCP server on stdio
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+
+import click
+
+from . import fetch as _fetch
+from . import search as _search
+from .setup_wizard import setup as _setup_cmd
+from .sources.registry import resolve_identifier
+
+
+@click.group()
+@click.version_option(package_name="doculayer")
+def main() -> None:
+    """DocuLayer: live documentation access for AI agents."""
+
+
+
+main.add_command(_setup_cmd)
+
+# ── search ─────────────────────────────────────────────────────────────────────
+
+@main.command()
+@click.argument("query")
+@click.option("--source", "-s", required=True, help="Package name or URL to search")
+@click.option("--results", "-n", default=3, show_default=True, help="Max results")
+def search(query: str, source: str, results: int) -> None:
+    """Search QUERY in live documentation for SOURCE."""
+
+    async def _run() -> None:
+        try:
+            hits = await _search(query, source, top_k=results)
+        except ValueError as e:
+            click.echo(str(e), err=True)
+            sys.exit(1)
+
+        if not hits:
+            click.echo(f"No results for '{query}' in {source}.", err=True)
+            sys.exit(1)
+
+        for i, r in enumerate(hits, 1):
+            click.echo(f"\n{'─' * 60}")
+            click.echo(f"  {i}. {r.section.title}  (score {r.score:.2f})")
+            click.echo(f"     {r.section.source_url}")
+            click.echo(f"{'─' * 60}")
+            click.echo(r.section.trimmed(200))
+
+    asyncio.run(_run())
+
+
+# ── fetch ──────────────────────────────────────────────────────────────────────
+
+@main.command()
+@click.argument("identifier")
+@click.option("--section", "-S", default=None, help="Section heading to extract")
+def fetch(identifier: str, section: str | None) -> None:
+    """Fetch documentation for IDENTIFIER."""
+
+    async def _run() -> None:
+        try:
+            content = await _fetch(identifier, section=section)
+        except ValueError as e:
+            click.echo(str(e), err=True)
+            sys.exit(1)
+        click.echo(content)
+
+    asyncio.run(_run())
+
+
+# ── resolve ────────────────────────────────────────────────────────────────────
+
+@main.command()
+@click.argument("identifier")
+def resolve(identifier: str) -> None:
+    """Print the resolved documentation URL for IDENTIFIER."""
+
+    async def _run() -> None:
+        url = await resolve_identifier(identifier)
+        if url:
+            click.echo(url)
+        else:
+            click.echo(f"Cannot resolve: {identifier}", err=True)
+            sys.exit(1)
+
+    asyncio.run(_run())
+
+
+# ── sources ────────────────────────────────────────────────────────────────────
+
+@main.command()
+def sources() -> None:
+    """List packages with built-in llms.txt support."""
+    from .sources.registry import KNOWN_LLMS_TXT, KNOWN_URLS
+
+    click.echo("Packages with llms.txt support (fast targeted access):")
+    for name in sorted(KNOWN_LLMS_TXT):
+        click.echo(f"  {name}")
+    if KNOWN_URLS:
+        click.echo()
+        click.echo("Packages with direct URL shortcuts:")
+        for name, url in sorted(KNOWN_URLS.items()):
+            click.echo(f"  {name:<16} {url}")
+    click.echo()
+    click.echo("Identifier formats:  bare-name · pypi: · npm: · gh: · https://")
+
+
+# ── mcp ────────────────────────────────────────────────────────────────────────
+
+@main.command("mcp")
+def mcp_serve() -> None:
+    """Start DocuLayer as an MCP server (stdio transport)."""
+    from .mcp_server import main as _mcp_main
+
+    _mcp_main()
+
+
+if __name__ == "__main__":
+    main()

doculayer/config.py

@@ -0,0 +1,31 @@
+"""DocuLayer configuration.
+
+All settings read from environment variables with sane defaults.
+No config files — no disk reads.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import os
+
+
+@dataclass
+class DocuLayerConfig:
+    # ── cache (in-memory only; no disk persistence) ──────────────────────────
+    cache_ttl: int      = int(os.getenv("DOCULAYER_CACHE_TTL",   "3600"))  # seconds
+    max_cache_size: int = int(os.getenv("DOCULAYER_MAX_CACHE",   "256"))   # entries
+
+    # ── HTTP fetch ────────────────────────────────────────────────────────────
+    fetch_timeout: float  = float(os.getenv("DOCULAYER_FETCH_TIMEOUT", "12.0"))
+    max_fetch_bytes: int  = int(os.getenv("DOCULAYER_MAX_BYTES",  str(512 * 1024)))
+
+    # ── output shaping ────────────────────────────────────────────────────────
+    max_section_words: int = int(os.getenv("DOCULAYER_MAX_WORDS", "400"))
+
+
+_default = DocuLayerConfig()
+
+
+def get_config() -> DocuLayerConfig:
+    return _default

doculayer/core/__init__.py

@@ -0,0 +1 @@
+# doculayer core

doculayer/core/cache.py

@@ -0,0 +1,73 @@
+"""Process-local TTL cache.
+
+Zero disk I/O by design. Cache entries evaporate when the process exits.
+This is the storage guarantee DocuLayer makes: no docs are persisted.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import Generic, Optional, TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass
+class _Entry(Generic[T]):
+    value: T
+    expires_at: float  # monotonic clock
+
+    @property
+    def alive(self) -> bool:
+        return time.monotonic() < self.expires_at
+
+
+class TTLCache(Generic[T]):
+    """In-memory TTL cache.
+
+    Eviction policy: expired-first, then earliest-expiring on overflow.
+    Thread safety: single-threaded (asyncio); no locks needed.
+    """
+
+    def __init__(self, ttl: int, max_size: int = 256) -> None:
+        self._ttl = ttl
+        self._max = max_size
+        self._store: dict[str, _Entry[T]] = {}
+
+    # ── public ────────────────────────────────────────────────────────────────
+
+    def get(self, key: str) -> Optional[T]:
+        entry = self._store.get(key)
+        if entry is None:
+            return None
+        if not entry.alive:
+            del self._store[key]
+            return None
+        return entry.value
+
+    def set(self, key: str, value: T) -> None:
+        if key not in self._store and len(self._store) >= self._max:
+            self._evict()
+        self._store[key] = _Entry(value=value, expires_at=time.monotonic() + self._ttl)
+
+    def invalidate(self, key: str) -> None:
+        self._store.pop(key, None)
+
+    def clear(self) -> None:
+        self._store.clear()
+
+    @property
+    def stats(self) -> dict[str, int]:
+        live = sum(1 for v in self._store.values() if v.alive)
+        return {"alive": live, "total": len(self._store)}
+
+    # ── private ───────────────────────────────────────────────────────────────
+
+    def _evict(self) -> None:
+        dead = [k for k, v in self._store.items() if not v.alive]
+        for k in dead:
+            del self._store[k]
+        if len(self._store) >= self._max:
+            victim = min(self._store, key=lambda k: self._store[k].expires_at)
+            del self._store[victim]

doculayer/core/citation.py

@@ -0,0 +1,48 @@
+"""Shared citation formatting for DocuLayer.
+
+Every piece of documentation returned to a caller MUST include provenance.
+This module provides a single source of truth for that formatting so no
+output path can silently omit a source attribution.
+
+Design guarantee
+----------------
+``attr_header`` and ``cited_block`` are the only functions that produce
+attribution strings.  Import them here — never duplicate the format.
+"""
+
+from __future__ import annotations
+
+import time
+
+
+def attr_header(url: str, fetched_at: float, from_cache: bool) -> str:
+    """Return a Markdown blockquote attribution line.
+
+    Example output::
+
+        > **Source**: https://fastapi.tiangolo.com/tutorial/  \\
+        > **Fetched**: 42s ago  *(cached)*
+
+    Args:
+        url:        The canonical URL the content was fetched from.
+        fetched_at: Unix timestamp of the fetch (``time.time()``).
+        from_cache: Whether the result was served from the in-memory cache.
+    """
+    age = time.time() - fetched_at
+    if age < 60:
+        age_s = f"{int(age)}s"
+    elif age < 3600:
+        age_s = f"{int(age / 60)}m"
+    else:
+        age_s = f"{int(age / 3600)}h"
+    note = "  *(cached)*" if from_cache else ""
+    return f"> **Source**: {url}  \n> **Fetched**: {age_s} ago{note}"
+
+
+def cited_block(content: str, url: str, fetched_at: float, from_cache: bool) -> str:
+    """Return *content* preceded by a citation header.
+
+    Use this whenever a whole section of documentation is returned so the
+    caller always sees provenance before the content.
+    """
+    return f"{attr_header(url, fetched_at, from_cache)}\n\n{content}"

doculayer/core/fetcher.py

@@ -0,0 +1,102 @@
+"""Async HTTP fetcher backed by TTLCache.
+
+Guarantees:
+  - No file-system writes.
+  - Stale content is never served past the TTL.
+  - Hard byte cap prevents oversized responses from swamping memory.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import time
+from dataclasses import dataclass
+
+import httpx
+
+from .cache import TTLCache
+
+
+@dataclass
+class FetchResult:
+    url: str            # final URL after redirects
+    content: str
+    content_type: str
+    fetched_at: float   # Unix timestamp
+    from_cache: bool
+
+    def age_label(self) -> str:
+        secs = time.time() - self.fetched_at
+        if secs < 60:
+            return f"{int(secs)}s"
+        if secs < 3600:
+            return f"{int(secs / 60)}m"
+        return f"{int(secs / 3600)}h"
+
+
+class DocFetcher:
+    """Fetch documentation pages with TTL in-memory caching."""
+
+    _HEADERS = {
+        "User-Agent": "DocuLayer/1.0 (live-doc-access-layer; no-hallucination)",
+        "Accept": "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    def __init__(
+        self,
+        ttl: int = 3600,
+        max_size: int = 256,
+        timeout: float = 12.0,
+        max_bytes: int = 524_288,
+    ) -> None:
+        self._cache: TTLCache[FetchResult] = TTLCache(ttl, max_size)
+        self._ttl = ttl
+        self._timeout = timeout
+        self._max_bytes = max_bytes
+
+    @staticmethod
+    def _key(url: str) -> str:
+        return hashlib.sha256(url.encode()).hexdigest()[:20]
+
+    async def fetch(self, url: str, force: bool = False) -> FetchResult:
+        """Fetch *url*, returning a cached result when still live."""
+        key = self._key(url)
+
+        if not force:
+            hit = self._cache.get(key)
+            if hit is not None:
+                return FetchResult(
+                    url=hit.url,
+                    content=hit.content,
+                    content_type=hit.content_type,
+                    fetched_at=hit.fetched_at,
+                    from_cache=True,
+                )
+
+        async with httpx.AsyncClient(
+            follow_redirects=True, timeout=self._timeout
+        ) as client:
+            resp = await client.get(url, headers=self._HEADERS)
+            resp.raise_for_status()
+
+            content = resp.text[: self._max_bytes]
+            ct = resp.headers.get("content-type", "text/html")
+
+            result = FetchResult(
+                url=str(resp.url),
+                content=content,
+                content_type=ct,
+                fetched_at=time.time(),
+                from_cache=False,
+            )
+            self._cache.set(key, result)
+            return result
+
+    @property
+    def cache_stats(self) -> dict[str, int]:
+        return self._cache.stats
+
+    @property
+    def ttl(self) -> int:
+        return self._ttl

doculayer/core/parser.py

@@ -0,0 +1,156 @@
+"""Parse HTML or Markdown documentation into a flat list of DocSection objects.
+
+Sections map 1-to-1 with headings.  The parser never generates text —
+every word in a DocSection came verbatim from the fetched source.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from dataclasses import dataclass, field
+from typing import Optional
+
+import markdownify
+from bs4 import BeautifulSoup
+
+from .citation import attr_header
+
+
+@dataclass
+class DocSection:
+    title: str
+    content: str        # Markdown, verbatim from source
+    level: int          # heading depth: 0 = pre-heading preamble, 1-6 = h1-h6
+    anchor: Optional[str]
+    source_url: str
+    fetched_at: float   = field(default_factory=time.time)
+    from_cache: bool    = False
+
+    def trimmed(self, max_words: int) -> str:
+        """Return content, hard-truncated to *max_words*."""
+        words = self.content.split()
+        if len(words) <= max_words:
+            return self.content
+        return " ".join(words[:max_words]) + "\n\n*(content truncated)*"
+
+    @property
+    def cited_content(self) -> str:
+        """Content with a Markdown blockquote citation prepended.
+
+        Use this property whenever returning documentation to a caller so
+        provenance is structurally inseparable from the text.  Example::
+
+            > **Source**: https://fastapi.tiangolo.com/tutorial/
+            > **Fetched**: 12s ago
+
+            ## Dependency Injection
+            ...
+        """
+        return f"{attr_header(self.source_url, self.fetched_at, self.from_cache)}\n\n{self.content}"
+
+
+# ── HTML noise selectors ──────────────────────────────────────────────────────
+
+_NOISE_SEL = [
+    "nav", "header", "footer", "aside",
+    ".sidebar", ".nav", ".navbar", ".header", ".footer",
+    ".toc", ".table-of-contents", "#sidebar", ".breadcrumb",
+    ".pagination", ".cookie-banner", ".advertisement",
+    "[role=navigation]", "[role=banner]", "[role=complementary]",
+    "script", "style", "noscript", "iframe",
+]
+
+# Heading pattern: lines starting with 1-6 # characters
+_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE)
+# Pandoc-style heading anchors: {#my-anchor}
+_ANCHOR_RE  = re.compile(r"\{#([^}]+)\}")
+
+
+class DocParser:
+    """Stateless parser for HTML and Markdown documentation pages."""
+
+    def parse(
+        self,
+        content: str,
+        url: str,
+        content_type: str = "text/html",
+        fetched_at: float = 0.0,
+        from_cache: bool = False,
+    ) -> list[DocSection]:
+        """Parse *content* into sections.
+
+        HTML is stripped and converted to Markdown first.
+        Markdown is split at heading boundaries.
+        """
+        ts = fetched_at or time.time()
+        if "html" in content_type.lower():
+            md = self._html_to_md(content)
+        else:
+            md = content
+        return self._split_md(md, url, ts, from_cache)
+
+    # ── private ───────────────────────────────────────────────────────────────
+
+    @staticmethod
+    def _html_to_md(html: str) -> str:
+        soup = BeautifulSoup(html, "lxml")
+
+        # Strip navigation / chrome
+        for sel in _NOISE_SEL:
+            for el in soup.select(sel):
+                el.decompose()
+
+        # Find main content region
+        main = (
+            soup.find("main")
+            or soup.find(id=re.compile(r"^(content|main|body)$", re.I))
+            or soup.find(
+                class_=re.compile(r"\b(content|main|body|article|post|docs)\b", re.I)
+            )
+            or soup.find("article")
+            or soup.body
+        )
+
+        return markdownify.markdownify(
+            str(main or soup), heading_style="ATX", strip=["img"]
+        )
+
+    @staticmethod
+    def _split_md(
+        md: str, url: str, fetched_at: float, from_cache: bool
+    ) -> list[DocSection]:
+        md = re.sub(r"\n{3,}", "\n\n", md.strip())
+        matches = list(_HEADING_RE.finditer(md))
+        sections: list[DocSection] = []
+
+        if not matches:
+            if md.strip():
+                sections.append(
+                    DocSection("Documentation", md.strip(), 0, None, url, fetched_at, from_cache)
+                )
+            return sections
+
+        # Content that appears before the first heading
+        preamble = md[: matches[0].start()].strip()
+        if preamble:
+            sections.append(DocSection("Overview", preamble, 0, None, url, fetched_at, from_cache))
+
+        for i, m in enumerate(matches):
+            level = len(m.group(1))
+            raw_title = m.group(2).strip()
+
+            anchor_m = _ANCHOR_RE.search(raw_title)
+            anchor   = anchor_m.group(1) if anchor_m else None
+            title    = _ANCHOR_RE.sub("", raw_title).strip()
+
+            body_start = m.end()
+            body_end   = matches[i + 1].start() if i + 1 < len(matches) else len(md)
+            body       = md[body_start:body_end].strip()
+
+            full = f"{'#' * level} {title}\n\n{body}" if body else f"{'#' * level} {title}"
+            sections.append(
+                DocSection(title, full, level, anchor, url, fetched_at, from_cache)
+            )
+
+        return sections