jevcrawl 0.1.0__py3-none-any.whl

jevcrawl/__init__.py

@@ -0,0 +1,6 @@
+"""jevcrawl: crawl websites for AI agents, with Jev making every crawl decision."""
+
+__version__ = "0.1.0"
+REPO_URL = "https://github.com/RayenMlayeh/jevcrawl"
+
+__all__ = ["REPO_URL", "__version__"]

jevcrawl/cli.py

@@ -0,0 +1,159 @@
+"""Command line entry point: ``jevcrawl`` (MCP server on stdio) and ``jevcrawl <command>``."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from typing import Any
+
+from jevcrawl import __version__
+from jevcrawl.config import Config, load_config
+from jevcrawl.crawl import CrawlError, crawl, map_site, scrape
+from jevcrawl.crawl.errors import AUTH_HELP
+from jevcrawl.log import configure_logging
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """The argument parser. With no command the MCP server starts."""
+    parser = argparse.ArgumentParser(
+        prog="jevcrawl",
+        description="Crawl websites for AI agents; Jev makes every crawl decision. "
+        "With no command, run the MCP server on stdio.",
+    )
+    parser.add_argument("--version", action="version", version=f"jevcrawl {__version__}")
+    parser.add_argument("-v", "--verbose", action="count", default=0, help="log to stderr")
+    sub = parser.add_subparsers(dest="command")
+
+    def common(p: argparse.ArgumentParser) -> None:
+        p.add_argument("--json", action="store_true", help="print the full result as JSON")
+
+    p = sub.add_parser("crawl", help="crawl from URL toward a goal")
+    p.add_argument("url")
+    p.add_argument("--goal", required=True, help="a specific question to answer")
+    p.add_argument("--max-pages", type=int, default=30)
+    p.add_argument("--max-seconds", type=float, default=120)
+    p.add_argument("--return-tokens", type=int, default=4000)
+    p.add_argument("--max-usd", type=float, default=0.05, help="Jev spend limit (max_decision_usd)")
+    common(p)
+
+    p = sub.add_parser("scrape", help="fetch one page, optionally pruned to a goal")
+    p.add_argument("url")
+    p.add_argument("--goal")
+    p.add_argument("--max-usd", type=float, default=0.05)
+    common(p)
+
+    p = sub.add_parser("map", help="rank the URLs of a site for a goal")
+    p.add_argument("url")
+    p.add_argument("--goal", required=True)
+    p.add_argument("--limit", type=int, default=50)
+    p.add_argument("--max-pages", type=int, default=10)
+    p.add_argument("--max-usd", type=float, default=0.05)
+    common(p)
+
+    sub.add_parser("doctor", help="check the API key and the connection to Jev")
+    return parser
+
+
+async def _run(args: argparse.Namespace, config: Config) -> dict[str, Any]:
+    if args.command == "crawl":
+        result = await crawl(
+            args.goal,
+            args.url,
+            config,
+            max_pages=args.max_pages,
+            max_seconds=args.max_seconds,
+            return_tokens=args.return_tokens,
+            max_decision_usd=args.max_usd,
+        )
+        return result.to_dict()
+    if args.command == "scrape":
+        return (await scrape(args.url, config, args.goal, max_decision_usd=args.max_usd)).to_dict()
+    return await map_site(
+        args.url,
+        args.goal,
+        config,
+        limit=args.limit,
+        max_pages=args.max_pages,
+        max_decision_usd=args.max_usd,
+    )
+
+
+def format_result(command: str, data: dict[str, Any]) -> str:
+    """Human-readable rendering of a result."""
+    lines: list[str] = []
+    if command == "map":
+        for item in data["urls"]:
+            lines.append(f"{item['priority']:.3f}  {item['url']}  ({item['anchor'] or '-'})")
+        if not data["urls"]:
+            lines.append("(no promising links found)")
+    else:
+        for title, key in (("kept", "blocks"), ("uncertain", "uncertain")):
+            for block in data[key]:
+                score = "  -  " if block["relevance"] is None else f"{block['relevance']:.2f}"
+                where = f"{block['url']} [{block['char_start']}:{block['char_end']}]"
+                heading = f"  ({block['heading_path']})" if block["heading_path"] else ""
+                lines.append(f"[{title} {score}] {where}{heading}")
+                lines.append("  " + block["text"].replace("\n", "\n  "))
+        if not (data["blocks"] or data["uncertain"]):
+            lines.append("(no relevant blocks found)")
+    s = data["stats"]
+    lines.append("")
+    summary = f"pages={s['pages_fetched']} stopped={s['stopped_because']}"
+    if "tokens_raw" in s:
+        summary += f" tokens={s['tokens_raw']:,}->{s['tokens_returned']:,}"
+    summary += f" jev_calls={s['jev_calls']} cost=${s['jev_cost_usd']:.4f}"
+    if "elapsed_seconds" in s:
+        summary += f" time={s['elapsed_seconds']:.1f}s"
+    if s["degraded"]:
+        summary += f" degraded ({s['degraded_reason']})"
+    lines.append(summary)
+    if s["needs_js_urls"]:
+        lines.append(f"needs JavaScript (not rendered): {', '.join(s['needs_js_urls'])}")
+    return "\n".join(lines)
+
+
+def _utf8_streams() -> None:
+    """Page text is arbitrary Unicode; never let a legacy console encoding crash the output."""
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)
+        if reconfigure is not None:
+            reconfigure(encoding="utf-8", errors="replace")
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Console script ``jevcrawl``."""
+    _utf8_streams()
+    args = build_parser().parse_args(argv)
+    configure_logging({0: "WARNING", 1: "INFO"}.get(args.verbose, "DEBUG"))
+    if args.command is None:
+        from jevcrawl.mcp_server import main as serve
+
+        serve()
+        return 0
+    if args.command == "doctor":
+        from jevcrawl.doctor import run_doctor
+
+        return asyncio.run(run_doctor(load_config()))
+    try:
+        data = asyncio.run(_run(args, load_config()))
+        if data["stats"]["stopped_because"] == "auth_error":
+            raise CrawlError("auth_error", AUTH_HELP)
+    except CrawlError as exc:
+        error = exc.to_dict()
+        if args.json:
+            print(json.dumps(error, indent=2))
+        else:
+            print(f"error: {error['message']}", file=sys.stderr)
+        return 1
+    print(
+        json.dumps(data, indent=2, ensure_ascii=False)
+        if args.json
+        else format_result(args.command, data)
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

jevcrawl/config.py

@@ -0,0 +1,63 @@
+"""Settings. Four environment variables; everything else is a sensible default."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+
+DEFAULT_CACHE_DIR = Path("~/.jevcrawl/cache").expanduser()
+
+
+@dataclass(frozen=True)
+class Thresholds:
+    """Decision thresholds, calibrated on real Jev scores (see README, "How it works")."""
+
+    keep: float = 0.35  # keep a block at or above this (recall first)
+    uncertain_low: float = 0.20  # blocks in [uncertain_low, keep) are returned as "uncertain"
+    stop: float = 0.85  # stop when "do the kept blocks answer the goal?" reaches this
+    link_floor: float = 0.10  # links below this priority wait in a reserve
+    hub_weight: float = 0.6  # weight of "leads to a section about the goal" in link priority
+    # How much a page's best block score scales the links found on it. Small on purpose: on
+    # listing pages the items are links, not answers, so their block scores are near 0.
+    page_weight: float = 0.2
+    depth_decay: float = 0.9  # per click from the start page
+
+
+@dataclass(frozen=True)
+class Config:
+    """Resolved settings for one run."""
+
+    api_key: str | None = None
+    base_url: str = "https://api.typesafe.ai"  # any Jev-compatible endpoint works
+    model: str = "jev-1.13.0"  # pinned: thresholds are calibrated per model version
+    price_per_mtok: float = 0.042  # USD per million Jev input tokens; output is free
+    cache_dir: Path = DEFAULT_CACHE_DIR
+    same_domain: bool = True  # stay on the start host and its subdomains
+    links_per_page: int = 100  # links per page that get a Jev question
+    block_tokens: int = 300  # target size of a text block
+    max_kept_per_page: int = 10
+    fetch_concurrency: int = 4  # pages fetched at the same time
+    use_sitemap: bool = True  # offer sitemap URLs that share words with the goal as links
+    sitemap_links: int = 60
+    keep_layout_tags: bool = False  # treat nav/aside/footer text as content
+    thresholds: Thresholds = field(default_factory=Thresholds)
+
+
+def load_config(env: Mapping[str, str] | None = None) -> Config:
+    """Settings from ``TYPESAFE_API_KEY``, ``JEVCRAWL_BASE_URL``, ``JEVCRAWL_MODEL`` and
+    ``JEVCRAWL_CACHE_DIR``; everything else keeps its default."""
+    env = os.environ if env is None else env
+
+    def get(name: str) -> str | None:
+        return env.get(name, "").strip() or None
+
+    defaults = Config()
+    cache_dir = get("JEVCRAWL_CACHE_DIR")
+    return Config(
+        api_key=get("TYPESAFE_API_KEY"),
+        base_url=(get("JEVCRAWL_BASE_URL") or defaults.base_url).rstrip("/"),
+        model=get("JEVCRAWL_MODEL") or defaults.model,
+        cache_dir=Path(cache_dir).expanduser() if cache_dir else defaults.cache_dir,
+    )

jevcrawl/crawl/__init__.py

@@ -0,0 +1,8 @@
+"""Crawl package: frontier, loop, stop conditions, result packing, scrape and map."""
+
+from jevcrawl.crawl.errors import CrawlError
+from jevcrawl.crawl.loop import crawl
+from jevcrawl.crawl.map import map_site
+from jevcrawl.crawl.scrape import scrape
+
+__all__ = ["CrawlError", "crawl", "map_site", "scrape"]

jevcrawl/crawl/errors.py

@@ -0,0 +1,22 @@
+"""Expected failures of a crawl, with a stable code and a message that says what to do."""
+
+from __future__ import annotations
+
+
+class CrawlError(Exception):
+    """An expected error. Tools return ``{"error": code, "message": ...}`` instead of raising."""
+
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+    def to_dict(self) -> dict[str, str]:
+        """The error payload returned to agents."""
+        return {"error": self.code, "message": self.message}
+
+
+AUTH_HELP = (
+    "The Jev API key is missing or was rejected. Set the TYPESAFE_API_KEY environment variable "
+    "(create a key at https://console.typesafe.ai/)."
+)

jevcrawl/crawl/frontier.py

@@ -0,0 +1,98 @@
+"""Best-first frontier: always fetch the link with the highest priority next."""
+
+from __future__ import annotations
+
+import heapq
+import itertools
+from dataclasses import dataclass
+
+from jevcrawl.config import Thresholds
+
+
+@dataclass(frozen=True)
+class FrontierItem:
+    """A queued URL and where it came from."""
+
+    url: str
+    priority: float
+    depth: int
+    parent_url: str | None = None
+    anchor: str = ""
+    p_target: float | None = None
+    p_hub: float | None = None
+
+
+def link_priority(
+    p_target: float, p_hub: float, page_rel: float, depth: int, profile: Thresholds
+) -> float:
+    """``max(p_target, hub_weight * p_hub) * page_factor * depth_decay ** (depth + 1)``.
+
+    ``page_factor = (1 - page_weight) + page_weight * page_rel``.
+
+    ``depth`` is the depth of the page the link was found on; ``page_rel`` is the highest block
+    relevance on that page (0 if none).
+    """
+    best = max(p_target, profile.hub_weight * p_hub)
+    page_factor = (1 - profile.page_weight) + profile.page_weight * page_rel
+    return best * page_factor * profile.depth_decay ** (depth + 1)
+
+
+class Frontier:
+    """A heap of ``(-priority, seq, item)``. Each URL is queued at most once.
+
+    :meth:`breadth_first` switches to ``(depth, seq)`` ordering for degraded mode.
+    """
+
+    def __init__(self) -> None:
+        self._seq = itertools.count()
+        self._heap: list[tuple[float, int, FrontierItem]] = []
+        self._queued: set[str] = set()
+        self._bfs = False
+
+    def __len__(self) -> int:
+        return len(self._heap)
+
+    @property
+    def queued(self) -> set[str]:
+        """URLs currently waiting in the frontier."""
+        return self._queued
+
+    @property
+    def is_breadth_first(self) -> bool:
+        """True once degraded mode switched the ordering."""
+        return self._bfs
+
+    def push(
+        self,
+        url: str,
+        priority: float,
+        depth: int,
+        parent_url: str | None = None,
+        *,
+        anchor: str = "",
+        p_target: float | None = None,
+        p_hub: float | None = None,
+    ) -> bool:
+        """Queue ``url``; returns False if it was already queued."""
+        if url in self._queued:
+            return False
+        item = FrontierItem(url, priority, depth, parent_url, anchor, p_target, p_hub)
+        heapq.heappush(self._heap, (self._key(item), next(self._seq), item))
+        self._queued.add(url)
+        return True
+
+    def pop(self) -> FrontierItem:
+        """Remove and return the next item to fetch."""
+        _, _, item = heapq.heappop(self._heap)
+        self._queued.discard(item.url)
+        return item
+
+    def breadth_first(self) -> None:
+        """Re-order the queue by depth (then arrival order); new pushes follow the same rule."""
+        self._bfs = True
+        old = list(self._heap)
+        self._heap = [(self._key(item), seq, item) for _, seq, item in old]
+        heapq.heapify(self._heap)
+
+    def _key(self, item: FrontierItem) -> float:
+        return float(item.depth) if self._bfs else -item.priority