bytecrawl 1.0.0__py3-none-any.whl

bytecrawl/__init__.py

@@ -0,0 +1,32 @@
+"""
+ByteCrawl — Universal scraping layer
+===================================
+A single API to scrape any site: static HTML, dynamic JS, hidden APIs,
+session login, crawling at scale and Markdown conversion for LLMs.
+
+Quickstart:
+
+    from bytecrawl import Scraper
+
+    bot = Scraper()
+    page = bot.fetch("https://books.toscrape.com")
+    books = page.extract("article.product_pod", {
+        "title": "h3 a::attr(title)",
+        "price": "p.price_color::text",
+    })
+
+Explicit strategies:  bot.static(url) · bot.api(url) · bot.browser(url)
+Crawling:             bot.crawl(url, item=..., fields=..., next_page=...)
+Graph crawling:       SharkSearch(query=...).crawl(url) · BFS · OPIC
+LLM:                  page.markdown() · page.tokens()
+"""
+
+from .core import Page, Scraper, Session
+from .crawler import BFS, OPIC, Crawler, CrawlResult, SharkSearch, cosine, pagerank
+
+__all__ = [
+    "Scraper", "Page", "Session",
+    "Crawler", "BFS", "SharkSearch", "OPIC", "CrawlResult",
+    "pagerank", "cosine",
+]
+__version__ = "1.0.0"

bytecrawl/core.py

@@ -0,0 +1,386 @@
+"""
+ByteCrawl library core.
+
+Design: a Scraper object that knows how to fetch a page with the strategy you
+want (or the best one automatically) and gives you back a Page object from
+which you extract data with simple CSS selectors.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import requests
+from bs4 import BeautifulSoup
+
+DEFAULT_UA = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/120.0 Safari/537.36"
+)
+
+def _decoded_html(r: requests.Response) -> str:
+    """Response text with the correct encoding.
+
+    requests falls back to ISO-8859-1 when the header has no charset
+    (RFC 2616), which breaks UTF-8 (£ -> £). When that happens, use
+    content-based detection instead.
+    """
+    if r.encoding is None or r.encoding.lower() == "iso-8859-1":
+        r.encoding = r.apparent_encoding
+    return r.text
+
+
+def _split_selector(spec: str) -> tuple[str, str, str | None]:
+    """Returns (css_selector, operation, argument).
+
+    operation: "text" (default) or "attr". argument: the attribute name.
+    Examples:
+      "p.price_color::text"   -> ("p.price_color", "text", None)
+      "h3 a::attr(title)"     -> ("h3 a", "attr", "title")
+      "div.quote"             -> ("div.quote", "text", None)
+    """
+    spec = spec.strip()
+    if "::attr(" in spec:
+        sel, rest = spec.split("::attr(", 1)
+        return sel.strip(), "attr", rest.rstrip(")").strip()
+    if spec.endswith("::text"):
+        return spec[: -len("::text")].strip(), "text", None
+    return spec, "text", None
+
+
+def _value_from(node, op: str, arg: str | None) -> str | None:
+    if node is None:
+        return None
+    if op == "attr":
+        return node.get(arg)
+    return node.get_text(strip=True)
+
+
+@dataclass
+class Page:
+    """An already-downloaded page. Extract data from it."""
+
+    url: str
+    html: str = ""
+    data: Any = None          # JSON if it came from an API
+    method: str = "static"    # static | api | browser
+    elapsed: float = 0.0
+    status: int = 200
+    _soup: BeautifulSoup | None = field(default=None, repr=False)
+
+    @property
+    def soup(self) -> BeautifulSoup:
+        if self._soup is None:
+            self._soup = BeautifulSoup(self.html or "", "lxml")
+        return self._soup
+
+    # --- extraction ---------------------------------------------------------
+    def css(self, spec: str) -> str | None:
+        """First match of a selector. Supports ::text and ::attr(name)."""
+        sel, op, arg = _split_selector(spec)
+        return _value_from(self.soup.select_one(sel), op, arg)
+
+    def css_all(self, spec: str) -> list[str]:
+        """All matches of a selector."""
+        sel, op, arg = _split_selector(spec)
+        return [v for n in self.soup.select(sel) if (v := _value_from(n, op, arg)) is not None]
+
+    def extract(self, item: str, fields: dict[str, str]) -> list[dict]:
+        """Extracts a list of records.
+
+        item:   CSS selector delimiting each record (e.g. "article.product_pod").
+        fields: dict {name: relative_selector}. The selector is applied INSIDE
+                each item. If a field name ends in "[]" it returns a list of values.
+
+        Example:
+            page.extract("div.quote", {
+                "quote": "span.text::text",
+                "author": "small.author::text",
+                "tags[]": "a.tag::text",
+            })
+        """
+        records = []
+        for node in self.soup.select(item):
+            row = {}
+            for name, spec in fields.items():
+                sel, op, arg = _split_selector(spec)
+                if name.endswith("[]"):
+                    row[name[:-2]] = [
+                        v for n in node.select(sel) if (v := _value_from(n, op, arg)) is not None
+                    ]
+                else:
+                    row[name] = _value_from(node.select_one(sel), op, arg)
+            records.append(row)
+        return records
+
+    def links(self) -> list[str]:
+        return [a["href"] for a in self.soup.select("a[href]")]
+
+    def json(self) -> Any:
+        return self.data
+
+    # --- LLM ----------------------------------------------------------------
+    def markdown(self, main_only: bool = True) -> str:
+        """Converts the page to clean Markdown (saves tokens for LLMs)."""
+        if main_only:
+            try:
+                import trafilatura
+
+                # include_links keeps link boundaries; without it adjacent
+                # nodes glue together ("Visit siteSierra")
+                md = trafilatura.extract(self.html, output_format="markdown", include_links=True)
+                if md:
+                    return _clean_markdown(md)
+            except ImportError:
+                pass
+        try:
+            from markdownify import markdownify
+        except ImportError as e:
+            raise ImportError(
+                "Page.markdown() needs the 'llm' extra: pip install bytecrawl[llm]"
+            ) from e
+
+        return _clean_markdown(markdownify(self.html, strip=["script", "style"]))
+
+    def tokens(self, of: str | None = None) -> int:
+        """Quick token estimate (~4 chars/token)."""
+        text = of if of is not None else (self.html or "")
+        return len(text) // 4
+
+
+def _clean_markdown(md: str) -> str:
+    """Removes conversion noise from marketing/SPA pages.
+
+    Carousels and marquees duplicate their content in the DOM to loop, so the
+    same block appears 2-3 times; animated counters leave their initial state.
+    Dedupe repeated blocks and merge headings split across lines.
+    """
+    blocks = re.split(r"\n{2,}", md)
+    seen: set[str] = set()
+    cleaned: list[str] = []
+    prev_key = ""
+    for block in blocks:
+        # Headings broken inside the block: "## Start scraping\n today"
+        if block.lstrip().startswith("#") and "\n" in block:
+            lines = block.split("\n")
+            joined = [lines[0].rstrip()]
+            for ln in lines[1:]:
+                s = ln.strip()
+                continues = s[:1].islower() or joined[-1].rstrip().endswith(("'", ","))
+                if s and len(s) < 60 and continues:
+                    joined[-1] = joined[-1] + " " + s
+                else:
+                    joined.append(ln)
+            block = "\n".join(joined)
+        key = " ".join(block.split())
+        if not key:
+            continue
+        # Consecutive duplicates (looping carousels repeat short labels too)
+        if key == prev_key:
+            continue
+        # Carousel/marquee duplicates: same long block seen before
+        if len(key) >= 40 and key in seen:
+            continue
+        seen.add(key)
+        prev_key = key
+        cleaned.append(block.rstrip())
+
+    # Text split by animated spans: "# Power AI agents with" + "clean web data"
+    merged: list[str] = []
+    for block in cleaned:
+        prev = merged[-1] if merged else ""
+        frag = block.strip()
+        joinable = (
+            prev
+            and not prev.startswith(("```", "-", "*", ">", "|"))
+            and "\n" not in prev
+            and "\n" not in frag
+            and not frag.startswith(("#", "```", "-", "*", ">", "|", "["))
+        )
+        visible_len = len(re.sub(r"\]\([^)]*\)", "]", frag))  # ignore link URLs
+        lower_continuation = (
+            visible_len < 70
+            and frag[:1].islower()
+            and not prev.rstrip().endswith((".", "!", "?", ":", "`"))
+        )
+        # Headings cut mid-phrase keep capitalization: "## Easily connect with your" + "AI agents"
+        heading_continuation = (
+            re.match(r"#{1,6} ", prev) is not None
+            and len(frag) < 30
+            and not frag.rstrip().endswith((".", "!", "?", ":"))
+            and not prev.rstrip().endswith((".", "!", "?", ":", "`"))
+        )
+        if joinable and (lower_continuation or heading_continuation):
+            merged[-1] = prev.rstrip() + " " + frag
+        else:
+            merged.append(block)
+
+    # Inline-span joins lose the space: "at scale.It's also open source"
+    out = []
+    for block in merged:
+        if "```" not in block:
+            block = re.sub(r"(?<=[a-z])([.!?])(?=[A-Z\[])", r"\1 ", block)
+        out.append(block)
+    return "\n\n".join(out).strip()
+
+
+class Session:
+    """Reusable authenticated session (cookies + headers persist)."""
+
+    def __init__(self, scraper: Scraper):
+        self._s = requests.Session()
+        self._s.headers.update({"User-Agent": scraper.user_agent})
+        self._scraper = scraper
+
+    def login(self, url: str, data: dict, csrf_field: str | None = None) -> Session:
+        """Logs in. If csrf_field is given, reads it from the form first."""
+        if csrf_field:
+            r = self._s.get(url, timeout=self._scraper.timeout)
+            r.raise_for_status()
+            form = BeautifulSoup(_decoded_html(r), "lxml")
+            token = form.select_one(f'input[name="{csrf_field}"]')
+            if token:
+                data = {**data, csrf_field: token.get("value", "")}
+        r = self._s.post(url, data=data, timeout=self._scraper.timeout)
+        r.raise_for_status()
+        return self
+
+    def bearer(self, token: str) -> Session:
+        self._s.headers["Authorization"] = f"Bearer {token}"
+        return self
+
+    def fetch(self, url: str) -> Page:
+        t0 = time.perf_counter()
+        r = self._s.get(url, timeout=self._scraper.timeout)
+        r.raise_for_status()
+        return Page(url=url, html=_decoded_html(r), method="static",
+                    elapsed=round(time.perf_counter() - t0, 3), status=r.status_code)
+
+
+class Scraper:
+    """Entry point. Pick a strategy or let it decide on its own."""
+
+    def __init__(self, user_agent: str = DEFAULT_UA, delay: float = 0.0, timeout: int = 15):
+        self.user_agent = user_agent
+        self.delay = delay
+        self.timeout = timeout
+        self._session = requests.Session()
+        self._session.headers.update({"User-Agent": user_agent})
+
+    # --- strategies -----------------------------------------------------------
+    def static(self, url: str) -> Page:
+        """Technique 1: static HTML with requests."""
+        t0 = time.perf_counter()
+        r = self._session.get(url, timeout=self.timeout)
+        r.raise_for_status()
+        self._wait()
+        return Page(url=url, html=_decoded_html(r), method="static",
+                    elapsed=round(time.perf_counter() - t0, 3), status=r.status_code)
+
+    def api(self, url: str, params: dict | None = None) -> Page:
+        """Technique 3: requests an API and stores the JSON."""
+        t0 = time.perf_counter()
+        r = self._session.get(url, params=params, timeout=self.timeout)
+        r.raise_for_status()
+        self._wait()
+        try:
+            data = r.json()
+        except ValueError as e:
+            ctype = r.headers.get("content-type", "unknown")
+            raise ValueError(f"{url} did not return JSON (content-type: {ctype})") from e
+        return Page(url=url, data=data, method="api",
+                    elapsed=round(time.perf_counter() - t0, 3), status=r.status_code)
+
+    def browser(self, url: str, wait: str | None = None, scroll: bool = False) -> Page:
+        """Technique 2: real browser (Playwright) for JS-rendered sites."""
+        try:
+            from playwright.sync_api import sync_playwright
+        except ImportError as e:
+            raise ImportError(
+                "Scraper.browser() needs the 'browser' extra: "
+                "pip install bytecrawl[browser] && playwright install chromium"
+            ) from e
+
+        t0 = time.perf_counter()
+        with sync_playwright() as p:
+            nav = p.chromium.launch(headless=True)
+            pg = nav.new_page()
+            resp = pg.goto(url, wait_until="networkidle")
+            if wait:
+                pg.wait_for_selector(wait)
+            if scroll:
+                pg.mouse.wheel(0, 100000)
+                pg.wait_for_timeout(500)
+            html = pg.content()
+            status = resp.status if resp else 0
+            nav.close()
+        self._wait()
+        return Page(url=url, html=html, method="browser",
+                    elapsed=round(time.perf_counter() - t0, 3), status=status)
+
+    def fetch(self, url: str, strategy: str = "auto") -> Page:
+        """Fetches the page. strategy: auto | static | browser.
+
+        'auto' downloads statically and, if the page looks empty (typical of
+        SPAs that render with JS), retries with a browser.
+        """
+        if strategy == "static":
+            return self.static(url)
+        if strategy == "browser":
+            return self.browser(url)
+        # auto
+        page = self.static(url)
+        text = page.soup.get_text(strip=True)
+        if len(text) < 200:  # heuristic: almost no content -> probably JS
+            return self.browser(url)
+        return page
+
+    # --- crawling -----------------------------------------------------------
+    def crawl(
+        self,
+        start: str,
+        item: str,
+        fields: dict[str, str],
+        next_page: str | None = None,
+        pages: int | None = None,
+        base: str | None = None,
+    ) -> list[dict]:
+        """Walks multiple pages following the next-page link and extracts 'fields'.
+
+        start:     initial URL.
+        item:      selector for each record.
+        fields:    fields to extract (see Page.extract).
+        next_page: selector for the next-page link (e.g. "li.next a::attr(href)").
+        pages:     optional page limit.
+        base:      prefix for relative URLs (otherwise inferred from the host).
+        """
+        from urllib.parse import urljoin
+
+        url = start
+        results: list[dict] = []
+        n = 0
+        while url:
+            page = self.static(url)
+            results.extend(page.extract(item, fields))
+            n += 1
+            if pages and n >= pages:
+                break
+            if not next_page:
+                break
+            sel, op, arg = _split_selector(next_page)
+            node = page.soup.select_one(sel)
+            op = op if op != "text" else "attr"
+            href = _value_from(node, op, arg or "href") if node else None
+            url = urljoin(base or url, href) if href else None
+        return results
+
+    # --- session --------------------------------------------------------------
+    def session(self) -> Session:
+        return Session(self)
+
+    def _wait(self):
+        if self.delay:
+            time.sleep(self.delay)

bytecrawl/crawler.py

@@ -0,0 +1,355 @@
+"""
+Graph crawlers for ByteCrawl: BFS, Shark-Search and OPIC.
+
+The web is a graph (pages = nodes, links = edges). A crawler decides in
+what ORDER to visit URLs with a limited request budget. Each strategy is
+a different answer to that question:
+
+  BFS          — level by level: closest to the seed first.
+  Shark-Search — topical best-first: chases pages relevant to a query
+                 (Hersovici et al., 1998). Links inherit score from their
+                 parent with decay; bad branches die out on their own.
+  OPIC         — On-line Page Importance Computation (Abiteboul et al., 2003):
+                 each page holds "cash" that it distributes to its links when
+                 visited. It is PageRank computed live, without the full graph.
+
+Usage:
+
+    from bytecrawl.crawler import BFS, SharkSearch, OPIC, pagerank
+
+    result = SharkSearch(query="machine learning").crawl(
+        "https://example.com", max_pages=100)
+    result.pages          # visited, in order
+    result.graph          # {url: [links]} for offline analysis
+    pagerank(result.graph)
+"""
+
+from __future__ import annotations
+
+import heapq
+import math
+import re
+import time
+from collections import Counter
+from dataclasses import dataclass, field
+from urllib.parse import urldefrag, urljoin, urlparse
+
+from .core import Scraper
+
+# Non-HTML extensions: not worth spending a request on them.
+_SKIP_EXT = re.compile(
+    r"\.(png|jpe?g|gif|svg|webp|ico|css|js|pdf|zip|gz|tar|mp[34]|avi|mov|woff2?|ttf|xml|rss)$",
+    re.IGNORECASE,
+)
+
+_WORD = re.compile(r"[a-záéíóúüñ0-9]+", re.IGNORECASE)
+
+
+def _tokens(text: str) -> list[str]:
+    return [w.lower() for w in _WORD.findall(text or "")]
+
+
+def cosine(text_a: str, text_b: str) -> float:
+    """Cosine similarity between two texts using term-frequency (TF) vectors.
+
+    The classic information-retrieval metric: 1.0 = same vocabulary in the
+    same proportions, 0.0 = not a single word in common.
+    """
+    a, b = Counter(_tokens(text_a)), Counter(_tokens(text_b))
+    if not a or not b:
+        return 0.0
+    common = set(a) & set(b)
+    dot = sum(a[w] * b[w] for w in common)
+    norm = math.sqrt(sum(v * v for v in a.values())) * math.sqrt(sum(v * v for v in b.values()))
+    return dot / norm if norm else 0.0
+
+
+# Common second-level suffixes (bbc.co.uk → "bbc.co.uk", not "co.uk").
+# A frozen shortlist instead of the full public-suffix list keeps us dependency-free.
+_SECOND_LEVEL = frozenset({"co", "com", "org", "net", "ac", "gov", "edu"})
+
+
+def _root_domain(netloc: str) -> str:
+    """firecrawl.dev, www.firecrawl.dev and docs.firecrawl.dev are the same site."""
+    labels = netloc.lower().split(":")[0].split(".")
+    take = 3 if len(labels) >= 3 and labels[-2] in _SECOND_LEVEL else 2
+    return ".".join(labels[-take:])
+
+
+def normalize(url: str, base: str) -> str | None:
+    """Resolve relative URLs, strip #fragments and filter out non-web-page URLs."""
+    absolute, _ = urldefrag(urljoin(base, url))
+    parsed = urlparse(absolute)
+    if parsed.scheme not in ("http", "https"):
+        return None
+    if _SKIP_EXT.search(parsed.path):
+        return None
+    return absolute
+
+
+class Frontier:
+    """Priority queue of pending URLs with dedup.
+
+    heapq is a min-heap, so we store -score to always pop the URL with the
+    HIGHEST score. If a URL gets re-pushed with a new score (happens in OPIC,
+    where cash accumulates), we use lazy deletion: stale entries are discarded
+    on pop by comparing against the current score.
+    """
+
+    def __init__(self):
+        self._heap: list[tuple[float, int, str]] = []
+        self._score: dict[str, float] = {}
+        self._counter = 0  # FIFO tie-break between equal scores
+
+    def push(self, url: str, score: float):
+        current = self._score.get(url)
+        if current is not None and score <= current:
+            return
+        self._score[url] = score
+        self._counter += 1
+        heapq.heappush(self._heap, (-score, self._counter, url))
+
+    def pop(self) -> tuple[str, float] | None:
+        while self._heap:
+            neg, _, url = heapq.heappop(self._heap)
+            if url in self._score and -neg == self._score[url]:
+                del self._score[url]
+                return url, -neg
+        return None
+
+    def __contains__(self, url: str) -> bool:
+        return url in self._score
+
+    def __len__(self) -> int:
+        return len(self._score)
+
+
+@dataclass
+class CrawlResult:
+    """What a crawl returns: pages, graph and numbers for comparison."""
+
+    strategy: str
+    pages: list[dict] = field(default_factory=list)   # url, title, score, relevance, depth, order
+    graph: dict[str, list[str]] = field(default_factory=dict)
+    stats: dict = field(default_factory=dict)
+
+    def relevant(self, threshold: float = 0.1) -> list[dict]:
+        return [p for p in self.pages if p["relevance"] >= threshold]
+
+    def top(self, n: int = 10) -> list[dict]:
+        return sorted(self.pages, key=lambda p: p["relevance"], reverse=True)[:n]
+
+
+class Crawler:
+    """BFS crawler. Subclasses only change HOW links are scored.
+
+    The loop is identical for every strategy (pop → fetch → extract links →
+    score → push); that keeps the comparison between strategies fair:
+    same code, different ordering function.
+    """
+
+    name = "bfs"
+
+    def __init__(self, query: str = "", delay: float = 0.2, timeout: int = 10,
+                 same_domain: bool = True):
+        self.query = query
+        self.same_domain = same_domain
+        self.scraper = Scraper(delay=delay, timeout=timeout)
+
+    # --- extension point -------------------------------------------------------
+    def score_links(self, url: str, links: list[dict], relevance: float,
+                    depth: int) -> list[tuple[str, float]]:
+        """BFS: the score only encodes depth (shallower = visited sooner).
+
+        links: [{url, anchor}]. Returns [(url, score)] for the frontier.
+        """
+        return [(link["url"], -(depth + 1)) for link in links]
+
+    def on_visit(self, url: str, links: list[dict]):
+        """Hook for strategy-specific state (OPIC distributes cash here)."""
+
+    def initial_score(self, url: str) -> float:
+        return 0.0
+
+    # --- shared loop -------------------------------------------------------------
+    def crawl(self, start: str, max_pages: int = 50, max_depth: int = 10) -> CrawlResult:
+        start_norm = normalize(start, start)
+        if not start_norm:
+            raise ValueError(f"Invalid URL: {start}")
+        domain = _root_domain(urlparse(start_norm).netloc)
+
+        frontier = Frontier()
+        frontier.push(start_norm, self.initial_score(start_norm))
+        visited: set[str] = set()
+        depth_of = {start_norm: 0}
+        result = CrawlResult(strategy=self.name)
+        errors = 0
+        t0 = time.perf_counter()
+
+        while len(visited) < max_pages:
+            item = frontier.pop()
+            if item is None:
+                break
+            url, score = item
+            if url in visited:
+                continue
+            visited.add(url)
+            depth = depth_of.get(url, 0)
+
+            try:
+                page = self.scraper.static(url)
+            except Exception:
+                errors += 1
+                continue
+
+            text = page.soup.get_text(" ", strip=True)
+            relevance = cosine(text, self.query) if self.query else 0.0
+            title = page.css("title") or url
+
+            links = []
+            for a in page.soup.select("a[href]"):
+                child = normalize(a["href"], url)
+                if not child or child == url:
+                    continue
+                if self.same_domain and _root_domain(urlparse(child).netloc) != domain:
+                    continue
+                links.append({"url": child, "anchor": a.get_text(" ", strip=True)})
+
+            result.graph[url] = [link["url"] for link in links]
+            result.pages.append({
+                "url": url, "title": title[:120], "score": round(score, 4),
+                "relevance": round(relevance, 4), "depth": depth,
+                "order": len(result.pages) + 1,
+            })
+
+            self.on_visit(url, links)
+
+            if depth < max_depth:
+                for child_url, child_score in self.score_links(url, links, relevance, depth):
+                    if child_url not in visited:
+                        depth_of.setdefault(child_url, depth + 1)
+                        frontier.push(child_url, child_score)
+
+        result.stats = {
+            "requests": len(visited),
+            "errors": errors,
+            "elapsed": round(time.perf_counter() - t0, 2),
+            "frontier_left": len(frontier),
+            "relevant_found": len(result.relevant()) if self.query else None,
+            "avg_relevance": round(
+                sum(p["relevance"] for p in result.pages) / len(result.pages), 4
+            ) if result.pages else 0.0,
+        }
+        return result
+
+
+class BFS(Crawler):
+    """Explicit alias of the base behavior."""
+
+    name = "bfs"
+
+
+class SharkSearch(Crawler):
+    """Topical best-first with score inheritance (Hersovici et al., 1998).
+
+    score(link) = γ·inherited + (1−γ)·local_signal
+      inherited    = δ·relevance(parent) if the parent was relevant,
+                     otherwise δ·inherited(parent)  → bad branches decay as δ^n.
+      local_signal = cosine(anchor + URL words, query).
+    """
+
+    name = "shark"
+
+    def __init__(self, query: str, delta: float = 0.5, gamma: float = 0.8, **kw):
+        super().__init__(query=query, **kw)
+        self.delta = delta
+        self.gamma = gamma
+        self._inherited: dict[str, float] = {}
+
+    def initial_score(self, url: str) -> float:
+        self._inherited[url] = 1.0
+        return 1.0
+
+    def score_links(self, url, links, relevance, depth):
+        # Inheritance: if the parent was relevant, children inherit its
+        # relevance; otherwise they inherit what the parent had inherited.
+        # Either way with delta decay: a branch with no signal fades as delta^n.
+        parent_inherited = self._inherited.get(url, 0.0)
+        inherited = self.delta * (relevance if relevance > 0.05 else parent_inherited)
+        scored = []
+        for link in links:
+            url_words = " ".join(_tokens(urlparse(link["url"]).path))
+            local = cosine(f'{link["anchor"]} {url_words}', self.query)
+            score = self.gamma * inherited + (1 - self.gamma) * local
+            self._inherited[link["url"]] = inherited
+            scored.append((link["url"], score))
+        return scored
+
+
+class OPIC(Crawler):
+    """Online structural importance (Abiteboul et al., 2003).
+
+    Each page holds cash. On visit: its cash is added to its history
+    (accumulated importance), reset to 0 and split evenly among its outgoing
+    links. The URL with the most pending cash is always visited next.
+    Total system cash is conserved (the algorithm's invariant).
+    Pages with no outgoing links (sinks) return their cash to the known
+    unvisited URLs: the paper's "virtual node", immediate version.
+    """
+
+    name = "opic"
+
+    def __init__(self, query: str = "", **kw):
+        super().__init__(query=query, **kw)
+        self.cash: dict[str, float] = {}
+        self.history: dict[str, float] = {}
+        self._known_unvisited: set[str] = set()
+        self._frontier_ref: Frontier | None = None
+
+    def initial_score(self, url: str) -> float:
+        self.cash[url] = 1.0
+        return 1.0
+
+    def on_visit(self, url: str, links: list[dict]):
+        amount = self.cash.pop(url, 0.0)
+        self.history[url] = self.history.get(url, 0.0) + amount
+        self._known_unvisited.discard(url)
+        targets = [link["url"] for link in links] or list(self._known_unvisited)
+        if not targets:
+            return
+        share = amount / len(targets)
+        for t in targets:
+            self.cash[t] = self.cash.get(t, 0.0) + share
+            self._known_unvisited.add(t)
+
+    def score_links(self, url, links, relevance, depth):
+        # Cash was already distributed in on_visit; the score IS the accumulated cash.
+        return [(link["url"], self.cash.get(link["url"], 0.0)) for link in links]
+
+
+def pagerank(graph: dict[str, list[str]], damping: float = 0.85,
+             iterations: int = 30) -> dict[str, float]:
+    """PageRank via power iteration over the crawled graph.
+
+    Offline version of the same concept OPIC approximates online: comparing
+    both rankings over the same graph is the interesting experiment.
+    """
+    nodes = set(graph) | {v for vs in graph.values() for v in vs}
+    if not nodes:
+        return {}
+    n = len(nodes)
+    rank = {u: 1.0 / n for u in nodes}
+    for _ in range(iterations):
+        new = {u: (1 - damping) / n for u in nodes}
+        for u in nodes:
+            out = [v for v in graph.get(u, []) if v in nodes]
+            if out:
+                share = damping * rank[u] / len(out)
+                for v in out:
+                    new[v] += share
+            else:  # sink: distribute to all (virtual node)
+                share = damping * rank[u] / n
+                for v in nodes:
+                    new[v] += share
+        rank = new
+    return dict(sorted(rank.items(), key=lambda kv: kv[1], reverse=True))

bytecrawl/mcp_http.py

@@ -0,0 +1,194 @@
+"""Hosted (HTTP) MCP server: ByteCrawl tools behind a public URL.
+
+Same four tools as the local stdio server (bytecrawl.mcp_server), hardened
+for the open internet:
+
+  1. SSRF guard    — every URL is resolved and rejected if any of its IPs is
+                     private/loopback/link-local (a hosted scraper must not be
+                     an open proxy into the host's network).
+  2. Hard caps     — focused_crawl is clamped to MAX_PAGES pages; the shared
+                     scraper keeps a polite delay. The cost of one request is
+                     bounded no matter what the client asks for.
+  3. Rate limit    — sliding-window per client IP, in memory. Best-effort on
+                     serverless (each warm instance counts separately); good
+                     enough to keep one abuser from exhausting the free tier.
+
+Serverless notes: runs stateless (no session affinity) and static-only —
+Playwright is not available on Vercel, so JS-heavy sites should use the local
+`bytecrawl-mcp` instead. Redirect chains are not re-validated hop by hop; the
+guard checks the URL each tool receives (and the crawler stays on the seed's
+registered domain).
+
+Self-host:  bytecrawl-mcp-http            (serves http://0.0.0.0:8000/mcp)
+Connect:    claude mcp add --transport http bytecrawl https://bytecrawl.vercel.app/mcp
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import socket
+import time
+from collections import defaultdict, deque
+from urllib.parse import urlparse
+
+from mcp.server.mcpserver import MCPServer
+from mcp.server.mcpserver.exceptions import ToolError
+
+from . import mcp_server as local
+
+MAX_PAGES = 10          # hard cap per focused_crawl call (local allows 50)
+RATE_LIMIT = 20         # requests per window per client IP
+RATE_WINDOW = 60.0      # seconds
+
+
+# --- 1. SSRF guard ----------------------------------------------------------
+def assert_public_url(url: str) -> None:
+    """Rejects URLs that could reach private infrastructure.
+
+    Resolves the hostname and requires every returned address to be globally
+    routable. Blocks localhost, RFC1918 ranges, link-local (cloud metadata
+    endpoints like 169.254.169.254) and other reserved space.
+    """
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise ValueError(f"Only http(s) URLs are allowed, got: {parsed.scheme or 'none'}")
+    host = parsed.hostname
+    if not host:
+        raise ValueError("URL has no hostname")
+    try:
+        infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
+    except socket.gaierror as e:
+        raise ValueError(f"Cannot resolve host: {host}") from e
+    for info in infos:
+        ip = ipaddress.ip_address(info[4][0])
+        if not ip.is_global or ip.is_multicast:
+            raise ValueError(
+                f"URL resolves to a non-public address ({ip}) — refusing to fetch it"
+            )
+
+
+def _guarded(url: str) -> None:
+    """SSRF check, re-raised as ToolError so MCP clients see the reason
+    (plain exceptions are masked as a generic 'error executing tool')."""
+    try:
+        assert_public_url(url)
+    except ValueError as e:
+        raise ToolError(str(e)) from e
+
+
+# --- 3. rate limit ----------------------------------------------------------
+class RateLimiter:
+    """Sliding-window counter per key. In-memory: per-instance on serverless."""
+
+    def __init__(self, limit: int = RATE_LIMIT, window: float = RATE_WINDOW):
+        self.limit = limit
+        self.window = window
+        self._hits: dict[str, deque] = defaultdict(deque)
+
+    def allow(self, key: str) -> bool:
+        now = time.monotonic()
+        hits = self._hits[key]
+        while hits and now - hits[0] > self.window:
+            hits.popleft()
+        if len(hits) >= self.limit:
+            return False
+        hits.append(now)
+        return True
+
+
+class RateLimitMiddleware:
+    """ASGI wrapper: answers 429 before the MCP app sees the request."""
+
+    def __init__(self, app, limiter: RateLimiter | None = None):
+        self.app = app
+        self.limiter = limiter or RateLimiter()
+
+    @staticmethod
+    def _client_ip(scope) -> str:
+        for name, value in scope.get("headers", []):
+            if name == b"x-forwarded-for":
+                return value.decode().split(",")[0].strip()
+        client = scope.get("client")
+        return client[0] if client else "unknown"
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] == "http" and not self.limiter.allow(self._client_ip(scope)):
+            await send({
+                "type": "http.response.start",
+                "status": 429,
+                "headers": [(b"content-type", b"application/json"),
+                            (b"retry-after", b"60")],
+            })
+            await send({
+                "type": "http.response.body",
+                "body": b'{"error": "rate limit exceeded, try again in a minute"}',
+            })
+            return
+        await self.app(scope, receive, send)
+
+
+# --- the hardened server ----------------------------------------------------
+server = MCPServer(
+    "bytecrawl",
+    instructions=(
+        "Hosted ByteCrawl: web scraping and focused crawling. Static HTML only "
+        "(no JS rendering here — run bytecrawl-mcp locally for that). "
+        f"Crawls are capped at {MAX_PAGES} pages per call."
+    ),
+)
+
+
+@server.tool(description=(
+    "Fetch a public web page and return it as clean Markdown for LLM "
+    "consumption. Static HTML only on the hosted server."
+))
+def fetch_markdown(url: str) -> dict:
+    _guarded(url)
+    return local.fetch_markdown(url)
+
+
+@server.tool(description=(
+    "Extract structured records from a public page with CSS selectors "
+    "(::text / ::attr(name); a field name ending in [] collects a list)."
+))
+def extract(url: str, item: str, fields: dict[str, str]) -> dict:
+    _guarded(url)
+    return local.extract(url, item, fields)
+
+
+@server.tool(description=(
+    "Focused crawl of a public site: visit the pages most relevant to 'query' "
+    "first. strategy: shark (default) | opic | bfs. Capped at "
+    f"{MAX_PAGES} pages per call on the hosted server."
+))
+def focused_crawl(url: str, query: str = "", strategy: str = "shark",
+                  max_pages: int = MAX_PAGES) -> dict:
+    _guarded(url)
+    return local.focused_crawl(url, query=query, strategy=strategy,
+                               max_pages=min(max_pages, MAX_PAGES))
+
+
+@server.tool(description="Fetch a public JSON API endpoint.")
+def fetch_json_api(url: str, params: dict | None = None) -> dict:
+    _guarded(url)
+    return local.fetch_json_api(url, params=params)
+
+
+def create_app(limiter: RateLimiter | None = None):
+    """ASGI app: streamable-HTTP MCP at /mcp, rate-limited. Stateless for
+    serverless (json_response avoids long-lived SSE streams on Vercel)."""
+    app = server.streamable_http_app(
+        streamable_http_path="/mcp", stateless_http=True, json_response=True,
+    )
+    return RateLimitMiddleware(app, limiter)
+
+
+def main() -> None:
+    """Entry point for bytecrawl-mcp-http (self-hosted)."""
+    import uvicorn
+
+    uvicorn.run(create_app(), host="0.0.0.0", port=8000)
+
+
+if __name__ == "__main__":
+    main()

bytecrawl/mcp_server.py

@@ -0,0 +1,112 @@
+"""MCP server: exposes ByteCrawl to AI agents (Claude Code, Claude Desktop, Cursor...).
+
+Any MCP-capable agent gets four tools that map to what ByteCrawl does well:
+clean Markdown for LLM ingestion, structured extraction with CSS selectors,
+focused crawling (the differentiator) and hidden JSON APIs.
+
+Run it:            bytecrawl-mcp                 (stdio transport)
+Claude Code:       claude mcp add bytecrawl -- bytecrawl-mcp
+Requires:          pip install bytecrawl[mcp]
+"""
+
+from __future__ import annotations
+
+try:
+    from mcp.server.mcpserver import MCPServer
+except ImportError as e:  # pragma: no cover - exercised only without the extra
+    raise ImportError(
+        "The MCP server needs the 'mcp' extra: pip install bytecrawl[mcp]"
+    ) from e
+
+from .core import Scraper
+from .crawler import BFS, OPIC, SharkSearch, pagerank
+
+# One polite scraper shared by every tool call.
+_scraper = Scraper(delay=0.2)
+
+_STRATEGIES = {"bfs": BFS, "shark": SharkSearch, "opic": OPIC}
+
+server = MCPServer(
+    "bytecrawl",
+    instructions=(
+        "Web scraping and focused crawling. Use fetch_markdown to read a page, "
+        "extract for structured records, focused_crawl to find the pages most "
+        "relevant to a topic within a site, and fetch_json_api for JSON endpoints."
+    ),
+)
+
+
+@server.tool(
+    description=(
+        "Fetch a web page and return it as clean Markdown, ready for LLM "
+        "consumption (5-10x fewer tokens than raw HTML)."
+    )
+)
+def fetch_markdown(url: str) -> dict:
+    page = _scraper.fetch(url)
+    md = page.markdown()
+    return {
+        "url": url,
+        "markdown": md,
+        "tokens_estimate": page.tokens(md),
+        "method": page.method,
+        "status": page.status,
+    }
+
+
+@server.tool(
+    description=(
+        "Extract structured records from a page with CSS selectors. "
+        "'item' delimits each record (e.g. 'article.product'); 'fields' maps "
+        "names to relative selectors supporting ::text and ::attr(name), e.g. "
+        '{"title": "h3 a::attr(title)", "price": "p.price::text"}. '
+        "A field name ending in [] collects a list."
+    )
+)
+def extract(url: str, item: str, fields: dict[str, str]) -> dict:
+    page = _scraper.fetch(url)
+    records = page.extract(item, fields)
+    return {"url": url, "count": len(records), "records": records}
+
+
+@server.tool(
+    description=(
+        "Crawl a site starting from 'url', visiting the pages most relevant to "
+        "'query' first (focused crawling). strategy: 'shark' (topical "
+        "best-first, default), 'opic' (structural importance) or 'bfs' "
+        "(level by level). Returns pages ranked by relevance plus crawl stats."
+    )
+)
+def focused_crawl(url: str, query: str = "", strategy: str = "shark",
+                  max_pages: int = 20) -> dict:
+    if strategy not in _STRATEGIES:
+        raise ValueError(f"strategy must be one of {sorted(_STRATEGIES)}")
+    max_pages = min(max_pages, 50)  # keep agent calls bounded and polite
+    crawler = _STRATEGIES[strategy](query=query, delay=0.2)
+    result = crawler.crawl(url, max_pages=max_pages)
+    ranks = pagerank(result.graph)
+    return {
+        "strategy": strategy,
+        "stats": result.stats,
+        "pages": result.top(max_pages) if query else result.pages,
+        "pagerank_top": [
+            {"url": u, "rank": round(r, 4)} for u, r in list(ranks.items())[:10]
+        ],
+    }
+
+
+@server.tool(
+    description="Fetch a JSON API endpoint (the 'hidden API' scraping technique)."
+)
+def fetch_json_api(url: str, params: dict | None = None) -> dict:
+    page = _scraper.api(url, params=params)
+    return {"url": url, "status": page.status, "data": page.json()}
+
+
+def main() -> None:
+    """Entry point for the bytecrawl-mcp console script (stdio transport)."""
+    server.run()
+
+
+if __name__ == "__main__":
+    main()

bytecrawl-1.0.0.dist-info/METADATA

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: bytecrawl
+Version: 1.0.0
+Summary: Focused crawling for LLM data collection: Shark-Search and OPIC crawlers plus a single scraping API (static HTML, dynamic JS, APIs, session login, Markdown for LLMs) on a 3-dependency core.
+Author-email: abrahamperz <aperez@adokmx.com>
+License: MIT
+Project-URL: Homepage, https://github.com/abrahamperz/ByteCrawl
+Project-URL: Repository, https://github.com/abrahamperz/ByteCrawl
+Keywords: scraping,crawler,focused-crawling,shark-search,opic,pagerank,beautifulsoup,playwright,markdown,llm
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Intended Audience :: Developers
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28
+Requires-Dist: beautifulsoup4>=4.11
+Requires-Dist: lxml>=4.9
+Provides-Extra: browser
+Requires-Dist: playwright>=1.40; extra == "browser"
+Provides-Extra: llm
+Requires-Dist: markdownify>=0.11; extra == "llm"
+Requires-Dist: trafilatura>=1.6; extra == "llm"
+Provides-Extra: mcp
+Requires-Dist: mcp>=2; python_version >= "3.10" and extra == "mcp"
+Requires-Dist: markdownify>=0.11; extra == "mcp"
+Requires-Dist: trafilatura>=1.6; extra == "mcp"
+Provides-Extra: all
+Requires-Dist: playwright>=1.40; extra == "all"
+Requires-Dist: markdownify>=0.11; extra == "all"
+Requires-Dist: trafilatura>=1.6; extra == "all"
+Requires-Dist: mcp>=2; python_version >= "3.10" and extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Dynamic: license-file
+
+# ByteCrawl
+
+[![CI](https://github.com/abrahamperz/ByteCrawl/actions/workflows/ci.yml/badge.svg)](https://github.com/abrahamperz/ByteCrawl/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/bytecrawl)](https://pypi.org/project/bytecrawl/)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://pypi.org/project/bytecrawl/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+**Give your AI agent focused web crawling.** ByteCrawl is an MCP server (and a
+small Python library) that doesn't just scrape a page — it crawls a whole site
+and returns the pages *most relevant* to your topic first, using Shark-Search
+and OPIC in pure Python.
+
+- **Webpage**: https://bytecrawl.vercel.app/
+- **Hosted MCP endpoint**: https://bytecrawl.vercel.app/mcp
+
+## Quick start (MCP — nothing to install)
+
+Point any MCP-capable agent (Claude Code, Claude Desktop, Cursor...) at the
+hosted endpoint:
+
+```bash
+claude mcp add --transport http bytecrawl https://bytecrawl.vercel.app/mcp
+```
+
+Now the agent has four tools:
+
+| Tool | What it does |
+|---|---|
+| `focused_crawl` | Crawl a site, rank pages by relevance to a query (Shark-Search / OPIC / BFS) |
+| `fetch_markdown` | One page → clean Markdown (5–10× fewer tokens than raw HTML) |
+| `extract` | Structured records via CSS selectors |
+| `fetch_json_api` | Hit a hidden JSON API |
+
+The hosted server is static-only, rate-limited per IP, caps crawls at 10 pages,
+and refuses non-public URLs (SSRF guard). For heavy use or JS-rendered sites,
+run it locally:
+
+```bash
+pip install bytecrawl[mcp]
+claude mcp add bytecrawl -- bytecrawl-mcp                 # full power, on your machine
+pip install bytecrawl[browser] && playwright install chromium   # + JS rendering
+```
+
+## Why focused crawling?
+
+Most crawlers visit pages in whatever order they find them. With a limited
+request budget, order is everything — Shark-Search chases the branches that
+smell like your query and lets the rest decay, so 100 requests get you the 100
+*most useful* pages, not the 100 closest to the seed.
+
+```python
+from bytecrawl import SharkSearch
+
+result = SharkSearch(query="vector databases").crawl(
+    "https://example.com", max_pages=100)
+
+for page in result.top(10):
+    print(f'{page["relevance"]:.3f}  {page["url"]}')
+```
+
+- **BFS** — level by level, closest to the seed first.
+- **Shark-Search** (Hersovici et al., 1998) — topical best-first; links inherit
+  their parent's relevance with decay.
+- **OPIC** (Abiteboul et al., 2003) — live PageRank via "cash" flow, no full
+  graph needed (a `pagerank()` implementation is included to compare against).
+
+Versus the alternatives: Scrapy is a framework you wire up yourself, Firecrawl
+is a paid SaaS — ByteCrawl is a plain library with a 3-package core and these
+frontier strategies built in.
+
+## Library API
+
+```python
+from bytecrawl import Scraper
+
+bot = Scraper()
+page = bot.fetch("https://books.toscrape.com")   # auto: static, browser fallback
+books = page.extract("article.product_pod",
+                     {"title": "h3 a::attr(title)", "price": "p.price_color::text"})
+page.markdown()   # clean Markdown for LLMs   ·   page.tokens()   # token estimate
+```
+
+```python
+bot.static(url)                                   # plain HTML
+bot.api(url, params={...})                        # hidden JSON API
+bot.browser(url, wait="div.results")              # JS via Playwright
+bot.crawl(url, item="article", fields={...},
+          next_page="li.next a::attr(href)")      # pagination
+bot.session().login(url, data, csrf_field="csrf_token")   # authenticated
+```
+
+## Install
+
+```bash
+pip install bytecrawl            # slim core (requests + beautifulsoup4 + lxml)
+pip install bytecrawl[llm]       # + Markdown for LLMs
+pip install bytecrawl[browser]   # + Playwright
+pip install bytecrawl[mcp]       # + local MCP server
+pip install bytecrawl[all]
+```
+
+## Learn each scraping technique
+
+A guided walkthrough with a runnable example against a practice site:
+[static HTML](docs/01-html-estatico.md) ·
+[dynamic JS](docs/02-js-dinamico.md) ·
+[hidden APIs](docs/03-api-oculta.md) ·
+[pagination](docs/04-crawling-paginacion.md) ·
+[login](docs/05-login-sesion.md) ·
+[graph crawling](docs/06-crawling-grafos.md) ·
+[Markdown for LLMs](docs/markdown-llms.md) ·
+[ethics](docs/nota-etica.md)
+
+## Contributing
+
+```bash
+pip install -e ".[llm,dev,mcp]"
+pytest              # 109 tests, no network required
+pytest -m live      # + live browser tests (needs the browser extra)
+ruff check bytecrawl tests
+```
+
+Scrape responsibly: respect `robots.txt`, terms of service and rate limits.
+ByteCrawl ships with a configurable delay between requests.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

bytecrawl-1.0.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+bytecrawl/__init__.py,sha256=8AGDzOjuwX8H0WbjmfVe-thBZzkq2VTEqsgLxjcnpAk,1034
+bytecrawl/core.py,sha256=c72k5DRyBfYgpwDXzo3fTVvdrQs1GXuw3RYsKIsMVdU,14480
+bytecrawl/crawler.py,sha256=Z0XQ1SRbN5ilZjTmo904lhHDQyAyem5Mo39k1GysGFw,13263
+bytecrawl/mcp_http.py,sha256=jA9ieAlIWaA0chzYzutHk0IATJr6Q3I36htSVC8RCPI,7187
+bytecrawl/mcp_server.py,sha256=aSWrYFLFxr_lGeqWpGW6stHnecpEhCL45cSCSwI7BwQ,3831
+bytecrawl-1.0.0.dist-info/licenses/LICENSE,sha256=iBV--gE_YY2iDtqnc4mtYyq4BhFdoKIr0VpptvV3foY,1071
+bytecrawl-1.0.0.dist-info/METADATA,sha256=Ibw2y_nCwJWlj_uhqsMXB7WlrkrQMBWNDBQ3I9wsbQE,6468
+bytecrawl-1.0.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+bytecrawl-1.0.0.dist-info/entry_points.txt,sha256=mqISq8YZx19UzEdoyF_bUGm1t8AQ3dhjF2o0tmLbf8g,105
+bytecrawl-1.0.0.dist-info/top_level.txt,sha256=XYFyykLgFrhCeCA3ZFwZm_4wJxZzmAw7T3kibTQKJHo,10
+bytecrawl-1.0.0.dist-info/RECORD,,

bytecrawl-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

bytecrawl-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+bytecrawl-mcp = bytecrawl.mcp_server:main
+bytecrawl-mcp-http = bytecrawl.mcp_http:main

bytecrawl-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Abraham Pérez
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bytecrawl-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+bytecrawl