curl-reap 0.1.0__tar.gz

curl_reap-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.pytest_cache/
+.reap_selectors.json
+*.jsonl
+.venv/
+venv/
+.DS_Store

curl_reap-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Anish
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

curl_reap-0.1.0/PKG-INFO

@@ -0,0 +1,143 @@
+Metadata-Version: 2.4
+Name: curl_reap
+Version: 0.1.0
+Summary: Reap the web: browser-grade TLS impersonation, self-healing selectors, and a concurrent crawl engine in one library.
+Project-URL: Homepage, https://github.com/anishfyi/curl_reap
+Project-URL: Repository, https://github.com/anishfyi/curl_reap
+Project-URL: Issues, https://github.com/anishfyi/curl_reap/issues
+Author-email: Anish <anishfyi@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: crawler,curl_cffi,impersonate,lxml,scraping,selectors,spider,tls-fingerprint
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: cssselect>=1.2
+Requires-Dist: curl-cffi>=0.7.0
+Requires-Dist: lxml>=5.0
+Provides-Extra: dev
+Requires-Dist: build>=1.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: twine>=5.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="assets/logo.svg" alt="curl_reap" width="420" />
+</p>
+
+<p align="center"><b>Reap the web.</b> Browser-grade TLS impersonation, self-healing selectors, and a concurrent crawl engine, in one small library.</p>
+
+<p align="center">
+  <code>pip install curl_reap</code>
+</p>
+
+---
+
+## Why
+
+Modern scraping needs three things, and today you reach for three different tools:
+
+1. **Get past the door.** Sites fingerprint your TLS handshake and block stock Python clients. `curl_cffi` solves this with real Chrome/Safari fingerprints.
+2. **Survive markup changes.** Plain CSS and XPath break the moment a site renames a class. Scrapling pioneered self-healing selectors that re-find the element anyway.
+3. **Crawl at scale.** Concurrency, throttling, retries, dedup, and pipelines. That is Scrapy.
+
+`curl_reap` takes the best idea from each and puts them behind one friendly API.
+
+| | curl_cffi | Scrapy | Scrapling | **curl_reap** |
+|---|:---:|:---:|:---:|:---:|
+| Real browser TLS / JA3 | yes | no | partial | **yes** |
+| Parser built in | no | yes | yes | **yes** |
+| Self-healing selectors | no | no | yes | **yes** |
+| Concurrent crawl engine | no | yes | no | **yes** |
+| AutoThrottle, retries, pipelines | no | yes | no | **yes** |
+| One small dependency set | yes | no | no | **yes** |
+
+## Install
+
+```bash
+pip install curl_reap
+```
+
+Requires Python 3.9+. Pulls in `curl_cffi`, `lxml`, and `cssselect`.
+
+## Quick start
+
+A one-shot fetch parses like parsel, but the request carries a genuine browser fingerprint:
+
+```python
+import curl_reap as reap
+
+page = reap.get("https://quotes.toscrape.com", impersonate="chrome124")
+print(page.css("span.text::text").getall())
+print(page.css_first("small.author::text"))
+```
+
+## Self-healing selectors
+
+Save an element once. Later, even if the site renames the class or moves the node, `auto_match` relocates it by structural signature:
+
+```python
+page = reap.get("https://shop.example.com/item/42")
+page.css_first("a.buy-btn").save("buy_button")     # remember its shape
+
+# weeks later, the class is now "purchase-cta" and the old selector misses:
+later = reap.get("https://shop.example.com/item/99")
+btn = later.css_first("a.buy-btn", auto_match=True, identifier="buy_button")
+print(btn.attr("href"))                            # found anyway
+```
+
+Other finders: `page.find_by_text("Sign in")` and `page.find_similar(some_element)`.
+
+## Crawl at scale
+
+A `Spider` yields items (dicts) and more `Request` objects. The engine handles concurrency, AutoThrottle, retries, dedup, and pipelines:
+
+```python
+import curl_reap as reap
+from curl_reap import JsonLinesPipeline
+
+class Quotes(reap.Spider):
+    start_urls = ["https://quotes.toscrape.com"]
+
+    def parse(self, page):
+        for q in page.css("div.quote"):
+            yield {
+                "text": q.css_first("span.text::text"),
+                "author": q.css_first("small.author::text"),
+            }
+        nxt = page.css_first("li.next a::attr(href)")
+        if nxt:
+            yield reap.Request("https://quotes.toscrape.com" + nxt, self.parse)
+
+items = reap.run(
+    Quotes,
+    concurrency=8,
+    throttle=True,                       # AutoThrottle adapts to server latency
+    pipelines=[JsonLinesPipeline("quotes.jsonl")],
+)
+print(len(items), "items reaped")
+```
+
+## API at a glance
+
+- `reap.get(url, impersonate="chrome124", **kw)` and `reap.post(...)` return a `Response` you can `.css()` / `.xpath()` directly.
+- `reap.Session(impersonate=..., headers=..., retries=...)` for a reusable client.
+- `Selector` / `SelectorList`: `.css`, `.css_first`, `.xpath`, `.find_by_text`, `.find_similar`, `.save`, `.re`, `.text`, `.attr`.
+- `reap.Spider`, `reap.Request`, `reap.run(spider, ...)`, `reap.Reaper(...)`.
+- Pipelines: `DedupPipeline`, `JsonLinesPipeline`, `CsvPipeline`, or subclass `Pipeline`.
+
+## Responsible use
+
+`curl_reap` impersonates a real browser at the TLS level, which is exactly what a normal browser does. It does **not** ship a challenge solver and it will not break CAPTCHAs or anti-bot walls (Cloudflare challenges, DataDome, PerimeterX, and similar). If a site has deliberately put up an access-control wall, that is a signal to stop. Respect robots.txt and each site's terms, throttle your crawls, and only collect data you are allowed to collect.
+
+## License
+
+MIT. See [LICENSE](LICENSE).

curl_reap-0.1.0/README.md

@@ -0,0 +1,112 @@
+<p align="center">
+  <img src="assets/logo.svg" alt="curl_reap" width="420" />
+</p>
+
+<p align="center"><b>Reap the web.</b> Browser-grade TLS impersonation, self-healing selectors, and a concurrent crawl engine, in one small library.</p>
+
+<p align="center">
+  <code>pip install curl_reap</code>
+</p>
+
+---
+
+## Why
+
+Modern scraping needs three things, and today you reach for three different tools:
+
+1. **Get past the door.** Sites fingerprint your TLS handshake and block stock Python clients. `curl_cffi` solves this with real Chrome/Safari fingerprints.
+2. **Survive markup changes.** Plain CSS and XPath break the moment a site renames a class. Scrapling pioneered self-healing selectors that re-find the element anyway.
+3. **Crawl at scale.** Concurrency, throttling, retries, dedup, and pipelines. That is Scrapy.
+
+`curl_reap` takes the best idea from each and puts them behind one friendly API.
+
+| | curl_cffi | Scrapy | Scrapling | **curl_reap** |
+|---|:---:|:---:|:---:|:---:|
+| Real browser TLS / JA3 | yes | no | partial | **yes** |
+| Parser built in | no | yes | yes | **yes** |
+| Self-healing selectors | no | no | yes | **yes** |
+| Concurrent crawl engine | no | yes | no | **yes** |
+| AutoThrottle, retries, pipelines | no | yes | no | **yes** |
+| One small dependency set | yes | no | no | **yes** |
+
+## Install
+
+```bash
+pip install curl_reap
+```
+
+Requires Python 3.9+. Pulls in `curl_cffi`, `lxml`, and `cssselect`.
+
+## Quick start
+
+A one-shot fetch parses like parsel, but the request carries a genuine browser fingerprint:
+
+```python
+import curl_reap as reap
+
+page = reap.get("https://quotes.toscrape.com", impersonate="chrome124")
+print(page.css("span.text::text").getall())
+print(page.css_first("small.author::text"))
+```
+
+## Self-healing selectors
+
+Save an element once. Later, even if the site renames the class or moves the node, `auto_match` relocates it by structural signature:
+
+```python
+page = reap.get("https://shop.example.com/item/42")
+page.css_first("a.buy-btn").save("buy_button")     # remember its shape
+
+# weeks later, the class is now "purchase-cta" and the old selector misses:
+later = reap.get("https://shop.example.com/item/99")
+btn = later.css_first("a.buy-btn", auto_match=True, identifier="buy_button")
+print(btn.attr("href"))                            # found anyway
+```
+
+Other finders: `page.find_by_text("Sign in")` and `page.find_similar(some_element)`.
+
+## Crawl at scale
+
+A `Spider` yields items (dicts) and more `Request` objects. The engine handles concurrency, AutoThrottle, retries, dedup, and pipelines:
+
+```python
+import curl_reap as reap
+from curl_reap import JsonLinesPipeline
+
+class Quotes(reap.Spider):
+    start_urls = ["https://quotes.toscrape.com"]
+
+    def parse(self, page):
+        for q in page.css("div.quote"):
+            yield {
+                "text": q.css_first("span.text::text"),
+                "author": q.css_first("small.author::text"),
+            }
+        nxt = page.css_first("li.next a::attr(href)")
+        if nxt:
+            yield reap.Request("https://quotes.toscrape.com" + nxt, self.parse)
+
+items = reap.run(
+    Quotes,
+    concurrency=8,
+    throttle=True,                       # AutoThrottle adapts to server latency
+    pipelines=[JsonLinesPipeline("quotes.jsonl")],
+)
+print(len(items), "items reaped")
+```
+
+## API at a glance
+
+- `reap.get(url, impersonate="chrome124", **kw)` and `reap.post(...)` return a `Response` you can `.css()` / `.xpath()` directly.
+- `reap.Session(impersonate=..., headers=..., retries=...)` for a reusable client.
+- `Selector` / `SelectorList`: `.css`, `.css_first`, `.xpath`, `.find_by_text`, `.find_similar`, `.save`, `.re`, `.text`, `.attr`.
+- `reap.Spider`, `reap.Request`, `reap.run(spider, ...)`, `reap.Reaper(...)`.
+- Pipelines: `DedupPipeline`, `JsonLinesPipeline`, `CsvPipeline`, or subclass `Pipeline`.
+
+## Responsible use
+
+`curl_reap` impersonates a real browser at the TLS level, which is exactly what a normal browser does. It does **not** ship a challenge solver and it will not break CAPTCHAs or anti-bot walls (Cloudflare challenges, DataDome, PerimeterX, and similar). If a site has deliberately put up an access-control wall, that is a signal to stop. Respect robots.txt and each site's terms, throttle your crawls, and only collect data you are allowed to collect.
+
+## License
+
+MIT. See [LICENSE](LICENSE).

curl_reap-0.1.0/assets/logo.svg

@@ -0,0 +1,28 @@
+<svg width="520" height="170" viewBox="0 0 520 170" fill="none" xmlns="http://www.w3.org/2000/svg" font-family="ui-monospace, SFMono-Regular, Menlo, monospace">
+  <defs>
+    <linearGradient id="blade" x1="20" y1="20" x2="120" y2="130" gradientUnits="userSpaceOnUse">
+      <stop offset="0" stop-color="#F0C24B"/>
+      <stop offset="1" stop-color="#C9871A"/>
+    </linearGradient>
+  </defs>
+
+  <!-- the reap mark: a sickle blade that curves like a curl -->
+  <g transform="translate(26,20)">
+    <!-- blade -->
+    <path d="M104 40 C 74 14, 26 30, 22 72 C 19 104, 44 124, 80 118"
+          stroke="url(#blade)" stroke-width="16" stroke-linecap="round" fill="none"/>
+    <!-- handle -->
+    <path d="M80 118 C 92 116, 104 122, 112 134"
+          stroke="#7A4E27" stroke-width="13" stroke-linecap="round" fill="none"/>
+    <!-- grain being reaped, inside the curl -->
+    <circle cx="112" cy="50" r="5.5" fill="#E7B23E"/>
+    <circle cx="120" cy="68" r="5" fill="#E7B23E"/>
+    <circle cx="105" cy="74" r="4.5" fill="#E7B23E"/>
+  </g>
+
+  <!-- wordmark -->
+  <text x="186" y="92" font-size="50" font-weight="700" letter-spacing="-1">
+    <tspan fill="#8A8178">curl_</tspan><tspan fill="#C9871A">reap</tspan>
+  </text>
+  <text x="188" y="124" font-size="18" font-weight="500" letter-spacing="3" fill="#9a9189">REAP THE WEB</text>
+</svg>

curl_reap-0.1.0/curl_reap/__init__.py

@@ -0,0 +1,47 @@
+"""curl_reap: reap the web.
+
+Three pillars in one library:
+  1. Transport: real browser TLS/JA3 impersonation (powered by curl_cffi) so your
+     requests are not fingerprinted as a bot.
+  2. Parsing: a fast lxml selector with parsel-style css/xpath plus self-healing
+     selectors that survive markup changes.
+  3. Orchestration: a small concurrent crawl engine with dedup, retries,
+     AutoThrottle, and item pipelines.
+
+Quick start:
+
+    import curl_reap as reap
+
+    page = reap.get("https://quotes.toscrape.com")
+    print(page.css("span.text::text").getall())
+
+    class Quotes(reap.Spider):
+        start_urls = ["https://quotes.toscrape.com"]
+        def parse(self, page):
+            for q in page.css("div.quote"):
+                yield {"text": q.css_first("span.text::text"),
+                       "author": q.css_first("small.author::text")}
+            nxt = page.css_first("li.next a::attr(href)")
+            if nxt:
+                yield reap.Request("https://quotes.toscrape.com" + nxt, self.parse)
+
+    items = reap.run(Quotes, concurrency=8)
+"""
+from .adaptive import relocate, save, signature, similarity
+from .engine import Reaper, run
+from .http import Response, Session, fetch, get, post
+from .parser import Selector, SelectorList
+from .pipelines import CsvPipeline, DedupPipeline, JsonLinesPipeline, Pipeline
+from .spider import Request, Spider
+from .throttle import AutoThrottle
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "get", "post", "fetch", "Session", "Response",
+    "Selector", "SelectorList",
+    "Spider", "Request", "Reaper", "run",
+    "Pipeline", "DedupPipeline", "JsonLinesPipeline", "CsvPipeline", "AutoThrottle",
+    "signature", "similarity", "save", "relocate",
+    "__version__",
+]

curl_reap-0.1.0/curl_reap/adaptive.py

@@ -0,0 +1,105 @@
+"""Self-healing selectors (the Scrapling-inspired pillar).
+
+Save a structural signature of an element once. Later, even if the site renames
+classes or reshuffles its DOM, relocate() finds the element again by scoring every
+node against that signature. This keeps scrapers alive across markup changes, which
+is the single biggest maintenance cost of plain CSS/XPath scrapers.
+"""
+from __future__ import annotations
+
+import json
+import os
+
+DEFAULT_STORE = ".reap_selectors.json"
+
+_WEIGHTS = {"tag": 2.0, "classes": 3.0, "id": 2.0, "attrs": 1.0, "text": 1.5, "path": 1.5}
+
+
+def signature(el):
+    """A compact, comparable description of one element and where it sits."""
+    parent = el.getparent()
+    sibs = list(parent) if parent is not None else [el]
+    idx = sibs.index(el) if el in sibs else 0
+    return {
+        "tag": str(el.tag),
+        "classes": sorted((el.get("class") or "").split()),
+        "id": el.get("id") or "",
+        "attrs": sorted(k for k in el.keys() if k not in ("class", "id")),
+        "text": (el.text or "").strip()[:48],
+        "depth": _depth(el),
+        "index": idx,
+        "path": _path(el),
+    }
+
+
+def _depth(el):
+    d = 0
+    p = el.getparent()
+    while p is not None:
+        d += 1
+        p = p.getparent()
+    return d
+
+
+def _path(el):
+    parts = []
+    cur = el
+    while cur is not None and isinstance(cur.tag, str):
+        parts.append(cur.tag)
+        cur = cur.getparent()
+    return "/".join(reversed(parts))
+
+
+def _jaccard(a, b):
+    a, b = set(a), set(b)
+    if not a and not b:
+        return 1.0
+    return len(a & b) / max(1, len(a | b))
+
+
+def similarity(a, b):
+    """0..1 similarity between two signatures."""
+    score = total = 0.0
+    score += _WEIGHTS["tag"] * (1.0 if a["tag"] == b["tag"] else 0.0)
+    score += _WEIGHTS["classes"] * _jaccard(a["classes"], b["classes"])
+    score += _WEIGHTS["id"] * (1.0 if a["id"] and a["id"] == b["id"] else 0.0)
+    score += _WEIGHTS["attrs"] * _jaccard(a["attrs"], b["attrs"])
+    score += _WEIGHTS["text"] * (1.0 if a["text"] and a["text"] == b["text"] else 0.0)
+    tail_a, tail_b = a["path"].split("/")[-3:], b["path"].split("/")[-3:]
+    path_sc = 1.0 if a["path"] == b["path"] else (0.5 if tail_a == tail_b else 0.0)
+    score += _WEIGHTS["path"] * path_sc
+    total = sum(_WEIGHTS.values())
+    return score / total
+
+
+def save(identifier, el, storage=None):
+    path = storage or DEFAULT_STORE
+    data = _load(path)
+    data[identifier] = signature(el)
+    with open(path, "w", encoding="utf-8") as fh:
+        json.dump(data, fh, indent=1)
+
+
+def relocate(identifier, tree, storage=None, threshold=0.6):
+    """Return the best-matching element for a saved identifier, or None."""
+    sig = _load(storage or DEFAULT_STORE).get(identifier)
+    if not sig:
+        return None
+    best, best_score = None, threshold
+    for e in tree.iter():
+        if not isinstance(e.tag, str):
+            continue
+        sc = similarity(sig, signature(e))
+        if sc > best_score:
+            best, best_score = e, sc
+    return best
+
+
+def _load(path):
+    if not os.path.exists(path):
+        return {}
+    try:
+        with open(path, encoding="utf-8") as fh:
+            return json.load(fh)
+    except Exception:  # noqa: BLE001
+        return {}

curl_reap-0.1.0/curl_reap/engine.py

@@ -0,0 +1,99 @@
+"""The crawl engine (the Scrapy idea, kept small): concurrent fetching with
+dedup, retries, AutoThrottle, and item pipelines, on top of the impersonating
+transport. Spider callbacks yield items (dicts) and further Requests.
+"""
+from __future__ import annotations
+
+import threading
+from concurrent.futures import ThreadPoolExecutor
+
+from .http import Session
+from .pipelines import DedupPipeline
+from .spider import Request
+from .throttle import AutoThrottle
+
+
+class Reaper:
+    def __init__(self, spider, concurrency=8, retries=2, throttle=True, delay=0.0,
+                 impersonate="chrome124", pipelines=None, dedup=True, on_item=None,
+                 max_pages=None):
+        self.spider = spider
+        self.concurrency = concurrency
+        self.max_pages = max_pages
+        self.session = Session(impersonate=impersonate, retries=retries)
+        self.throttle = AutoThrottle(base_delay=delay, target_concurrency=concurrency, enabled=throttle)
+        self.pipelines = list(pipelines or [])
+        if dedup and not any(isinstance(p, DedupPipeline) for p in self.pipelines):
+            self.pipelines.insert(0, DedupPipeline())
+        self.on_item = on_item
+        self._seen = set()
+        self.items = []
+        self._lock = threading.Lock()
+        self.stats = {"requests": 0, "items": 0, "errors": 0, "dropped": 0}
+
+    def _fetch_and_parse(self, req):
+        with self._lock:
+            fp = req.fingerprint()
+            if fp in self._seen:
+                return []
+            self._seen.add(fp)
+            if self.max_pages and self.stats["requests"] >= self.max_pages:
+                return []
+        self.throttle.wait()
+        import time
+        t0 = time.time()
+        try:
+            resp = self.session.request(req.method, req.url, meta=req.meta, **req.kw)
+        except Exception:  # noqa: BLE001
+            with self._lock:
+                self.stats["errors"] += 1
+            return []
+        self.throttle.observe(time.time() - t0)
+        with self._lock:
+            self.stats["requests"] += 1
+        callback = req.callback or self.spider.parse
+        produced = []
+        try:
+            for out in (callback(resp) or []):
+                produced.append(out)
+        except Exception:  # noqa: BLE001
+            with self._lock:
+                self.stats["errors"] += 1
+        return produced
+
+    def run(self):
+        for p in self.pipelines:
+            p.open()
+        frontier = list(self.spider.start())
+        with ThreadPoolExecutor(max_workers=self.concurrency) as pool:
+            while frontier:
+                futures = [pool.submit(self._fetch_and_parse, r) for r in frontier]
+                frontier = []
+                for fut in futures:
+                    for out in fut.result():
+                        if isinstance(out, Request):
+                            frontier.append(out)
+                        elif out is not None:
+                            self._emit(out)
+        for p in self.pipelines:
+            p.close()
+        return self.items
+
+    def _emit(self, item):
+        for p in self.pipelines:
+            item = p.process(item)
+            if item is None:
+                with self._lock:
+                    self.stats["dropped"] += 1
+                return
+        with self._lock:
+            self.items.append(item)
+            self.stats["items"] += 1
+        if self.on_item:
+            self.on_item(item)
+
+
+def run(spider, **kw):
+    """Run a Spider (class or instance) to completion. Returns the scraped items."""
+    sp = spider() if isinstance(spider, type) else spider
+    return Reaper(sp, **kw).run()

curl_reap-0.1.0/curl_reap/http.py

@@ -0,0 +1,128 @@
+"""Transport layer: curl_cffi sessions with real browser TLS/JA3 impersonation.
+
+This is the "get past the front door" pillar (the curl_cffi strength). Every
+request carries a genuine Chrome/Safari TLS + HTTP2 fingerprint, which is what
+defeats fingerprint-based bot detection that blocks stock Python clients.
+"""
+from __future__ import annotations
+
+from curl_cffi import requests as _cffi
+
+from .parser import Selector
+
+DEFAULT_IMPERSONATE = "chrome124"
+
+
+class Response:
+    """A fetched page. Behaves like a parser (css/xpath pass through to a Selector)."""
+
+    def __init__(self, raw, meta=None):
+        self.raw = raw
+        self.status = raw.status_code
+        self.url = str(raw.url)
+        self.headers = dict(raw.headers)
+        self.text = raw.text
+        self.content = raw.content
+        self.meta = meta or {}
+        self._sel = None
+
+    @property
+    def ok(self):
+        return 200 <= self.status < 300
+
+    def selector(self):
+        if self._sel is None:
+            self._sel = Selector(content=self.text, url=self.url, status=self.status, headers=self.headers)
+        return self._sel
+
+    # parser pass-throughs so a Response is usable directly as a page
+    def css(self, *a, **k):
+        return self.selector().css(*a, **k)
+
+    def css_first(self, *a, **k):
+        return self.selector().css_first(*a, **k)
+
+    def xpath(self, *a, **k):
+        return self.selector().xpath(*a, **k)
+
+    def find_by_text(self, *a, **k):
+        return self.selector().find_by_text(*a, **k)
+
+    def find_similar(self, *a, **k):
+        return self.selector().find_similar(*a, **k)
+
+    def re(self, *a, **k):
+        return self.selector().re(*a, **k)
+
+    def save(self, *a, **k):
+        return self.selector().save(*a, **k)
+
+    def json(self):
+        return self.raw.json()
+
+    def __repr__(self):
+        return f"<Response {self.status} {self.url}>"
+
+
+class Session:
+    """A reusable curl_cffi session with impersonation, default headers, retries."""
+
+    def __init__(self, impersonate=DEFAULT_IMPERSONATE, headers=None, timeout=30,
+                 retries=2, proxies=None, **kw):
+        self.impersonate = impersonate
+        self.timeout = timeout
+        self.retries = retries
+        self._headers = dict(headers or {})
+        self._s = _cffi.Session(impersonate=impersonate, proxies=proxies, **kw)
+
+    def request(self, method, url, **kw):
+        kw.setdefault("impersonate", self.impersonate)
+        kw.setdefault("timeout", self.timeout)
+        merged = dict(self._headers)
+        merged.update(kw.pop("headers", {}) or {})
+        if merged:
+            kw["headers"] = merged
+        meta = kw.pop("meta", None)
+        retries = kw.pop("retries", self.retries)
+        last = None
+        for _ in range(retries + 1):
+            try:
+                return Response(self._s.request(method, url, **kw), meta=meta)
+            except Exception as exc:  # noqa: BLE001
+                last = exc
+        raise last
+
+    def get(self, url, **kw):
+        return self.request("GET", url, **kw)
+
+    def post(self, url, **kw):
+        return self.request("POST", url, **kw)
+
+    def close(self):
+        try:
+            self._s.close()
+        except Exception:  # noqa: BLE001
+            pass
+
+
+_default = None
+
+
+def _session():
+    global _default
+    if _default is None:
+        _default = Session()
+    return _default
+
+
+def get(url, **kw):
+    """One-shot GET with a shared impersonating session. Returns a Response."""
+    return _session().get(url, **kw)
+
+
+def post(url, **kw):
+    return _session().post(url, **kw)
+
+
+def fetch(url, **kw):
+    return get(url, **kw)

curl_reap-0.1.0/curl_reap/parser.py

@@ -0,0 +1,172 @@
+"""Parsing layer: a fast lxml selector with parsel-style ergonomics plus the
+Scrapling-style extras (find by text, find similar, and self-healing selectors).
+
+Supports CSS with ::text and ::attr(name) pseudo elements, XPath, regex, and an
+auto_match mode that re-locates an element from a saved signature when the site
+changes its markup (see adaptive.py).
+"""
+from __future__ import annotations
+
+import re
+
+import lxml.html
+
+_ATTR_RE = re.compile(r"::attr\(\s*([\w:-]+)\s*\)\s*$")
+_TEXT_RE = re.compile(r"::text\s*$")
+
+
+def _parse_pseudo(query):
+    m = _ATTR_RE.search(query)
+    if m:
+        return ("attr", m.group(1), _ATTR_RE.sub("", query).strip())
+    if _TEXT_RE.search(query):
+        return ("text", None, _TEXT_RE.sub("", query).strip())
+    return (None, None, query)
+
+
+def _text_of(el):
+    try:
+        return el.text_content().strip()
+    except Exception:  # noqa: BLE001
+        return (getattr(el, "text", "") or "").strip()
+
+
+class SelectorList(list):
+    """A list of Selectors or strings with parsel-style get / getall helpers."""
+
+    def get(self, default=None):
+        return self[0] if self else default
+
+    def getall(self):
+        return list(self)
+
+    def text(self):
+        out = SelectorList()
+        for s in self:
+            out.append(s.text if isinstance(s, Selector) else s)
+        return out
+
+    def attr(self, name, default=None):
+        out = SelectorList()
+        for s in self:
+            if isinstance(s, Selector):
+                out.append(s.attr(name, default))
+        return out
+
+    def css(self, query, **kw):
+        out = SelectorList()
+        for s in self:
+            if isinstance(s, Selector):
+                out.extend(s.css(query, **kw))
+        return out
+
+
+class Selector:
+    """Wraps one lxml element (or a parsed document)."""
+
+    def __init__(self, content=None, element=None, url=None, status=None, headers=None):
+        if element is not None:
+            self._el = element
+        elif content is not None:
+            self._el = lxml.html.fromstring(content)
+        else:
+            self._el = lxml.html.fromstring("<html></html>")
+        self.url = url
+        self.status = status
+        self.headers = dict(headers or {})
+
+    # --- selection ---------------------------------------------------------
+    def css(self, query, auto_match=False, identifier=None, storage=None):
+        kind, attr, q = _parse_pseudo(query)
+        try:
+            els = self._el.cssselect(q) if q else [self._el]
+        except Exception:  # noqa: BLE001
+            els = []
+        if not els and auto_match:
+            from .adaptive import relocate
+            found = relocate(identifier or query, self._root(), storage=storage)
+            els = [found] if found is not None else []
+        if kind == "attr":
+            return SelectorList(e.get(attr) for e in els)
+        if kind == "text":
+            return SelectorList(_text_of(e) for e in els)
+        return SelectorList(Selector(element=e, url=self.url) for e in els)
+
+    def css_first(self, query, default=None, **kw):
+        res = self.css(query, **kw)
+        return res[0] if res else default
+
+    def xpath(self, query):
+        try:
+            res = self._el.xpath(query)
+        except Exception:  # noqa: BLE001
+            return SelectorList()
+        out = SelectorList()
+        for r in res:
+            out.append(r if isinstance(r, str) else Selector(element=r, url=self.url))
+        return out
+
+    # --- Scrapling-style finders ------------------------------------------
+    def find_by_text(self, text, partial=True, first=False):
+        out = SelectorList()
+        for e in self._el.iter():
+            if not isinstance(e.tag, str):
+                continue
+            t = (e.text or "").strip()
+            hit = (text in t) if partial else (text == t)
+            if hit:
+                out.append(Selector(element=e, url=self.url))
+                if first:
+                    break
+        return out
+
+    def find_similar(self, sample, threshold=0.6, limit=None):
+        """Return elements structurally similar to a sample Selector."""
+        from .adaptive import signature, similarity
+        target = signature(sample._el if isinstance(sample, Selector) else sample)
+        scored = []
+        for e in self._el.iter():
+            if not isinstance(e.tag, str) or e is getattr(sample, "_el", None):
+                continue
+            sc = similarity(target, signature(e))
+            if sc >= threshold:
+                scored.append((sc, e))
+        scored.sort(key=lambda x: -x[0])
+        if limit:
+            scored = scored[:limit]
+        return SelectorList(Selector(element=e, url=self.url) for _, e in scored)
+
+    def save(self, identifier, storage=None):
+        """Persist this element's signature so css(auto_match=True) can re-find it."""
+        from .adaptive import save as _save
+        _save(identifier, self._el, storage=storage)
+        return self
+
+    # --- value access ------------------------------------------------------
+    @property
+    def text(self):
+        return _text_of(self._el)
+
+    @property
+    def attrib(self):
+        return dict(self._el.attrib)
+
+    def attr(self, name, default=None):
+        return self._el.get(name, default)
+
+    @property
+    def html(self):
+        return lxml.html.tostring(self._el, encoding="unicode")
+
+    def re(self, pattern, flags=0):
+        return SelectorList(re.findall(pattern, self.html, flags))
+
+    def _root(self):
+        root = self._el
+        while root.getparent() is not None:
+            root = root.getparent()
+        return root
+
+    def __repr__(self):
+        t = getattr(self._el, "tag", "?")
+        return f"<Selector {t}>"

curl_reap-0.1.0/curl_reap/pipelines.py

@@ -0,0 +1,79 @@
+"""Item pipelines (the Scrapy idea): each scraped item flows through a chain that
+can validate, transform, dedup, or export it. A pipeline returning None drops the
+item.
+"""
+from __future__ import annotations
+
+import csv
+import json
+
+
+class Pipeline:
+    def open(self):
+        pass
+
+    def process(self, item):
+        return item
+
+    def close(self):
+        pass
+
+
+class DedupPipeline(Pipeline):
+    """Drop items already seen. key=None dedups on the whole item."""
+
+    def __init__(self, key=None):
+        self.key = key
+        self.seen = set()
+
+    def process(self, item):
+        try:
+            k = item.get(self.key) if self.key else json.dumps(item, sort_keys=True, default=str)
+        except Exception:  # noqa: BLE001
+            k = str(item)
+        if k in self.seen:
+            return None
+        self.seen.add(k)
+        return item
+
+
+class JsonLinesPipeline(Pipeline):
+    """Stream items to a .jsonl file as they are scraped."""
+
+    def __init__(self, path):
+        self.path = path
+        self._fh = None
+
+    def open(self):
+        self._fh = open(self.path, "w", encoding="utf-8")
+
+    def process(self, item):
+        self._fh.write(json.dumps(item, ensure_ascii=False, default=str) + "\n")
+        return item
+
+    def close(self):
+        if self._fh:
+            self._fh.close()
+
+
+class CsvPipeline(Pipeline):
+    """Collect items and write a CSV on close (header from the first item)."""
+
+    def __init__(self, path):
+        self.path = path
+        self._rows = []
+
+    def process(self, item):
+        if isinstance(item, dict):
+            self._rows.append(item)
+        return item
+
+    def close(self):
+        if not self._rows:
+            return
+        cols = list({k: None for row in self._rows for k in row})
+        with open(self.path, "w", newline="", encoding="utf-8") as fh:
+            w = csv.DictWriter(fh, fieldnames=cols)
+            w.writeheader()
+            for row in self._rows:
+                w.writerow(row)

curl_reap-0.1.0/curl_reap/spider.py

@@ -0,0 +1,33 @@
+"""Spider and Request: the unit of work for the crawl engine."""
+from __future__ import annotations
+
+
+class Request:
+    """A pending fetch plus the callback that parses its Response."""
+
+    def __init__(self, url, callback=None, method="GET", meta=None, **kw):
+        self.url = url
+        self.callback = callback
+        self.method = method
+        self.meta = meta or {}
+        self.kw = kw
+
+    def fingerprint(self):
+        return f"{self.method}:{self.url}"
+
+    def __repr__(self):
+        return f"<Request {self.method} {self.url}>"
+
+
+class Spider:
+    """Subclass this: set start_urls and implement parse(self, page)."""
+
+    name = "reap"
+    start_urls = []
+
+    def start(self):
+        for url in self.start_urls:
+            yield Request(url, self.parse)
+
+    def parse(self, page):
+        raise NotImplementedError("Spider.parse must be implemented")

curl_reap-0.1.0/curl_reap/throttle.py

@@ -0,0 +1,31 @@
+"""AutoThrottle: adapt the delay to the server's observed latency so the crawl
+stays polite and avoids IP bans, the way Scrapy's AutoThrottle does.
+"""
+from __future__ import annotations
+
+import threading
+import time
+
+
+class AutoThrottle:
+    def __init__(self, base_delay=0.0, target_concurrency=8, max_delay=10.0, enabled=True):
+        self.delay = base_delay
+        self.target = max(1, target_concurrency)
+        self.max_delay = max_delay
+        self.enabled = enabled
+        self._latencies = []
+        self._lock = threading.Lock()
+
+    def observe(self, latency):
+        if not self.enabled:
+            return
+        with self._lock:
+            self._latencies.append(latency)
+            self._latencies = self._latencies[-20:]
+            avg = sum(self._latencies) / len(self._latencies)
+            # aim for ~target concurrent requests: per-request delay = latency / target
+            self.delay = min(self.max_delay, max(0.0, avg / self.target))
+
+    def wait(self):
+        if self.enabled and self.delay > 0:
+            time.sleep(self.delay)

curl_reap-0.1.0/pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "curl_reap"
+version = "0.1.0"
+description = "Reap the web: browser-grade TLS impersonation, self-healing selectors, and a concurrent crawl engine in one library."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "Anish", email = "anishfyi@gmail.com" }]
+keywords = ["scraping", "crawler", "curl_cffi", "tls-fingerprint", "impersonate", "selectors", "lxml", "spider"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Internet :: WWW/HTTP",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+  "curl_cffi>=0.7.0",
+  "lxml>=5.0",
+  "cssselect>=1.2",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7.0", "build>=1.0", "twine>=5.0"]
+
+[project.urls]
+Homepage = "https://github.com/anishfyi/curl_reap"
+Repository = "https://github.com/anishfyi/curl_reap"
+Issues = "https://github.com/anishfyi/curl_reap/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["curl_reap"]
+
+[tool.hatch.build.targets.sdist]
+include = ["curl_reap", "README.md", "LICENSE", "assets/logo.svg"]