scraper-rust 0.3.3__tar.gz → 0.4.1__tar.gz

{scraper_rust-0.3.3 → scraper_rust-0.4.1}/Cargo.lock

@@ -545,7 +545,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2"
 dependencies = [
  "log",
- "markup5ever",
+ "markup5ever 0.38.0",
+]
+
+[[package]]
+name = "html5ever"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8"
+dependencies = [
+ "log",
+ "markup5ever 0.39.0",
 ]
 
 [[package]]
@@ -1215,6 +1225,17 @@ dependencies = [
  "web_atoms",
 ]
 
+[[package]]
+name = "markup5ever"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de"
+dependencies = [
+ "log",
+ "tendril",
+ "web_atoms",
+]
+
 [[package]]
 name = "memchr"
 version = "2.8.0"
@@ -1820,7 +1841,7 @@ dependencies = [
  "cssparser",
  "ego-tree",
  "getopts",
- "html5ever",
+ "html5ever 0.38.0",
  "precomputed-hash",
  "selectors",
  "tendril",
@@ -1828,8 +1849,9 @@ dependencies = [
 
 [[package]]
 name = "scraper-rs"
-version = "0.3.3"
+version = "0.4.1"
 dependencies = [
+ "html5ever 0.39.0",
  "pyo3",
  "pyo3-async-runtimes",
  "scraper",

{scraper_rust-0.3.3 → scraper_rust-0.4.1}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scraper-rs"
-version = "0.3.3"
+version = "0.4.1"
 edition = "2024"
 readme = "README.md"
 
@@ -20,6 +20,7 @@ tokio = { version = "1", features = ["rt", "macros"] }
 scraper = { version = "0.25.0", git = "https://github.com/rust-scraper/scraper", features = ["atomic"], rev = "d07444effb9ca192d9b3919fb920ab9c6c701c4f" }
 
 xee-xpath = "0.1.5"
+html5ever = "0.39.0"
 
 [profile.release]
 lto = "fat"

{scraper_rust-0.3.3 → scraper_rust-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scraper-rust
-Version: 0.3.3
+Version: 0.4.1
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -80,7 +80,7 @@ python examples/demo_prettify_url.py https://example.com --max-lines 80
 
 ### Async usage
 
-The `scraper_rs.asyncio` module keeps the event loop responsive. `parse` yields once (`asyncio.sleep(0)`) before constructing a sync `Document` in the current thread, while `select`/`xpath` helpers run in a thread pool. Parsed documents and elements are wrapped with awaitable selector methods for nested queries:
+The `scraper_rs.asyncio` module exposes an async-first surface for coroutine code. `AsyncDocument` stores shareable HTML/text state instead of a thread-affine sync `Document`, and all selectors are awaitable for consistent async calling style:
 
 ```py
 import asyncio
@@ -103,8 +103,7 @@ asyncio.run(main())
 ```
 
 All async functions accept the same keyword arguments as their sync counterparts (`max_size_bytes`, `truncate_on_limit`, etc.).
-Async wrappers expose the underlying sync objects via `.document` and `.element` if you need direct access.
-`AsyncDocument` supports `async with` for automatic cleanup in coroutine code.
+`AsyncDocument` supports `async with` for automatic cleanup in coroutine code, and `AsyncElement` / `AsyncDocument` both expose async `.prettify()` helpers.
 
 ### Large documents and memory safety
 
@@ -137,7 +136,7 @@ Note: Truncation happens at valid UTF-8 character boundaries to prevent encoding
 - `.xpath(expr)` / `.xpath_first(expr)` evaluate XPath expressions that return element nodes.
 - `.prettify()` renders the current DOM as an indented string for readable output/debugging.
 - `.text` returns normalized text; `Document.html` is the original input HTML; `Element.html` is inner HTML.
-- `scraper_rs.asyncio` exposes async `parse`/`select`/`xpath` wrappers to keep the event loop responsive.
+- `scraper_rs.asyncio` exposes async `parse`/`select`/`xpath` wrappers plus awaitable `AsyncDocument` / `AsyncElement` methods.
 - `Element` exposes `.tag`, `.text`, `.html`, `.attrs` plus helpers `.attr(name)`, `.get(name, default)`, `.to_dict()`.
 - Elements support nested CSS and XPath selection via `.select(css)`, `.select_first(css)`, `.find(css)`, `.css(css)`, `.xpath(expr)`, `.xpath_first(expr)`.
 - Elements also expose `.prettify()` to format element HTML with indentation.
@@ -167,6 +166,7 @@ If you have `just` installed, the repo includes helpers: `just build` (local whe
 ## Projects Using scraper-rs
 
 - [**silkworm**](https://github.com/BitingSnakes/silkworm) - Async web scraping framework on top of Rust
+- [**silkworm-mcp**](https://github.com/BitingSnakes/silkworm-mcp) - An MCP server for silkworm
 
 ## Development
 

{scraper_rust-0.3.3 → scraper_rust-0.4.1}/README.md

@@ -53,7 +53,7 @@ python examples/demo_prettify_url.py https://example.com --max-lines 80
 
 ### Async usage
 
-The `scraper_rs.asyncio` module keeps the event loop responsive. `parse` yields once (`asyncio.sleep(0)`) before constructing a sync `Document` in the current thread, while `select`/`xpath` helpers run in a thread pool. Parsed documents and elements are wrapped with awaitable selector methods for nested queries:
+The `scraper_rs.asyncio` module exposes an async-first surface for coroutine code. `AsyncDocument` stores shareable HTML/text state instead of a thread-affine sync `Document`, and all selectors are awaitable for consistent async calling style:
 
 ```py
 import asyncio
@@ -76,8 +76,7 @@ asyncio.run(main())
 ```
 
 All async functions accept the same keyword arguments as their sync counterparts (`max_size_bytes`, `truncate_on_limit`, etc.).
-Async wrappers expose the underlying sync objects via `.document` and `.element` if you need direct access.
-`AsyncDocument` supports `async with` for automatic cleanup in coroutine code.
+`AsyncDocument` supports `async with` for automatic cleanup in coroutine code, and `AsyncElement` / `AsyncDocument` both expose async `.prettify()` helpers.
 
 ### Large documents and memory safety
 
@@ -110,7 +109,7 @@ Note: Truncation happens at valid UTF-8 character boundaries to prevent encoding
 - `.xpath(expr)` / `.xpath_first(expr)` evaluate XPath expressions that return element nodes.
 - `.prettify()` renders the current DOM as an indented string for readable output/debugging.
 - `.text` returns normalized text; `Document.html` is the original input HTML; `Element.html` is inner HTML.
-- `scraper_rs.asyncio` exposes async `parse`/`select`/`xpath` wrappers to keep the event loop responsive.
+- `scraper_rs.asyncio` exposes async `parse`/`select`/`xpath` wrappers plus awaitable `AsyncDocument` / `AsyncElement` methods.
 - `Element` exposes `.tag`, `.text`, `.html`, `.attrs` plus helpers `.attr(name)`, `.get(name, default)`, `.to_dict()`.
 - Elements support nested CSS and XPath selection via `.select(css)`, `.select_first(css)`, `.find(css)`, `.css(css)`, `.xpath(expr)`, `.xpath_first(expr)`.
 - Elements also expose `.prettify()` to format element HTML with indentation.
@@ -140,6 +139,7 @@ If you have `just` installed, the repo includes helpers: `just build` (local whe
 ## Projects Using scraper-rs
 
 - [**silkworm**](https://github.com/BitingSnakes/silkworm) - Async web scraping framework on top of Rust
+- [**silkworm-mcp**](https://github.com/BitingSnakes/silkworm-mcp) - An MCP server for silkworm
 
 ## Development
 

{scraper_rust-0.3.3 → scraper_rust-0.4.1}/docs/architecture.md

@@ -4,7 +4,7 @@
 
 `scraper_rs` is a Python extension module implemented in Rust with PyO3. The Rust code in `src/lib.rs` defines the public API (`Document`, `Element`, and top-level helper functions) and is exported as the Python module `scraper_rs`. The Python package in `scraper_rs/__init__.py` simply re-exports the extension module.
 
-Async support lives in pure Python in `scraper_rs/asyncio.py`. It wraps the Rust async helpers defined in `src/lib.rs` and exposes `AsyncDocument` and `AsyncElement` wrappers to make selectors awaitable.
+Async support lives in pure Python in `scraper_rs/asyncio.py`. It exposes `AsyncDocument` and `AsyncElement` wrappers to make selectors awaitable without storing a sync `Document` object in coroutine-facing state.
 
 ## Data flow (sync)
 
@@ -33,16 +33,15 @@ Key code references:
 The async layer is a thin wrapper over the sync API:
 
 - The Python module `scraper_rs/asyncio.py` exposes `AsyncDocument` and `AsyncElement`.
-- Top-level async functions (`select`, `xpath`, `select_first`, `first`, etc) call Rust helpers in `src/lib.rs` such as `select_async`, `first_async`, and `xpath_async`.
-- The Rust async helpers use `pyo3_async_runtimes::tokio::future_into_py_with_locals` and `tokio::task::spawn_blocking` to run blocking parsing and selection on a thread pool.
-- `parse` is implemented in Python (`asyncio.sleep(0)` + sync `Document(...)` construction).
-- `AsyncDocument` / `AsyncElement` selector methods pass HTML strings into Rust async helpers, so selectors parse per call rather than reusing the wrapped `Document` DOM.
-- Nested async selection on elements is implemented via `_select_fragment_async` and `_xpath_fragment_async` in `src/lib.rs`, which parse the element's inner HTML as a fragment.
+- `AsyncDocument` stores only shareable `html` and `text` state; it does not retain a sync `Document`.
+- Top-level async functions (`select`, `xpath`, `select_first`, `first`, etc) yield once and then call the corresponding synchronous Rust helper from `src/lib.rs`.
+- `parse` yields once, constructs a temporary sync `Document`, copies out `html` and `text`, then drops the sync object.
+- `AsyncElement` wraps an immutable sync `Element` snapshot and forwards nested selectors/prettify through awaitable methods.
+- Selector calls still parse per call rather than reusing a persistent DOM, which keeps async objects thread-safe and lightweight.
 
 Code references:
 - Async wrappers: `scraper_rs/asyncio.py`
-- Rust async entry points: `src/lib.rs` (`select_async`, `select_first_async`, `first_async`, `xpath_async`, `xpath_first_async`)
-- Fragment helpers: `src/lib.rs` (`_select_fragment_async`, `_xpath_fragment_async`)
+- Rust sync entry points: `src/lib.rs` (`Document`, `Element`, `select`, `select_first`, `first`, `xpath`, `xpath_first`)
 
 ## Module wiring
 
@@ -50,7 +49,7 @@ The Rust module initializer in `src/lib.rs` registers all classes and functions
 
 - `#[pymodule] fn scraper_rs(...)` adds `Document`, `Element`, and the top-level functions.
 - The version string is exposed as `__version__` from `env!("CARGO_PKG_VERSION")`.
-- `scraper_rs/__init__.py` re-exports the extension module and forwards `__all__` if it exists.
+- `scraper_rs/__init__.py` re-exports the supported sync API explicitly.
 
 ## Types and public surface
 

{scraper_rust-0.3.3 → scraper_rust-0.4.1}/docs/async-api.md

@@ -4,13 +4,12 @@ The async API lives in `scraper_rs/asyncio.py` and provides awaitable wrappers a
 
 ## Key types
 
-- `AsyncDocument`: wraps a synchronous `Document` and exposes awaitable selectors.
-- `AsyncElement`: wraps a synchronous `Element` and exposes awaitable selectors.
-- `AsyncDocument.document` and `AsyncElement.element` provide access to the underlying sync objects.
+- `AsyncDocument`: stores shareable document state (`html`, `text`) and exposes awaitable selectors.
+- `AsyncElement`: wraps an immutable element snapshot and exposes awaitable nested selectors.
 
 Implementation references:
 - Python wrappers: `scraper_rs/asyncio.py`
-- Rust async helpers: `src/lib.rs` (`select_async`, `select_first_async`, `first_async`, `xpath_async`, `xpath_first_async`)
+- Sync primitives: `src/lib.rs` (`Document`, `Element`, `select`, `select_first`, `first`, `xpath`, `xpath_first`)
 
 ## Top-level async functions
 
@@ -43,7 +42,7 @@ asyncio.run(main())
 
 ## Context management and cleanup
 
-- `AsyncDocument` supports `async with` and closes the underlying `Document` on exit (including exception paths).
+- `AsyncDocument` supports `async with` and clears its stored HTML/text state on exit (including exception paths).
 - `AsyncDocument` also supports sync `with`, but `async with` is preferred inside coroutine code.
 
 ```py
@@ -54,21 +53,15 @@ async with doc:
 
 ## How async execution works
 
-- `scraper_rs/asyncio.py` calls Rust helpers such as `select_async` and `xpath_async`.
-- In `src/lib.rs`, each async helper uses `pyo3_async_runtimes::tokio::future_into_py_with_locals` and `tokio::task::spawn_blocking` to run parsing and selection on a thread pool.
-- `parse` is implemented in Python: it yields to the event loop with `asyncio.sleep(0)` and then constructs a sync `Document` in the current thread.
-- `AsyncDocument` and `AsyncElement` methods pass HTML strings into Rust async helpers, so each async selector call re-parses HTML/fragment input.
+- `scraper_rs/asyncio.py` presents an async API even though the parsing primitives remain synchronous in Rust.
+- Each async entry point yields to the event loop once before invoking the corresponding synchronous Rust helper.
+- `AsyncDocument` stores only `html` and `text`, so coroutine code does not hold a thread-affine sync `Document`.
+- Selector calls still parse per call, just like the sync top-level helper functions.
 
 ## Nested selection on AsyncElement
 
-Nested selectors on `AsyncElement` call fragment helpers defined in `src/lib.rs`:
-
-- `_select_fragment_async`
-- `_select_first_fragment_async`
-- `_xpath_fragment_async`
-- `_xpath_first_fragment_async`
-
-These helpers parse the element's inner HTML fragment each time (see `scraper_rs/asyncio.py` and `src/lib.rs`). This keeps the API simple but means repeated async queries re-parse the fragment.
+Nested selectors on `AsyncElement` call the wrapped sync `Element` snapshot methods from `src/lib.rs`.
+Those methods operate on the stored element HTML fragment each time, which keeps the async wrapper simple but means repeated nested queries re-parse the fragment.
 
 ## Performance notes
 

scraper_rust-0.4.1/examples/demo_async_document.py

@@ -0,0 +1,83 @@
+"""Example showing the async document-oriented API."""
+
+from __future__ import annotations
+
+import asyncio
+
+from scraper_rs import asyncio as async_scraper
+
+HTML = """
+<html>
+  <body>
+    <main>
+      <section class="card" data-id="1">
+        <h2>Alpha</h2>
+        <a href="/alpha">Read more</a>
+      </section>
+      <section class="card featured" data-id="2">
+        <h2>Beta</h2>
+        <a href="/beta">Read more</a>
+      </section>
+      <section class="card" data-id="3">
+        <h2>Gamma</h2>
+        <a href="/gamma">Read more</a>
+      </section>
+    </main>
+  </body>
+</html>
+"""
+
+
+async def main() -> None:
+    print("AsyncDocument workflow")
+    print()
+
+    async with await async_scraper.parse(HTML) as doc:
+        print(doc)
+        print(f"Document text: {doc.text}")
+        print()
+
+        cards = await doc.select(".card")
+        print(f"Found {len(cards)} cards")
+
+        for card in cards:
+            title = await card.select_first("h2")
+            link = await card.select_first("a[href]")
+            print(
+                f"  - id={card.attr('data-id')}: "
+                f"{title.text if title else 'missing title'} "
+                f"-> {link.attr('href') if link else 'missing link'}"
+            )
+
+        print()
+
+        featured = await doc.find(".featured")
+        if featured:
+            featured_link = await featured.find("a[href]")
+            print("Featured card")
+            print(f"  text: {featured.text}")
+            print(
+                f"  href: {featured_link.attr('href') if featured_link else 'missing'}"
+            )
+            print()
+
+        first_link, first_featured = await asyncio.gather(
+            doc.select_first("a[href]"),
+            doc.xpath_first("//section[contains(@class, 'featured')]"),
+        )
+        print("Concurrent queries")
+        print(f"  first link: {first_link.attr('href') if first_link else 'missing'}")
+        print(
+            "  featured via xpath: "
+            f"{first_featured.attr('data-id') if first_featured else 'missing'}"
+        )
+        print()
+
+        pretty = await doc.prettify()
+        print("Prettified HTML preview")
+        for line in pretty.splitlines()[:10]:
+            print(f"  {line}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

{scraper_rust-0.3.3 → scraper_rust-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "scraper-rust"
-version = "0.3.3"
+version = "0.4.1"
 description = "Python bindings around rust-scraper/scraper with PyO3"
 authors = [{ name = "Yehor Smoliakov", email = "egorsmkv@gmail.com" }]
 requires-python = ">=3.10"

scraper_rust-0.4.1/scraper_rs/__init__.py

@@ -0,0 +1,21 @@
+"""scraper_rs - Python bindings for the Rust scraper crate."""
+
+from . import scraper_rs as _scraper_rs
+
+__doc__ = _scraper_rs.__doc__
+__all__ = [
+    "Document",
+    "Element",
+    "__version__",
+    "first",
+    "parse",
+    "parse_document",
+    "parse_fragment",
+    "prettify",
+    "select",
+    "select_first",
+    "xpath",
+    "xpath_first",
+]
+
+globals().update({name: getattr(_scraper_rs, name) for name in __all__})