voidcrawl 0.3.3__tar.gz → 0.3.5__tar.gz

{voidcrawl-0.3.3 → voidcrawl-0.3.5}/Cargo.lock

@@ -188,9 +188,9 @@ dependencies = [
 
 [[package]]
 name = "bitflags"
-version = "2.11.1"
+version = "2.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
 dependencies = [
  "serde_core",
 ]
@@ -342,9 +342,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.44"
+version = "0.4.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
 dependencies = [
  "iana-time-zone",
  "num-traits",
@@ -1044,9 +1044,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
 [[package]]
 name = "http"
-version = "1.4.1"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"
+checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
 dependencies = [
  "bytes",
  "itoa",
@@ -1287,9 +1287,9 @@ dependencies = [
 
 [[package]]
 name = "ignore"
-version = "0.4.25"
+version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3d782a365a015e0f5c04902246139249abf769125006fbe7649e2ee88169b4a"
+checksum = "b915661dd01db3f05050265b2477bcc6527b3792388e2749b41623cc592be67d"
 dependencies = [
  "crossbeam-deque",
  "globset",
@@ -1415,13 +1415,12 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.99"
+version = "0.3.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
+checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162"
 dependencies = [
  "cfg-if",
  "futures-util",
- "once_cell",
  "wasm-bindgen",
 ]
 
@@ -1478,9 +1477,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.30"
+version = "0.4.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"
+checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a"
 
 [[package]]
 name = "logos"
@@ -1950,9 +1949,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.12.3"
+version = "1.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1973,9 +1972,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.10"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "reqwest"
@@ -2124,9 +2123,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-native-certs"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
 dependencies = [
  "openssl-probe",
  "rustls-pki-types",
@@ -2817,9 +2816,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.23.2"
+version = "1.23.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"
+checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7"
 dependencies = [
  "getrandom 0.4.2",
  "js-sys",
@@ -2846,7 +2845,7 @@ checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
 
 [[package]]
 name = "void_crawl"
-version = "0.3.3"
+version = "0.3.5"
 dependencies = [
  "futures",
  "pyo3",
@@ -2858,12 +2857,13 @@ dependencies = [
 
 [[package]]
 name = "void_crawl_core"
-version = "0.3.3"
+version = "0.3.5"
 dependencies = [
  "chromiumoxide",
  "fd-lock",
  "futures",
  "infer",
+ "regex",
  "reqwest",
  "rustls",
  "serde",
@@ -2876,7 +2876,7 @@ dependencies = [
 
 [[package]]
 name = "voidcrawl-mcp"
-version = "0.3.3"
+version = "0.3.5"
 dependencies = [
  "anyhow",
  "base64",
@@ -2972,9 +2972,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.122"
+version = "0.2.123"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
+checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -2985,9 +2985,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.72"
+version = "0.4.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
+checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -2995,9 +2995,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.122"
+version = "0.2.123"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
+checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -3005,9 +3005,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.122"
+version = "0.2.123"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
+checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b"
 dependencies = [
  "bumpalo",
  "proc-macro2",
@@ -3018,9 +3018,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.122"
+version = "0.2.123"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
+checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92"
 dependencies = [
  "unicode-ident",
 ]
@@ -3258,9 +3258,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.99"
+version = "0.3.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
+checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -3277,9 +3277,9 @@ dependencies = [
 
 [[package]]
 name = "which"
-version = "8.0.2"
+version = "8.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459"
+checksum = "c789537cf2f7f55be8e6192f92e464174ee55f91af622777f7f1ceb0dbccd03e"
 dependencies = [
  "libc",
 ]
@@ -3638,9 +3638,9 @@ dependencies = [
 
 [[package]]
 name = "yoke"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -3661,18 +3661,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.50"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
+checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.50"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
+checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
 dependencies = [
  "proc-macro2",
  "quote",

{voidcrawl-0.3.3 → voidcrawl-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidcrawl
-Version: 0.3.3
+Version: 0.3.5
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: click>=8.3.2
 Requires-Dist: pydantic>=2
 Requires-Dist: rich>=14.3.3 ; extra == 'debug'
-Requires-Dist: voidcrawl-mcp==0.3.3 ; extra == 'mcp'
+Requires-Dist: voidcrawl-mcp==0.3.5 ; extra == 'mcp'
 Provides-Extra: debug
 Provides-Extra: mcp
 License-File: LICENSE.md
@@ -177,6 +177,7 @@ uv run pytest tests/ -v
 ## Documentation
 
 - [Full API reference](docs/api-reference.md)
+- [Anti-bot / CDN detection](docs/antibot.md)
 - [Examples](examples/)
 
 ## Contact

{voidcrawl-0.3.3 → voidcrawl-0.3.5}/README.md

@@ -156,6 +156,7 @@ uv run pytest tests/ -v
 ## Documentation
 
 - [Full API reference](docs/api-reference.md)
+- [Anti-bot / CDN detection](docs/antibot.md)
 - [Examples](examples/)
 
 ## Contact

voidcrawl-0.3.5/crates/core/CORPUS.md

@@ -0,0 +1,54 @@
+# Anti-bot signature corpus
+
+The signature corpus embedded in [`src/antibot.rs`](src/antibot.rs) (the
+`CORPUS_JSON` const) fingerprints which anti-bot / WAF / CDN vendor is gating an
+HTTP response, and whether that vendor is *actively challenging* us (a wall) vs.
+merely *present* (a CDN fronting a site that served us fine).
+
+## Provenance & license
+
+The vendor list and the `signals[]` / `challenge[]` split are modeled on
+**`albinstman/antibot-print`** (MIT) — <https://github.com/albinstman/antibot-print>.
+
+- **Upstream license:** MIT (compatible with this Apache-2.0 / MIT crate).
+- **Reference commit:** *record the SHA you authored against here when syncing.*
+- **What we took:** the *idea* and the vendor taxonomy. The actual regex
+  patterns are **first-party** — hand-authored against the vendors we actually
+  meet, not a wholesale vendoring of the upstream JSON. This keeps the ruleset
+  small, readable, owned, and unit-testable, and avoids tracking upstream churn
+  or importing a pattern that bloats compile.
+
+## Governance
+
+- The corpus is **not** auto-synced. Treat any change as a reviewable diff.
+- **Bump `CORPUS_VERSION`** in `src/antibot.rs` on every edit. Verdicts are
+  recorded with this version so a replay-grade archive can reproduce a
+  classification deterministically — a verdict is captured data, never
+  recomputed at replay time against a newer corpus.
+- Drift is caught loudly by the **offline accuracy benchmark**
+  (`tests/antibot_accuracy.rs`): a held-out labeled corpus scored for
+  precision/recall. If a vendor changes its markers and a signature rots, that
+  test fails rather than silently lowering field accuracy.
+- The benchmark corpus is deliberately **disjoint** from the live canaries
+  (`fortress.theplumber.dev`, `bot.sannysoft.com`) so we measure the detector,
+  not one target.
+
+## Adding / editing a vendor
+
+1. Edit `CORPUS_JSON` in `src/antibot.rs`. Patterns are RE2-style (no
+   backreferences / lookaround) so they run under the linear-time `regex`
+   engine and are safe on attacker-controlled input. Matched case-insensitively
+   against the normalized `S:`/`H:`/`B:` form.
+   - `H:` patterns are anchored at the header line start (`h:server: …`).
+   - `B:` patterns match their marker *anywhere* in the (bounded) body.
+   - Put markers that only appear during an **active block** in `challenge[]`;
+     put mere-presence markers (CDN headers) in `signals[]`.
+2. Add a labeled fixture to `tests/antibot_accuracy.rs`.
+3. Bump `CORPUS_VERSION`.
+4. `cargo test -p void_crawl_core antibot` + `--test antibot_accuracy`.
+
+## Vendor coverage
+
+WAF/CDN: Cloudflare, Akamai, Imperva/Incapsula, AWS WAF, F5 BigIP, Sucuri,
+CloudFront. Bot detection: DataDome, PerimeterX/HUMAN, Kasada. Challenge
+widgets: reCAPTCHA, hCaptcha (Turnstile is folded into the `cloudflare` vendor).

{voidcrawl-0.3.3 → voidcrawl-0.3.5}/crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "void_crawl_core"
-version = "0.3.3"
+version = "0.3.5"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
@@ -22,6 +22,9 @@ futures = "0.3"
 thiserror = "2"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+# Linear-time (RE2-style) regex engine for anti-bot vendor fingerprinting.
+# No backtracking → safe to run on attacker-controlled responses every fetch.
+regex = "1"
 reqwest = { version = "0.13", default-features = false, features = ["rustls-no-provider", "json"] }
 rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
 tempfile = "3"

{voidcrawl-0.3.3 → voidcrawl-0.3.5}/crates/core/README.md

@@ -156,6 +156,7 @@ uv run pytest tests/ -v
 ## Documentation
 
 - [Full API reference](docs/api-reference.md)
+- [Anti-bot / CDN detection](docs/antibot.md)
 - [Examples](examples/)
 
 ## Contact