domq 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- domq-0.4.0/.gitignore +12 -0
- domq-0.4.0/CHANGELOG.md +273 -0
- domq-0.4.0/CODE_OF_CONDUCT.md +18 -0
- domq-0.4.0/CONTRIBUTING.md +69 -0
- domq-0.4.0/Cargo.toml +89 -0
- domq-0.4.0/LICENSE +21 -0
- domq-0.4.0/LICENSE-MIT +21 -0
- domq-0.4.0/PKG-INFO +310 -0
- domq-0.4.0/README.md +459 -0
- domq-0.4.0/ROADMAP.md +392 -0
- domq-0.4.0/SECURITY.md +45 -0
- domq-0.4.0/docs/ARCHITECTURE.md +195 -0
- domq-0.4.0/docs/PUBLISHING.md +83 -0
- domq-0.4.0/docs/assets/banner-dark.svg +51 -0
- domq-0.4.0/docs/assets/banner-light.svg +51 -0
- domq-0.4.0/docs/assets/demo.svg +32 -0
- domq-0.4.0/docs/assets/logo.svg +29 -0
- domq-0.4.0/docs/assets/social-preview.png +0 -0
- domq-0.4.0/docs/assets/social-preview.svg +51 -0
- domq-0.4.0/domq/__init__.py +38 -0
- domq-0.4.0/domq/__init__.pyi +279 -0
- domq-0.4.0/domq/_extract.py +633 -0
- domq-0.4.0/domq/py.typed +0 -0
- domq-0.4.0/examples/parse.rs +37 -0
- domq-0.4.0/examples/query.rs +40 -0
- domq-0.4.0/examples/to_markdown.rs +34 -0
- domq-0.4.0/pyproject.toml +54 -0
- domq-0.4.0/python/.gitignore +8 -0
- domq-0.4.0/python/Cargo.lock +137 -0
- domq-0.4.0/python/Cargo.toml +37 -0
- domq-0.4.0/python/LICENSE-MIT +21 -0
- domq-0.4.0/python/README.md +282 -0
- domq-0.4.0/python/src/lib.rs +364 -0
- domq-0.4.0/python/tests/test_domq.py +382 -0
- domq-0.4.0/python/tests/test_extract.py +1197 -0
- domq-0.4.0/scripts/gen_entities.py +76 -0
- domq-0.4.0/src/builder.rs +489 -0
- domq-0.4.0/src/edit.rs +770 -0
- domq-0.4.0/src/entities.rs +502 -0
- domq-0.4.0/src/entities_full.rs +2137 -0
- domq-0.4.0/src/error.rs +95 -0
- domq-0.4.0/src/events.rs +204 -0
- domq-0.4.0/src/extract/links.rs +380 -0
- domq-0.4.0/src/extract/meta.rs +388 -0
- domq-0.4.0/src/extract/mod.rs +52 -0
- domq-0.4.0/src/extract/readable.rs +537 -0
- domq-0.4.0/src/extract/table.rs +429 -0
- domq-0.4.0/src/from_html.rs +1000 -0
- domq-0.4.0/src/lib.rs +449 -0
- domq-0.4.0/src/node.rs +203 -0
- domq-0.4.0/src/query.rs +1537 -0
- domq-0.4.0/src/scan.rs +173 -0
- domq-0.4.0/src/serialize/chunk.rs +564 -0
- domq-0.4.0/src/serialize/html.rs +250 -0
- domq-0.4.0/src/serialize/json.rs +166 -0
- domq-0.4.0/src/serialize/markdown.rs +1030 -0
- domq-0.4.0/src/serialize/mod.rs +101 -0
- domq-0.4.0/src/serialize/serde_impl.rs +82 -0
- domq-0.4.0/src/serialize/text.rs +256 -0
- domq-0.4.0/src/source.rs +45 -0
- domq-0.4.0/src/tags.rs +241 -0
- domq-0.4.0/src/token.rs +57 -0
- domq-0.4.0/src/tokenizer.rs +406 -0
- domq-0.4.0/src/tree.rs +742 -0
- domq-0.4.0/src/url.rs +348 -0
- domq-0.4.0/tests/chunks.rs +354 -0
- domq-0.4.0/tests/corpus/basic.html +4 -0
- domq-0.4.0/tests/corpus/document.html +15 -0
- domq-0.4.0/tests/corpus/messy.html +7 -0
- domq-0.4.0/tests/corpus/omitted.html +26 -0
- domq-0.4.0/tests/corpus/ruby.html +3 -0
- domq-0.4.0/tests/corpus/select.html +12 -0
- domq-0.4.0/tests/corpus/tables.html +16 -0
- domq-0.4.0/tests/corpus.rs +36 -0
- domq-0.4.0/tests/editing.rs +928 -0
- domq-0.4.0/tests/editing_properties.proptest-regressions +7 -0
- domq-0.4.0/tests/editing_properties.rs +465 -0
- domq-0.4.0/tests/entities.rs +170 -0
- domq-0.4.0/tests/events.rs +559 -0
- domq-0.4.0/tests/extract.rs +1039 -0
- domq-0.4.0/tests/extract_derive_properties.rs +187 -0
- domq-0.4.0/tests/extract_properties.rs +331 -0
- domq-0.4.0/tests/html_output.rs +133 -0
- domq-0.4.0/tests/json_output.rs +46 -0
- domq-0.4.0/tests/links.rs +371 -0
- domq-0.4.0/tests/main_content.rs +367 -0
- domq-0.4.0/tests/markdown_output.rs +579 -0
- domq-0.4.0/tests/metadata.rs +380 -0
- domq-0.4.0/tests/output_properties.rs +259 -0
- domq-0.4.0/tests/pages/blog.html +100 -0
- domq-0.4.0/tests/pages/docs.html +61 -0
- domq-0.4.0/tests/pages/messy.html +24 -0
- domq-0.4.0/tests/pages/reader/blog-comments.html +179 -0
- domq-0.4.0/tests/pages/reader/docs.html +95 -0
- domq-0.4.0/tests/pages/reader/garbage.html +6 -0
- domq-0.4.0/tests/pages/reader/news.html +100 -0
- domq-0.4.0/tests/pages/reader/paragraphs.html +10 -0
- domq-0.4.0/tests/pages/reader/product.html +95 -0
- domq-0.4.0/tests/parsing.rs +748 -0
- domq-0.4.0/tests/properties.rs +93 -0
- domq-0.4.0/tests/query.rs +776 -0
- domq-0.4.0/tests/query_memory.rs +109 -0
- domq-0.4.0/tests/query_properties.proptest-regressions +23 -0
- domq-0.4.0/tests/query_properties.rs +458 -0
- domq-0.4.0/tests/serde_roundtrip.rs +27 -0
- domq-0.4.0/tests/snapshots/corpus__basic.snap +5 -0
- domq-0.4.0/tests/snapshots/corpus__document.snap +5 -0
- domq-0.4.0/tests/snapshots/corpus__html_basic.snap +8 -0
- domq-0.4.0/tests/snapshots/corpus__html_document.snap +19 -0
- domq-0.4.0/tests/snapshots/corpus__html_messy.snap +11 -0
- domq-0.4.0/tests/snapshots/corpus__html_omitted.snap +30 -0
- domq-0.4.0/tests/snapshots/corpus__html_ruby.snap +7 -0
- domq-0.4.0/tests/snapshots/corpus__html_select.snap +16 -0
- domq-0.4.0/tests/snapshots/corpus__html_tables.snap +20 -0
- domq-0.4.0/tests/snapshots/corpus__messy.snap +5 -0
- domq-0.4.0/tests/snapshots/corpus__omitted.snap +5 -0
- domq-0.4.0/tests/snapshots/corpus__ruby.snap +5 -0
- domq-0.4.0/tests/snapshots/corpus__select.snap +5 -0
- domq-0.4.0/tests/snapshots/corpus__tables.snap +5 -0
- domq-0.4.0/tests/snapshots/markdown_output__page_blog.snap +66 -0
- domq-0.4.0/tests/snapshots/markdown_output__page_docs.snap +56 -0
- domq-0.4.0/tests/snapshots/markdown_output__page_messy.snap +25 -0
- domq-0.4.0/tests/snapshots/text_output__page_blog.snap +60 -0
- domq-0.4.0/tests/snapshots/text_output__page_docs.snap +51 -0
- domq-0.4.0/tests/snapshots/text_output__page_messy.snap +22 -0
- domq-0.4.0/tests/source_spans.rs +195 -0
- domq-0.4.0/tests/tables.rs +337 -0
- domq-0.4.0/tests/text_output.rs +234 -0
- domq-0.4.0/tests/url.rs +250 -0
domq-0.4.0/.gitignore
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/target
|
|
2
|
+
**/target
|
|
3
|
+
Cargo.lock
|
|
4
|
+
/wasm/pkg
|
|
5
|
+
# wasm build output for the playground (built in CI by .github/workflows/pages.yml)
|
|
6
|
+
/playground/pkg
|
|
7
|
+
*.profraw
|
|
8
|
+
.DS_Store
|
|
9
|
+
# cargo-fuzz working dirs (not the tests/corpus/ fixtures!)
|
|
10
|
+
/fuzz/corpus/
|
|
11
|
+
/fuzz/artifacts/
|
|
12
|
+
/.claude/worktrees/
|
domq-0.4.0/CHANGELOG.md
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format is based on
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/), and this project adheres to
|
|
5
|
+
[Semantic Versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [Unreleased] — 0.4.0
|
|
8
|
+
|
|
9
|
+
"Declarative scraping", and one name everywhere.
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
- **Renamed to `domq`.** The crate is `domq` (was `dom-tree-rs`, imported as
|
|
13
|
+
`domtree`), the CLI crate `domq-cli` with the binary `domq` (was
|
|
14
|
+
`domtree-cli` / `domtree`), the npm package `domq-wasm` (was `dom-tree-wasm`; npm reserves plain `domq`), the
|
|
15
|
+
PyPI package and Python import `domq` (was `dom-tree-rs` / `domtree`), and
|
|
16
|
+
the Homebrew formula `iamsaquib8/tap/domq`. The repository moves to
|
|
17
|
+
`github.com/iamsaquib8/domq`; the old packages get a final release pointing
|
|
18
|
+
here.
|
|
19
|
+
- **MSRV raised to Rust 1.88** (from 1.85).
|
|
20
|
+
- **Python 3.11+** (3.9 and 3.10 are end of life); wheels are abi3 for
|
|
21
|
+
CPython ≥ 3.11.
|
|
22
|
+
- `NodeRef::to_text()` / `to_markdown()` called *on* a hidden element
|
|
23
|
+
(`<title>`, `<script>`, `<style>`, `<template>`, …) now return its content
|
|
24
|
+
instead of `""`; hiding applies only to descendants. `to_text` gives a
|
|
25
|
+
`<script>`/`<style>` source verbatim. Document-level output is unchanged.
|
|
26
|
+
- CI and release builds use the current Node LTS; the derive crate uses
|
|
27
|
+
`syn` 3.
|
|
28
|
+
|
|
29
|
+
### Added
|
|
30
|
+
- **`#[derive(FromHtml)]`** (new `derive` feature, companion crate
|
|
31
|
+
`domq-derive`): declarative extraction into structs with
|
|
32
|
+
`#[html(select, attr, html, outer_html, markdown, default, with, skip)]`,
|
|
33
|
+
`Option<T>`, `Vec<T>`, nested and generic structs, a struct-level
|
|
34
|
+
`#[html(select = "…")]`, and compile-time diagnostics for misused
|
|
35
|
+
attributes.
|
|
36
|
+
- `Dom::extract`, `Dom::extract_all`, `NodeRef::extract`, the
|
|
37
|
+
`FromHtml` / `FromHtmlValue` traits, `Scope`, and `ExtractError`, whose
|
|
38
|
+
messages name the field path, selector and source:
|
|
39
|
+
`Product.reviews[2].stars (".stars" @data-value): cannot parse "x" as u8: …`.
|
|
40
|
+
Zero-dependency and `no_std`; hand-written impls need no feature.
|
|
41
|
+
- **Python `domq.extract` / `extract_all`**: the same declarative extraction
|
|
42
|
+
into dataclasses with `Annotated[T, Select(...)]` (`Attr`, `Html`,
|
|
43
|
+
`OuterHtml`, `Markdown` shorthands), `Optional`, `list`, nested dataclasses,
|
|
44
|
+
`default=`, `with_=`, a class-level `__select__`, and `ExtractError` with the
|
|
45
|
+
same paths and messages as Rust.
|
|
46
|
+
|
|
47
|
+
## [0.3.0] — 2026-10-21
|
|
48
|
+
|
|
49
|
+
"One-line scraping": find things and get their content out, in Rust, in the
|
|
50
|
+
shell, in JavaScript and in Python. This release covers what the roadmap had
|
|
51
|
+
planned as phases 0.3 to 0.7, plus distribution (npm, PyPI, Homebrew, release
|
|
52
|
+
binaries) and the fixes from two independent reviews.
|
|
53
|
+
|
|
54
|
+
### Added
|
|
55
|
+
- **CSS selector engine v2** (`query` feature, now **on by default**):
|
|
56
|
+
`+`/`~` combinators; every attribute operator (`^=` `$=` `*=` `~=` `|=`) with
|
|
57
|
+
the `i`/`s` flags; `:first-child`, `:last-child`, `:only-child`,
|
|
58
|
+
`:nth-child(An+B [of S])`, `:nth-last-child`, the `*-of-type` family,
|
|
59
|
+
`:empty`, `:root`, `:scope`, `:not()`, `:is()`, `:where()`, `:has()` (with
|
|
60
|
+
relative selectors), `:checked`/`:disabled`/`:enabled`, CSS escapes, and the
|
|
61
|
+
non-standard `:contains("text")`. Matching never backtracks exponentially and
|
|
62
|
+
never recurses on document depth.
|
|
63
|
+
- `Selector::parse` (and `FromStr`) — compile once, reuse, and get a
|
|
64
|
+
`SelectorError` with a byte position instead of silently matching nothing.
|
|
65
|
+
`Dom::select` accepts `&str`, `&String`, or `&Selector`.
|
|
66
|
+
- Scoped queries: `Dom::select_first`, `NodeRef::select`, `select_first`,
|
|
67
|
+
`matches`, `closest`.
|
|
68
|
+
- `to_text()` on `Dom`/`NodeRef` — readable plain text (block elements become
|
|
69
|
+
line breaks, whitespace collapsed, `<script>`/`<style>`/`<head>` dropped).
|
|
70
|
+
- `to_markdown()` (new `markdown` feature, on by default) — zero-dependency HTML
|
|
71
|
+
→ CommonMark + GFM: headings, emphasis, links, images, nested lists, code
|
|
72
|
+
blocks with languages, blockquotes, tables.
|
|
73
|
+
- `NodeRef::inner_html`, `outer_html`, `next_element_sibling`,
|
|
74
|
+
`prev_element_sibling`.
|
|
75
|
+
- Editing: `Dom::remove` (detach a subtree), `Dom::set_attr`, `Dom::remove_attr`.
|
|
76
|
+
- **`domtree` CLI** (`cargo install domtree-cli`) — htmlq-compatible flags
|
|
77
|
+
(`-a`, `-t`, `-p`, `-w`, `-r`, `-b`, `-B`, `-f`, `-o`) plus `--markdown`,
|
|
78
|
+
`--json`, `--count`, `--first`; zero dependencies. Prebuilt binaries are
|
|
79
|
+
attached to GitHub releases.
|
|
80
|
+
|
|
81
|
+
- **`dom-tree-wasm` 0.3.0** (npm): `parse` now returns a typed object
|
|
82
|
+
(`parseJson` keeps the string); new `select`, `selectFirst`, `toText`,
|
|
83
|
+
`toMarkdown`, `toHtml`, `errors`, and a parse-once `Document` class. Invalid
|
|
84
|
+
selectors throw a `SelectorError` with a position. Full TypeScript types,
|
|
85
|
+
works in Node with no setup, ~41 KB gzipped. **Breaking:** `parseErrorCount`
|
|
86
|
+
and `query` are removed, and `dom-node.d.ts` is replaced by the generated
|
|
87
|
+
types.
|
|
88
|
+
- **Live playground** on GitHub Pages: paste HTML, try selectors, and see the
|
|
89
|
+
tree, text, Markdown, JSON and repaired HTML, all running locally in wasm.
|
|
90
|
+
- `CODE_OF_CONDUCT.md`, `SECURITY.md`, `docs/ARCHITECTURE.md`, issue and PR
|
|
91
|
+
templates.
|
|
92
|
+
- **Streaming events:** `domtree::events(html)` yields `Start` / `End`
|
|
93
|
+
(including implied ends) / `Text` / `Comment` / `Doctype`, each with its byte
|
|
94
|
+
range, applying exactly the same recovery as `parse` but without building a
|
|
95
|
+
tree; `Events::errors()` matches `Dom::errors()`.
|
|
96
|
+
- **`simd` feature** (off by default): `memchr`-backed scanning, identical
|
|
97
|
+
output, ~24% faster on text-heavy input. The default build stays
|
|
98
|
+
zero-dependency.
|
|
99
|
+
- Benchmarks against `scraper` and `html5ever`, text- and table-heavy inputs,
|
|
100
|
+
a memory report, and a weekly `Benchmarks` workflow; a `differential` fuzz
|
|
101
|
+
target against html5ever.
|
|
102
|
+
- **Editing, part 2:** `Dom::set_text`, `rename`, `create_element`,
|
|
103
|
+
`create_text`, `append_child`, `insert_before`, `wrap`, `unwrap`, and
|
|
104
|
+
parsed-fragment insertion (`append_html`, `prepend_html`,
|
|
105
|
+
`insert_html_before`, `insert_html_after`, returning the inserted ids). Moves
|
|
106
|
+
never create cycles; `nodes()`, `select` and `find_by_*` stay in document
|
|
107
|
+
order after any edit. `NodeRef::is_detached()`.
|
|
108
|
+
- **Source spans:** `NodeRef::source_range()` (byte range in the input) and
|
|
109
|
+
`domtree::line_col(src, byte)`.
|
|
110
|
+
- **`entities-full` feature:** all 2,125 WHATWG named character references
|
|
111
|
+
(off by default; the default table has the 178 most common).
|
|
112
|
+
- `Dom::main_content()` — a deterministic "reader mode" heuristic (semantic
|
|
113
|
+
markup first, then Readability-style paragraph scoring) that picks the
|
|
114
|
+
article and drops nav, sidebars, comments, ads and footers.
|
|
115
|
+
- `Dom::to_markdown_chunks(max_chars)` / `NodeRef::to_markdown_chunks` —
|
|
116
|
+
heading-aware Markdown chunks for RAG (`Chunk { heading_path, markdown }`,
|
|
117
|
+
`Chunk::context()`), never splitting inside code blocks or tables unless a
|
|
118
|
+
single block exceeds the budget.
|
|
119
|
+
- `Dom::links(base)` (readable link text, resolved URLs, `<base href>`,
|
|
120
|
+
`javascript:` skipped), `Dom::resources(base)` (every embedded URL incl.
|
|
121
|
+
`srcset` candidates), `Dom::base_url(base)`.
|
|
122
|
+
- `Dom::metadata()` — title, description, canonical, lang, site name, image,
|
|
123
|
+
author, published date, keywords, OpenGraph and Twitter fields, raw JSON-LD
|
|
124
|
+
and feed links (`serde::Serialize` under `serde`).
|
|
125
|
+
- `NodeRef::table_rows()` (colspan/rowspan expanded, bounded on hostile input)
|
|
126
|
+
and `domtree::to_csv` (RFC 4180).
|
|
127
|
+
- `domtree::resolve_url` / `resolve_srcset` — zero-dependency RFC 3986 and
|
|
128
|
+
spec-compliant `srcset` resolution, moved into the library from the CLI.
|
|
129
|
+
- **Python package** `dom-tree-rs` on PyPI (import `domtree`): `parse`,
|
|
130
|
+
`Document.select`/`select_first`/`to_text`/`to_markdown`/`to_html`/`to_json`,
|
|
131
|
+
`Element` with attributes, text, HTML and traversal; typed stubs; abi3 wheels
|
|
132
|
+
for Linux, macOS and Windows; the GIL is released while parsing large inputs.
|
|
133
|
+
- **Homebrew formula** for the `domtree` CLI (`packaging/homebrew/`), filled in
|
|
134
|
+
with release checksums by the release workflow.
|
|
135
|
+
- **WASI build** of the CLI (`domtree.wasm`, attached to releases, smoke-tested
|
|
136
|
+
under wasmtime), including quiet exit on a closed stdout pipe.
|
|
137
|
+
- **CLI structured extraction:** `domtree -e NAME=SELECTOR[@attr][[]]` builds a
|
|
138
|
+
JSON object of named fields, or one object per match of a scope selector
|
|
139
|
+
(JSON array, or `--jsonl`).
|
|
140
|
+
- **CLI output modes:** `--jsonl` (JSON Lines), `--csv` (tables as RFC 4180
|
|
141
|
+
CSV), `--main` (reader-mode narrowing before selecting), `--links`
|
|
142
|
+
(url/text, TSV or JSON), `--meta` (page metadata as JSON).
|
|
143
|
+
- **`domtree lint`:** images without `alt`, empty links and buttons, duplicate
|
|
144
|
+
ids, skipped heading levels, and every recovered parse error, as
|
|
145
|
+
`file:line:col: level: message [rule]` or `--format json`; exit 0/1/2 for CI.
|
|
146
|
+
Omitted optional end tags are notes, missing required ones warnings.
|
|
147
|
+
- **CLI encoding detection:** BOM, `<meta charset>`, UTF-8, then Windows-1252;
|
|
148
|
+
`--encoding` override.
|
|
149
|
+
- Prebuilt CLI binaries for Linux arm64.
|
|
150
|
+
|
|
151
|
+
### Changed
|
|
152
|
+
- Top-level nodes are now linked as siblings (`next_sibling`/`prev_sibling`),
|
|
153
|
+
so sibling combinators and `:first-child` work on multi-root fragments.
|
|
154
|
+
- Numeric character references in `0x80–0x9F` are remapped per the WHATWG
|
|
155
|
+
Windows-1252 table (`–` → `–`), as browsers do; `�`, surrogates and
|
|
156
|
+
out-of-range values become U+FFFD.
|
|
157
|
+
- Default features are now `json`, `query`, `markdown` (still zero
|
|
158
|
+
dependencies). Use `default-features = false` for the smallest build.
|
|
159
|
+
- **Faster parsing:** one attribute arena per document (nodes are now 52
|
|
160
|
+
bytes), no per-tag allocations, and a word-at-a-time byte search. Parse time
|
|
161
|
+
is 25–37% lower on markup and 69% lower on text-heavy pages; a parse makes
|
|
162
|
+
~20 allocations instead of thousands; peak memory is 10–14% lower, and unused
|
|
163
|
+
arena capacity is released after parsing.
|
|
164
|
+
- More implicit-close rules (WHATWG optional tags): the full list of elements
|
|
165
|
+
that close `<p>`; a heading closes an open heading; an unclosed `<head>` ends
|
|
166
|
+
at the first body content (previously such pages lost their body in
|
|
167
|
+
`to_text`); ruby `rb`/`rt`/`rtc`/`rp`; `<hr>` closes an open
|
|
168
|
+
`option`/`optgroup`; captions, `colgroup` and `col` close open cells, rows
|
|
169
|
+
and row groups.
|
|
170
|
+
- Nodes are 64 bytes (were 80); `NodeId` is non-zero, so `Option<NodeId>` is
|
|
171
|
+
4 bytes.
|
|
172
|
+
|
|
173
|
+
- CLI: undeclared non-UTF-8 input now decodes as Windows-1252 (as browsers
|
|
174
|
+
do) instead of lossy UTF-8, e.g. `caf\xe9` → `café`.
|
|
175
|
+
|
|
176
|
+
### Security / hardening
|
|
177
|
+
- Edited content can no longer inject markup on output: text set inside
|
|
178
|
+
`<script>`, `<style>`, `<textarea>` or `<title>` cannot close the element
|
|
179
|
+
early (a `</script` is written `<\/script`; parsed documents serialize
|
|
180
|
+
byte-for-byte as before), and `set_attr`, `rename`, `create_element` and
|
|
181
|
+
`wrap` refuse names that are empty or contain whitespace, quotes, `<`, `>`,
|
|
182
|
+
`/`, `=` or NUL (tag names must start with an ASCII letter).
|
|
183
|
+
`create_element` now returns `Option<NodeId>`.
|
|
184
|
+
- Attacker-chosen tag names could make `parse`/`events` quadratic (40,000
|
|
185
|
+
colliding names: 6.4 s → 2 ms). An end tag now looks for its element among
|
|
186
|
+
the 256 innermost and 8 outermost open elements; an element open only in
|
|
187
|
+
between is left open and the end tag is reported as `StrayEndTag`.
|
|
188
|
+
- `resolve_url` reads `\` as `/` in http(s)/ftp/ws(s)/file URLs, as browsers
|
|
189
|
+
do (`\\evil.com\x` resolves to evil.com, not a relative path), and
|
|
190
|
+
`base_url` ignores a `<base>` inside `<template>`, `<svg>` or `<math>`.
|
|
191
|
+
|
|
192
|
+
### Fixed
|
|
193
|
+
- Linear-time fixes found in review: scoped selector queries
|
|
194
|
+
(`tr.select(…)`, `matches`, `closest`) no longer pay for the whole document
|
|
195
|
+
per call (160k rows: 18 s → 70 ms) and selector memo memory is bounded;
|
|
196
|
+
`append_html` no longer slows with the parent's depth (3.4 s → 1.4 ms);
|
|
197
|
+
chunk heading paths are capped at 256 characters per title; `table_rows` is
|
|
198
|
+
linear on nested tables (a cell's text no longer includes tables nested in
|
|
199
|
+
it — **behaviour change**); CLI `--links SELECTOR` and nested `-e` scopes
|
|
200
|
+
are linear.
|
|
201
|
+
- `UnknownEntity` only reports WHATWG's 106 legacy names when their `;` is
|
|
202
|
+
missing (so `x&order=1`, `?a=1&or` are clean), and its position points at
|
|
203
|
+
the `&`, also inside attribute values.
|
|
204
|
+
- `colspan`/`rowspan` accept a leading `+` and skip only ASCII whitespace;
|
|
205
|
+
`Link::text` leaves out content hidden with `hidden` or inline
|
|
206
|
+
`display:none` / `visibility:hidden`.
|
|
207
|
+
- CLI: `lint` no longer reports correctly closed elements as unclosed
|
|
208
|
+
(`</span foo>`, a final `</span` without `>`), and does report an outer
|
|
209
|
+
element whose same-named child ends at the same spot; `-B` resolves
|
|
210
|
+
`<base href>` against `--base` once; combining two output formats is a
|
|
211
|
+
usage error; `--links -w` skips links without text.
|
|
212
|
+
- `Dom::errors()` no longer flags valid HTML: a bare `&` (`a & b`,
|
|
213
|
+
`?a=1&b=2`) isn't an `UnknownEntity` (reference-shaped `&…` still is, with
|
|
214
|
+
WHATWG's attribute-value exception), and an end tag closing elements whose
|
|
215
|
+
end tag is optional (`<ul><li>a</ul>`) is `ImplicitlyClosed`, not
|
|
216
|
+
`MismatchedEndTag`.
|
|
217
|
+
- Quadratic parsing of many unmatched end tags over a deep open-element stack,
|
|
218
|
+
and quadratic entity decoding when the lookahead ended inside a multi-byte
|
|
219
|
+
character.
|
|
220
|
+
- CLI `--base` keeps commas inside `srcset` URLs, drops empty candidates, and
|
|
221
|
+
rewrites the first of a repeated attribute (the one that counts).
|
|
222
|
+
|
|
223
|
+
## [0.2.1] — 2026-06-24
|
|
224
|
+
|
|
225
|
+
### Added
|
|
226
|
+
- Zero-dependency HTML re-serialization (`Dom::to_html`, `NodeRef::to_html`) —
|
|
227
|
+
round-trips a parsed tree back to HTML with correct escaping and void/raw-text
|
|
228
|
+
handling. Always available (no feature required).
|
|
229
|
+
- `#![forbid(unsafe_code)]` — the crate contains no `unsafe`.
|
|
230
|
+
|
|
231
|
+
### Changed
|
|
232
|
+
- Moved to **edition 2024**; minimum supported Rust is now **1.85**
|
|
233
|
+
(developed and tested on the latest stable).
|
|
234
|
+
|
|
235
|
+
## [0.2.0] — 2026-06-23
|
|
236
|
+
|
|
237
|
+
A full rewrite. The 0.1 line was a learning-project parser that panicked on
|
|
238
|
+
almost any real-world HTML; 0.2 is a robust, tiny, zero-dependency library.
|
|
239
|
+
|
|
240
|
+
### Added
|
|
241
|
+
- Error-tolerant parser that **never panics**; recovery diagnostics via
|
|
242
|
+
`Dom::errors()`.
|
|
243
|
+
- Streaming, allocation-light `Tokenizer` (`domtree::tokenize`) implementing
|
|
244
|
+
`Iterator<Item = Token>`.
|
|
245
|
+
- Arena-based tree with O(1) `parent`/`children`/`descendants`/`ancestors`/
|
|
246
|
+
sibling traversal through a `Copy` `NodeRef` handle.
|
|
247
|
+
- `find_by_tag` / `find_by_id` / `find_by_class` helpers.
|
|
248
|
+
- Built-in, zero-dependency JSON serialization (`Dom::to_json`, `NodeRef::to_json`)
|
|
249
|
+
behind the default `json` feature.
|
|
250
|
+
- Optional `serde::Serialize` (`serde` feature) producing the identical JSON shape.
|
|
251
|
+
- Optional CSS-subset selector engine (`Dom::select`, `query` feature):
|
|
252
|
+
type/`*`/`#id`/`.class`/`[attr]`/`[attr=value]`, descendant and child
|
|
253
|
+
combinators, selector lists.
|
|
254
|
+
- `no_std` + `alloc` support; new `std` feature for std-only conveniences.
|
|
255
|
+
- Coverage for void elements, self-closing tags, comments, doctype, CDATA,
|
|
256
|
+
raw-text `<script>`/`<style>`, escapable raw text, boolean/unquoted attributes,
|
|
257
|
+
hyphenated/namespaced/custom tags, HTML entities (named + numeric), and
|
|
258
|
+
implicit closing for the common cases.
|
|
259
|
+
- Rebuilt WebAssembly bindings (`dom-tree-wasm`): ~19 KB gzipped, zero JS deps.
|
|
260
|
+
- `cargo-fuzz` harness, criterion benchmarks, and a full GitHub Actions CI.
|
|
261
|
+
|
|
262
|
+
### Changed
|
|
263
|
+
- **Breaking:** the public API is redesigned. The crate is now used as `domtree`;
|
|
264
|
+
`parse` returns an owned `Dom`; `NodeRef::kind()` returns a borrowed `Copy`
|
|
265
|
+
`NodeKind<'a>`. The old owned `Node`/`Props`/`NodeData` types are gone.
|
|
266
|
+
- `serde`/`serde_json` are no longer dependencies of the default build.
|
|
267
|
+
|
|
268
|
+
### Performance
|
|
269
|
+
- ~250 MiB/s on a synthetic article workload — within ~5–15 % of `tl` and
|
|
270
|
+
roughly on par on large documents — while remaining zero-dependency.
|
|
271
|
+
|
|
272
|
+
## [0.1.1] — 2020
|
|
273
|
+
- Initial learning-project parser (HTML → debug tree / `serde_json` string).
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Code of conduct
|
|
2
|
+
|
|
3
|
+
This project follows the
|
|
4
|
+
[Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/).
|
|
5
|
+
|
|
6
|
+
In short: be welcoming and patient, assume good intent, and critique code and
|
|
7
|
+
ideas rather than people. Harassment, insults, and personal or political
|
|
8
|
+
attacks are not acceptable in issues, pull requests, discussions, or any other
|
|
9
|
+
project space.
|
|
10
|
+
|
|
11
|
+
## Reporting
|
|
12
|
+
|
|
13
|
+
If someone's behaviour makes you uncomfortable, contact the maintainer
|
|
14
|
+
privately at the email address listed in `Cargo.toml` (`authors`). Reports are
|
|
15
|
+
handled confidentially. The maintainer may remove, edit, or reject comments,
|
|
16
|
+
commits, issues, and other contributions that break this code of conduct, and
|
|
17
|
+
may temporarily or permanently ban a contributor for repeated or serious
|
|
18
|
+
violations, following the Contributor Covenant's enforcement guidelines.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Thanks for your interest! This is a small, focused crate — contributions that
|
|
4
|
+
keep it small and focused are very welcome.
|
|
5
|
+
|
|
6
|
+
## Two load-bearing principles
|
|
7
|
+
|
|
8
|
+
1. **Never panic on input.** `parse` must always return a best-effort tree for
|
|
9
|
+
*any* byte sequence. No `unwrap`/`expect`/`panic`/indexing on input-derived
|
|
10
|
+
data — the lib denies those clippy lints. Use `get(..)` and iterators. New
|
|
11
|
+
code should be exercised by the proptest suite and ideally the fuzz harness.
|
|
12
|
+
2. **Stay tiny and zero-dependency by default.** The default build has no
|
|
13
|
+
external runtime dependencies. New dependencies must be optional, behind a
|
|
14
|
+
feature, and justified. Prefer hand-rolled, `no_std`-friendly code.
|
|
15
|
+
|
|
16
|
+
## Scope
|
|
17
|
+
|
|
18
|
+
This is intentionally a *forgiving* parser, not a WHATWG-compliant one. Proposals
|
|
19
|
+
to add the full tree-construction algorithm (adoption agency, foster parenting,
|
|
20
|
+
auto `<html>`/`<head>`/`<body>`) are out of scope — that's what `html5ever` is
|
|
21
|
+
for. Bug fixes, more entity coverage, more implicit-close cases, performance, and
|
|
22
|
+
ergonomics are all in scope.
|
|
23
|
+
|
|
24
|
+
## Development
|
|
25
|
+
|
|
26
|
+
```sh
|
|
27
|
+
cargo test --all-features # unit + corpus snapshots + proptest + doctests
|
|
28
|
+
cargo clippy --all-features --all-targets -- -D warnings
|
|
29
|
+
cargo fmt --all
|
|
30
|
+
|
|
31
|
+
# #[derive(FromHtml)]: end-to-end tests, then the macro's compile-fail tests
|
|
32
|
+
# (after changing a diagnostic: TRYBUILD=overwrite cargo test -p domq-derive)
|
|
33
|
+
cargo test --features derive
|
|
34
|
+
cargo test -p domq-derive
|
|
35
|
+
|
|
36
|
+
# CLI (workspace member)
|
|
37
|
+
cargo test -p domq-cli
|
|
38
|
+
cargo run -p domq-cli -- --help
|
|
39
|
+
|
|
40
|
+
# WASM
|
|
41
|
+
cd wasm && wasm-pack build --target web --release && wasm-pack test --node
|
|
42
|
+
|
|
43
|
+
# Playground (after building the wasm into playground/pkg — see playground/README.md)
|
|
44
|
+
python3 -m http.server -d playground 8000
|
|
45
|
+
|
|
46
|
+
# Benchmarks (see benchmarks/README.md); the memory report; the simd feature
|
|
47
|
+
cd benchmarks && cargo bench
|
|
48
|
+
cd benchmarks && cargo run --release --example memory
|
|
49
|
+
cargo test --features simd
|
|
50
|
+
|
|
51
|
+
# Fuzzing (nightly)
|
|
52
|
+
cargo +nightly fuzz run parse
|
|
53
|
+
cargo +nightly fuzz run differential # our tree vs html5ever on the claimed subset
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Snapshot tests use [`insta`]: after an intentional output change, run
|
|
57
|
+
`INSTA_UPDATE=always cargo test --test corpus` and review the diff before
|
|
58
|
+
committing the updated `.snap` files.
|
|
59
|
+
|
|
60
|
+
[`insta`]: https://insta.rs/
|
|
61
|
+
|
|
62
|
+
## Before opening a PR
|
|
63
|
+
|
|
64
|
+
- `cargo fmt`, `cargo clippy` clean, tests pass.
|
|
65
|
+
- Add a test for the behaviour you changed (a corpus fixture is great for
|
|
66
|
+
parsing changes).
|
|
67
|
+
- Keep the MSRV at 1.88 (edition 2024); the CI `msrv` job enforces it. Raise it only in a
|
|
68
|
+
minor release, and never to a Rust release less than about six months old.
|
|
69
|
+
- Add an entry to `CHANGELOG.md` for anything user-visible.
|
domq-0.4.0/Cargo.toml
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "domq"
|
|
3
|
+
version = "0.4.0"
|
|
4
|
+
edition = "2024"
|
|
5
|
+
rust-version = "1.88"
|
|
6
|
+
authors = ["iamsaquib8 <saquibulhassan6@gmail.com>"]
|
|
7
|
+
description = "Tiny, zero-dependency, forgiving HTML parser: turn messy real-world HTML into a clean DOM tree (and JSON). WASM-first, no_std-friendly."
|
|
8
|
+
repository = "https://github.com/iamsaquib8/domq"
|
|
9
|
+
homepage = "https://github.com/iamsaquib8/domq"
|
|
10
|
+
documentation = "https://docs.rs/domq"
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
license = "MIT"
|
|
13
|
+
keywords = ["html", "parser", "css-selector", "markdown", "wasm"]
|
|
14
|
+
categories = ["parser-implementations", "wasm", "web-programming", "no-std"]
|
|
15
|
+
exclude = ["/.github", "/benchmarks", "/fuzz", "/wasm", "/cli", "/derive", "/python", "/packaging", "/playground", "/docs/launch"]
|
|
16
|
+
|
|
17
|
+
[lib]
|
|
18
|
+
name = "domq"
|
|
19
|
+
|
|
20
|
+
[features]
|
|
21
|
+
default = ["json", "query", "markdown"]
|
|
22
|
+
# Built-in, zero-dependency JSON serializer (`Dom::to_json`).
|
|
23
|
+
json = []
|
|
24
|
+
# Optional `serde::Serialize` impls on the tree (pulls in `serde`).
|
|
25
|
+
serde = ["dep:serde"]
|
|
26
|
+
# Hand-rolled CSS selector engine (`Dom::select`, `Selector`). Zero-dependency.
|
|
27
|
+
query = []
|
|
28
|
+
# Zero-dependency HTML -> Markdown (`Dom::to_markdown`).
|
|
29
|
+
markdown = []
|
|
30
|
+
# The complete WHATWG named character reference table (2,125 names) instead of
|
|
31
|
+
# the curated ~180. Off by default to keep the default build small.
|
|
32
|
+
entities-full = []
|
|
33
|
+
# Enables std-only conveniences (e.g. `std::error::Error` for `ParseError`).
|
|
34
|
+
# The crate is `no_std + alloc` by default.
|
|
35
|
+
std = []
|
|
36
|
+
# SIMD-accelerated scanning of text and attribute values via `memchr` (adds
|
|
37
|
+
# that one dependency, `no_std`). Off by default: the default build stays
|
|
38
|
+
# zero-dependency and produces exactly the same tokens.
|
|
39
|
+
simd = ["dep:memchr"]
|
|
40
|
+
# `#[derive(FromHtml)]`: declarative extraction into structs. Re-exports the
|
|
41
|
+
# proc-macro from the `domq-derive` companion crate (which pulls in `syn`,
|
|
42
|
+
# `quote` and `proc-macro2` at compile time only). The traits it implements
|
|
43
|
+
# (`FromHtml`, `Dom::extract`, …) come with `query` and need no extra crates.
|
|
44
|
+
derive = ["query", "dep:domq-derive"]
|
|
45
|
+
|
|
46
|
+
[[example]]
|
|
47
|
+
name = "to_markdown"
|
|
48
|
+
required-features = ["query", "markdown"]
|
|
49
|
+
|
|
50
|
+
[[test]]
|
|
51
|
+
name = "extract"
|
|
52
|
+
required-features = ["derive", "markdown"]
|
|
53
|
+
|
|
54
|
+
[[test]]
|
|
55
|
+
name = "extract_derive_properties"
|
|
56
|
+
required-features = ["derive"]
|
|
57
|
+
|
|
58
|
+
[dependencies]
|
|
59
|
+
serde = { version = "1", default-features = false, features = ["alloc"], optional = true }
|
|
60
|
+
memchr = { version = "2.7", default-features = false, optional = true }
|
|
61
|
+
domq-derive = { path = "derive", version = "0.4.0", optional = true }
|
|
62
|
+
|
|
63
|
+
[dev-dependencies]
|
|
64
|
+
# Snapshot testing for the corpus suite (dev-only; not a runtime dependency).
|
|
65
|
+
insta = "1"
|
|
66
|
+
# Property tests: "never panics", arena-invariant checks.
|
|
67
|
+
proptest = "1"
|
|
68
|
+
# Drives the `serde` Serialize impl in the JSON-equality roundtrip test.
|
|
69
|
+
serde_json = "1"
|
|
70
|
+
|
|
71
|
+
[package.metadata.docs.rs]
|
|
72
|
+
# Document all features (incl. query + serde) on docs.rs.
|
|
73
|
+
all-features = true
|
|
74
|
+
rustdoc-args = ["--cfg", "docsrs"]
|
|
75
|
+
|
|
76
|
+
[profile.release]
|
|
77
|
+
opt-level = 3
|
|
78
|
+
lto = true
|
|
79
|
+
|
|
80
|
+
[workspace]
|
|
81
|
+
resolver = "2"
|
|
82
|
+
# The `domq` command-line tool and the `#[derive(FromHtml)]` proc-macro
|
|
83
|
+
# crate. They share this crate's lockfile and target dir.
|
|
84
|
+
members = ["cli", "derive"]
|
|
85
|
+
# The WASM bindings, the Python bindings, the comparison benchmarks, and the
|
|
86
|
+
# fuzz harness are independent crates with their own (heavier) dependency
|
|
87
|
+
# graphs and release profiles, so they are kept out of this crate's pristine,
|
|
88
|
+
# minimal-dep graph.
|
|
89
|
+
exclude = ["wasm", "python", "benchmarks", "fuzz"]
|
domq-0.4.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 iamsaquib8
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
domq-0.4.0/LICENSE-MIT
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 iamsaquib8
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|