domq 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. domq-0.4.0/.gitignore +12 -0
  2. domq-0.4.0/CHANGELOG.md +273 -0
  3. domq-0.4.0/CODE_OF_CONDUCT.md +18 -0
  4. domq-0.4.0/CONTRIBUTING.md +69 -0
  5. domq-0.4.0/Cargo.toml +89 -0
  6. domq-0.4.0/LICENSE +21 -0
  7. domq-0.4.0/LICENSE-MIT +21 -0
  8. domq-0.4.0/PKG-INFO +310 -0
  9. domq-0.4.0/README.md +459 -0
  10. domq-0.4.0/ROADMAP.md +392 -0
  11. domq-0.4.0/SECURITY.md +45 -0
  12. domq-0.4.0/docs/ARCHITECTURE.md +195 -0
  13. domq-0.4.0/docs/PUBLISHING.md +83 -0
  14. domq-0.4.0/docs/assets/banner-dark.svg +51 -0
  15. domq-0.4.0/docs/assets/banner-light.svg +51 -0
  16. domq-0.4.0/docs/assets/demo.svg +32 -0
  17. domq-0.4.0/docs/assets/logo.svg +29 -0
  18. domq-0.4.0/docs/assets/social-preview.png +0 -0
  19. domq-0.4.0/docs/assets/social-preview.svg +51 -0
  20. domq-0.4.0/domq/__init__.py +38 -0
  21. domq-0.4.0/domq/__init__.pyi +279 -0
  22. domq-0.4.0/domq/_extract.py +633 -0
  23. domq-0.4.0/domq/py.typed +0 -0
  24. domq-0.4.0/examples/parse.rs +37 -0
  25. domq-0.4.0/examples/query.rs +40 -0
  26. domq-0.4.0/examples/to_markdown.rs +34 -0
  27. domq-0.4.0/pyproject.toml +54 -0
  28. domq-0.4.0/python/.gitignore +8 -0
  29. domq-0.4.0/python/Cargo.lock +137 -0
  30. domq-0.4.0/python/Cargo.toml +37 -0
  31. domq-0.4.0/python/LICENSE-MIT +21 -0
  32. domq-0.4.0/python/README.md +282 -0
  33. domq-0.4.0/python/src/lib.rs +364 -0
  34. domq-0.4.0/python/tests/test_domq.py +382 -0
  35. domq-0.4.0/python/tests/test_extract.py +1197 -0
  36. domq-0.4.0/scripts/gen_entities.py +76 -0
  37. domq-0.4.0/src/builder.rs +489 -0
  38. domq-0.4.0/src/edit.rs +770 -0
  39. domq-0.4.0/src/entities.rs +502 -0
  40. domq-0.4.0/src/entities_full.rs +2137 -0
  41. domq-0.4.0/src/error.rs +95 -0
  42. domq-0.4.0/src/events.rs +204 -0
  43. domq-0.4.0/src/extract/links.rs +380 -0
  44. domq-0.4.0/src/extract/meta.rs +388 -0
  45. domq-0.4.0/src/extract/mod.rs +52 -0
  46. domq-0.4.0/src/extract/readable.rs +537 -0
  47. domq-0.4.0/src/extract/table.rs +429 -0
  48. domq-0.4.0/src/from_html.rs +1000 -0
  49. domq-0.4.0/src/lib.rs +449 -0
  50. domq-0.4.0/src/node.rs +203 -0
  51. domq-0.4.0/src/query.rs +1537 -0
  52. domq-0.4.0/src/scan.rs +173 -0
  53. domq-0.4.0/src/serialize/chunk.rs +564 -0
  54. domq-0.4.0/src/serialize/html.rs +250 -0
  55. domq-0.4.0/src/serialize/json.rs +166 -0
  56. domq-0.4.0/src/serialize/markdown.rs +1030 -0
  57. domq-0.4.0/src/serialize/mod.rs +101 -0
  58. domq-0.4.0/src/serialize/serde_impl.rs +82 -0
  59. domq-0.4.0/src/serialize/text.rs +256 -0
  60. domq-0.4.0/src/source.rs +45 -0
  61. domq-0.4.0/src/tags.rs +241 -0
  62. domq-0.4.0/src/token.rs +57 -0
  63. domq-0.4.0/src/tokenizer.rs +406 -0
  64. domq-0.4.0/src/tree.rs +742 -0
  65. domq-0.4.0/src/url.rs +348 -0
  66. domq-0.4.0/tests/chunks.rs +354 -0
  67. domq-0.4.0/tests/corpus/basic.html +4 -0
  68. domq-0.4.0/tests/corpus/document.html +15 -0
  69. domq-0.4.0/tests/corpus/messy.html +7 -0
  70. domq-0.4.0/tests/corpus/omitted.html +26 -0
  71. domq-0.4.0/tests/corpus/ruby.html +3 -0
  72. domq-0.4.0/tests/corpus/select.html +12 -0
  73. domq-0.4.0/tests/corpus/tables.html +16 -0
  74. domq-0.4.0/tests/corpus.rs +36 -0
  75. domq-0.4.0/tests/editing.rs +928 -0
  76. domq-0.4.0/tests/editing_properties.proptest-regressions +7 -0
  77. domq-0.4.0/tests/editing_properties.rs +465 -0
  78. domq-0.4.0/tests/entities.rs +170 -0
  79. domq-0.4.0/tests/events.rs +559 -0
  80. domq-0.4.0/tests/extract.rs +1039 -0
  81. domq-0.4.0/tests/extract_derive_properties.rs +187 -0
  82. domq-0.4.0/tests/extract_properties.rs +331 -0
  83. domq-0.4.0/tests/html_output.rs +133 -0
  84. domq-0.4.0/tests/json_output.rs +46 -0
  85. domq-0.4.0/tests/links.rs +371 -0
  86. domq-0.4.0/tests/main_content.rs +367 -0
  87. domq-0.4.0/tests/markdown_output.rs +579 -0
  88. domq-0.4.0/tests/metadata.rs +380 -0
  89. domq-0.4.0/tests/output_properties.rs +259 -0
  90. domq-0.4.0/tests/pages/blog.html +100 -0
  91. domq-0.4.0/tests/pages/docs.html +61 -0
  92. domq-0.4.0/tests/pages/messy.html +24 -0
  93. domq-0.4.0/tests/pages/reader/blog-comments.html +179 -0
  94. domq-0.4.0/tests/pages/reader/docs.html +95 -0
  95. domq-0.4.0/tests/pages/reader/garbage.html +6 -0
  96. domq-0.4.0/tests/pages/reader/news.html +100 -0
  97. domq-0.4.0/tests/pages/reader/paragraphs.html +10 -0
  98. domq-0.4.0/tests/pages/reader/product.html +95 -0
  99. domq-0.4.0/tests/parsing.rs +748 -0
  100. domq-0.4.0/tests/properties.rs +93 -0
  101. domq-0.4.0/tests/query.rs +776 -0
  102. domq-0.4.0/tests/query_memory.rs +109 -0
  103. domq-0.4.0/tests/query_properties.proptest-regressions +23 -0
  104. domq-0.4.0/tests/query_properties.rs +458 -0
  105. domq-0.4.0/tests/serde_roundtrip.rs +27 -0
  106. domq-0.4.0/tests/snapshots/corpus__basic.snap +5 -0
  107. domq-0.4.0/tests/snapshots/corpus__document.snap +5 -0
  108. domq-0.4.0/tests/snapshots/corpus__html_basic.snap +8 -0
  109. domq-0.4.0/tests/snapshots/corpus__html_document.snap +19 -0
  110. domq-0.4.0/tests/snapshots/corpus__html_messy.snap +11 -0
  111. domq-0.4.0/tests/snapshots/corpus__html_omitted.snap +30 -0
  112. domq-0.4.0/tests/snapshots/corpus__html_ruby.snap +7 -0
  113. domq-0.4.0/tests/snapshots/corpus__html_select.snap +16 -0
  114. domq-0.4.0/tests/snapshots/corpus__html_tables.snap +20 -0
  115. domq-0.4.0/tests/snapshots/corpus__messy.snap +5 -0
  116. domq-0.4.0/tests/snapshots/corpus__omitted.snap +5 -0
  117. domq-0.4.0/tests/snapshots/corpus__ruby.snap +5 -0
  118. domq-0.4.0/tests/snapshots/corpus__select.snap +5 -0
  119. domq-0.4.0/tests/snapshots/corpus__tables.snap +5 -0
  120. domq-0.4.0/tests/snapshots/markdown_output__page_blog.snap +66 -0
  121. domq-0.4.0/tests/snapshots/markdown_output__page_docs.snap +56 -0
  122. domq-0.4.0/tests/snapshots/markdown_output__page_messy.snap +25 -0
  123. domq-0.4.0/tests/snapshots/text_output__page_blog.snap +60 -0
  124. domq-0.4.0/tests/snapshots/text_output__page_docs.snap +51 -0
  125. domq-0.4.0/tests/snapshots/text_output__page_messy.snap +22 -0
  126. domq-0.4.0/tests/source_spans.rs +195 -0
  127. domq-0.4.0/tests/tables.rs +337 -0
  128. domq-0.4.0/tests/text_output.rs +234 -0
  129. domq-0.4.0/tests/url.rs +250 -0
domq-0.4.0/.gitignore ADDED
@@ -0,0 +1,12 @@
1
+ /target
2
+ **/target
3
+ Cargo.lock
4
+ /wasm/pkg
5
+ # wasm build output for the playground (built in CI by .github/workflows/pages.yml)
6
+ /playground/pkg
7
+ *.profraw
8
+ .DS_Store
9
+ # cargo-fuzz working dirs (not the tests/corpus/ fixtures!)
10
+ /fuzz/corpus/
11
+ /fuzz/artifacts/
12
+ /.claude/worktrees/
@@ -0,0 +1,273 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format is based on
4
+ [Keep a Changelog](https://keepachangelog.com/), and this project adheres to
5
+ [Semantic Versioning](https://semver.org/).
6
+
7
+ ## [Unreleased] — 0.4.0
8
+
9
+ "Declarative scraping", and one name everywhere.
10
+
11
+ ### Changed
12
+ - **Renamed to `domq`.** The crate is `domq` (was `dom-tree-rs`, imported as
13
+ `domtree`), the CLI crate `domq-cli` with the binary `domq` (was
14
+ `domtree-cli` / `domtree`), the npm package `domq-wasm` (was `dom-tree-wasm`; npm reserves plain `domq`), the
15
+ PyPI package and Python import `domq` (was `dom-tree-rs` / `domtree`), and
16
+ the Homebrew formula `iamsaquib8/tap/domq`. The repository moves to
17
+ `github.com/iamsaquib8/domq`; the old packages get a final release pointing
18
+ here.
19
+ - **MSRV raised to Rust 1.88** (from 1.85).
20
+ - **Python 3.11+** (3.9 and 3.10 are end of life); wheels are abi3 for
21
+ CPython ≥ 3.11.
22
+ - `NodeRef::to_text()` / `to_markdown()` called *on* a hidden element
23
+ (`<title>`, `<script>`, `<style>`, `<template>`, …) now return its content
24
+ instead of `""`; hiding applies only to descendants. `to_text` gives a
25
+ `<script>`/`<style>` source verbatim. Document-level output is unchanged.
26
+ - CI and release builds use the current Node LTS; the derive crate uses
27
+ `syn` 3.
28
+
29
+ ### Added
30
+ - **`#[derive(FromHtml)]`** (new `derive` feature, companion crate
31
+ `domq-derive`): declarative extraction into structs with
32
+ `#[html(select, attr, html, outer_html, markdown, default, with, skip)]`,
33
+ `Option<T>`, `Vec<T>`, nested and generic structs, a struct-level
34
+ `#[html(select = "…")]`, and compile-time diagnostics for misused
35
+ attributes.
36
+ - `Dom::extract`, `Dom::extract_all`, `NodeRef::extract`, the
37
+ `FromHtml` / `FromHtmlValue` traits, `Scope`, and `ExtractError`, whose
38
+ messages name the field path, selector and source:
39
+ `Product.reviews[2].stars (".stars" @data-value): cannot parse "x" as u8: …`.
40
+ Zero-dependency and `no_std`; hand-written impls need no feature.
41
+ - **Python `domq.extract` / `extract_all`**: the same declarative extraction
42
+ into dataclasses with `Annotated[T, Select(...)]` (`Attr`, `Html`,
43
+ `OuterHtml`, `Markdown` shorthands), `Optional`, `list`, nested dataclasses,
44
+ `default=`, `with_=`, a class-level `__select__`, and `ExtractError` with the
45
+ same paths and messages as Rust.
46
+
47
+ ## [0.3.0] — 2026-10-21
48
+
49
+ "One-line scraping": find things and get their content out, in Rust, in the
50
+ shell, in JavaScript and in Python. This release covers what the roadmap had
51
+ planned as phases 0.3 to 0.7, plus distribution (npm, PyPI, Homebrew, release
52
+ binaries) and the fixes from two independent reviews.
53
+
54
+ ### Added
55
+ - **CSS selector engine v2** (`query` feature, now **on by default**):
56
+ `+`/`~` combinators; every attribute operator (`^=` `$=` `*=` `~=` `|=`) with
57
+ the `i`/`s` flags; `:first-child`, `:last-child`, `:only-child`,
58
+ `:nth-child(An+B [of S])`, `:nth-last-child`, the `*-of-type` family,
59
+ `:empty`, `:root`, `:scope`, `:not()`, `:is()`, `:where()`, `:has()` (with
60
+ relative selectors), `:checked`/`:disabled`/`:enabled`, CSS escapes, and the
61
+ non-standard `:contains("text")`. Matching never backtracks exponentially and
62
+ never recurses on document depth.
63
+ - `Selector::parse` (and `FromStr`) — compile once, reuse, and get a
64
+ `SelectorError` with a byte position instead of silently matching nothing.
65
+ `Dom::select` accepts `&str`, `&String`, or `&Selector`.
66
+ - Scoped queries: `Dom::select_first`, `NodeRef::select`, `select_first`,
67
+ `matches`, `closest`.
68
+ - `to_text()` on `Dom`/`NodeRef` — readable plain text (block elements become
69
+ line breaks, whitespace collapsed, `<script>`/`<style>`/`<head>` dropped).
70
+ - `to_markdown()` (new `markdown` feature, on by default) — zero-dependency HTML
71
+ → CommonMark + GFM: headings, emphasis, links, images, nested lists, code
72
+ blocks with languages, blockquotes, tables.
73
+ - `NodeRef::inner_html`, `outer_html`, `next_element_sibling`,
74
+ `prev_element_sibling`.
75
+ - Editing: `Dom::remove` (detach a subtree), `Dom::set_attr`, `Dom::remove_attr`.
76
+ - **`domtree` CLI** (`cargo install domtree-cli`) — htmlq-compatible flags
77
+ (`-a`, `-t`, `-p`, `-w`, `-r`, `-b`, `-B`, `-f`, `-o`) plus `--markdown`,
78
+ `--json`, `--count`, `--first`; zero dependencies. Prebuilt binaries are
79
+ attached to GitHub releases.
80
+
81
+ - **`dom-tree-wasm` 0.3.0** (npm): `parse` now returns a typed object
82
+ (`parseJson` keeps the string); new `select`, `selectFirst`, `toText`,
83
+ `toMarkdown`, `toHtml`, `errors`, and a parse-once `Document` class. Invalid
84
+ selectors throw a `SelectorError` with a position. Full TypeScript types,
85
+ works in Node with no setup, ~41 KB gzipped. **Breaking:** `parseErrorCount`
86
+ and `query` are removed, and `dom-node.d.ts` is replaced by the generated
87
+ types.
88
+ - **Live playground** on GitHub Pages: paste HTML, try selectors, and see the
89
+ tree, text, Markdown, JSON and repaired HTML, all running locally in wasm.
90
+ - `CODE_OF_CONDUCT.md`, `SECURITY.md`, `docs/ARCHITECTURE.md`, issue and PR
91
+ templates.
92
+ - **Streaming events:** `domtree::events(html)` yields `Start` / `End`
93
+ (including implied ends) / `Text` / `Comment` / `Doctype`, each with its byte
94
+ range, applying exactly the same recovery as `parse` but without building a
95
+ tree; `Events::errors()` matches `Dom::errors()`.
96
+ - **`simd` feature** (off by default): `memchr`-backed scanning, identical
97
+ output, ~24% faster on text-heavy input. The default build stays
98
+ zero-dependency.
99
+ - Benchmarks against `scraper` and `html5ever`, text- and table-heavy inputs,
100
+ a memory report, and a weekly `Benchmarks` workflow; a `differential` fuzz
101
+ target against html5ever.
102
+ - **Editing, part 2:** `Dom::set_text`, `rename`, `create_element`,
103
+ `create_text`, `append_child`, `insert_before`, `wrap`, `unwrap`, and
104
+ parsed-fragment insertion (`append_html`, `prepend_html`,
105
+ `insert_html_before`, `insert_html_after`, returning the inserted ids). Moves
106
+ never create cycles; `nodes()`, `select` and `find_by_*` stay in document
107
+ order after any edit. `NodeRef::is_detached()`.
108
+ - **Source spans:** `NodeRef::source_range()` (byte range in the input) and
109
+ `domtree::line_col(src, byte)`.
110
+ - **`entities-full` feature:** all 2,125 WHATWG named character references
111
+ (off by default; the default table has the 178 most common).
112
+ - `Dom::main_content()` — a deterministic "reader mode" heuristic (semantic
113
+ markup first, then Readability-style paragraph scoring) that picks the
114
+ article and drops nav, sidebars, comments, ads and footers.
115
+ - `Dom::to_markdown_chunks(max_chars)` / `NodeRef::to_markdown_chunks` —
116
+ heading-aware Markdown chunks for RAG (`Chunk { heading_path, markdown }`,
117
+ `Chunk::context()`), never splitting inside code blocks or tables unless a
118
+ single block exceeds the budget.
119
+ - `Dom::links(base)` (readable link text, resolved URLs, `<base href>`,
120
+ `javascript:` skipped), `Dom::resources(base)` (every embedded URL incl.
121
+ `srcset` candidates), `Dom::base_url(base)`.
122
+ - `Dom::metadata()` — title, description, canonical, lang, site name, image,
123
+ author, published date, keywords, OpenGraph and Twitter fields, raw JSON-LD
124
+ and feed links (`serde::Serialize` under `serde`).
125
+ - `NodeRef::table_rows()` (colspan/rowspan expanded, bounded on hostile input)
126
+ and `domtree::to_csv` (RFC 4180).
127
+ - `domtree::resolve_url` / `resolve_srcset` — zero-dependency RFC 3986 and
128
+ spec-compliant `srcset` resolution, moved into the library from the CLI.
129
+ - **Python package** `dom-tree-rs` on PyPI (import `domtree`): `parse`,
130
+ `Document.select`/`select_first`/`to_text`/`to_markdown`/`to_html`/`to_json`,
131
+ `Element` with attributes, text, HTML and traversal; typed stubs; abi3 wheels
132
+ for Linux, macOS and Windows; the GIL is released while parsing large inputs.
133
+ - **Homebrew formula** for the `domtree` CLI (`packaging/homebrew/`), filled in
134
+ with release checksums by the release workflow.
135
+ - **WASI build** of the CLI (`domtree.wasm`, attached to releases, smoke-tested
136
+ under wasmtime), including quiet exit on a closed stdout pipe.
137
+ - **CLI structured extraction:** `domtree -e NAME=SELECTOR[@attr][[]]` builds a
138
+ JSON object of named fields, or one object per match of a scope selector
139
+ (JSON array, or `--jsonl`).
140
+ - **CLI output modes:** `--jsonl` (JSON Lines), `--csv` (tables as RFC 4180
141
+ CSV), `--main` (reader-mode narrowing before selecting), `--links`
142
+ (url/text, TSV or JSON), `--meta` (page metadata as JSON).
143
+ - **`domtree lint`:** images without `alt`, empty links and buttons, duplicate
144
+ ids, skipped heading levels, and every recovered parse error, as
145
+ `file:line:col: level: message [rule]` or `--format json`; exit 0/1/2 for CI.
146
+ Omitted optional end tags are notes, missing required ones warnings.
147
+ - **CLI encoding detection:** BOM, `<meta charset>`, UTF-8, then Windows-1252;
148
+ `--encoding` override.
149
+ - Prebuilt CLI binaries for Linux arm64.
150
+
151
+ ### Changed
152
+ - Top-level nodes are now linked as siblings (`next_sibling`/`prev_sibling`),
153
+ so sibling combinators and `:first-child` work on multi-root fragments.
154
+ - Numeric character references in `0x80–0x9F` are remapped per the WHATWG
155
+ Windows-1252 table (`&#150;` → `–`), as browsers do; `&#0;`, surrogates and
156
+ out-of-range values become U+FFFD.
157
+ - Default features are now `json`, `query`, `markdown` (still zero
158
+ dependencies). Use `default-features = false` for the smallest build.
159
+ - **Faster parsing:** one attribute arena per document (nodes are now 52
160
+ bytes), no per-tag allocations, and a word-at-a-time byte search. Parse time
161
+ is 25–37% lower on markup and 69% lower on text-heavy pages; a parse makes
162
+ ~20 allocations instead of thousands; peak memory is 10–14% lower, and unused
163
+ arena capacity is released after parsing.
164
+ - More implicit-close rules (WHATWG optional tags): the full list of elements
165
+ that close `<p>`; a heading closes an open heading; an unclosed `<head>` ends
166
+ at the first body content (previously such pages lost their body in
167
+ `to_text`); ruby `rb`/`rt`/`rtc`/`rp`; `<hr>` closes an open
168
+ `option`/`optgroup`; captions, `colgroup` and `col` close open cells, rows
169
+ and row groups.
170
+ - Nodes are 64 bytes (were 80); `NodeId` is non-zero, so `Option<NodeId>` is
171
+ 4 bytes.
172
+
173
+ - CLI: undeclared non-UTF-8 input now decodes as Windows-1252 (as browsers
174
+ do) instead of lossy UTF-8, e.g. `caf\xe9` → `café`.
175
+
176
+ ### Security / hardening
177
+ - Edited content can no longer inject markup on output: text set inside
178
+ `<script>`, `<style>`, `<textarea>` or `<title>` cannot close the element
179
+ early (a `</script` is written `<\/script`; parsed documents serialize
180
+ byte-for-byte as before), and `set_attr`, `rename`, `create_element` and
181
+ `wrap` refuse names that are empty or contain whitespace, quotes, `<`, `>`,
182
+ `/`, `=` or NUL (tag names must start with an ASCII letter).
183
+ `create_element` now returns `Option<NodeId>`.
184
+ - Attacker-chosen tag names could make `parse`/`events` quadratic (40,000
185
+ colliding names: 6.4 s → 2 ms). An end tag now looks for its element among
186
+ the 256 innermost and 8 outermost open elements; an element open only in
187
+ between is left open and the end tag is reported as `StrayEndTag`.
188
+ - `resolve_url` reads `\` as `/` in http(s)/ftp/ws(s)/file URLs, as browsers
189
+ do (`\\evil.com\x` resolves to evil.com, not a relative path), and
190
+ `base_url` ignores a `<base>` inside `<template>`, `<svg>` or `<math>`.
191
+
192
+ ### Fixed
193
+ - Linear-time fixes found in review: scoped selector queries
194
+ (`tr.select(…)`, `matches`, `closest`) no longer pay for the whole document
195
+ per call (160k rows: 18 s → 70 ms) and selector memo memory is bounded;
196
+ `append_html` no longer slows with the parent's depth (3.4 s → 1.4 ms);
197
+ chunk heading paths are capped at 256 characters per title; `table_rows` is
198
+ linear on nested tables (a cell's text no longer includes tables nested in
199
+ it — **behaviour change**); CLI `--links SELECTOR` and nested `-e` scopes
200
+ are linear.
201
+ - `UnknownEntity` only reports WHATWG's 106 legacy names when their `;` is
202
+ missing (so `x&order=1`, `?a=1&or` are clean), and its position points at
203
+ the `&`, also inside attribute values.
204
+ - `colspan`/`rowspan` accept a leading `+` and skip only ASCII whitespace;
205
+ `Link::text` leaves out content hidden with `hidden` or inline
206
+ `display:none` / `visibility:hidden`.
207
+ - CLI: `lint` no longer reports correctly closed elements as unclosed
208
+ (`</span foo>`, a final `</span` without `>`), and does report an outer
209
+ element whose same-named child ends at the same spot; `-B` resolves
210
+ `<base href>` against `--base` once; combining two output formats is a
211
+ usage error; `--links -w` skips links without text.
212
+ - `Dom::errors()` no longer flags valid HTML: a bare `&` (`a & b`,
213
+ `?a=1&b=2`) isn't an `UnknownEntity` (reference-shaped `&…` still is, with
214
+ WHATWG's attribute-value exception), and an end tag closing elements whose
215
+ end tag is optional (`<ul><li>a</ul>`) is `ImplicitlyClosed`, not
216
+ `MismatchedEndTag`.
217
+ - Quadratic parsing of many unmatched end tags over a deep open-element stack,
218
+ and quadratic entity decoding when the lookahead ended inside a multi-byte
219
+ character.
220
+ - CLI `--base` keeps commas inside `srcset` URLs, drops empty candidates, and
221
+ rewrites the first of a repeated attribute (the one that counts).
222
+
223
+ ## [0.2.1] — 2026-06-24
224
+
225
+ ### Added
226
+ - Zero-dependency HTML re-serialization (`Dom::to_html`, `NodeRef::to_html`) —
227
+ round-trips a parsed tree back to HTML with correct escaping and void/raw-text
228
+ handling. Always available (no feature required).
229
+ - `#![forbid(unsafe_code)]` — the crate contains no `unsafe`.
230
+
231
+ ### Changed
232
+ - Moved to **edition 2024**; minimum supported Rust is now **1.85**
233
+ (developed and tested on the latest stable).
234
+
235
+ ## [0.2.0] — 2026-06-23
236
+
237
+ A full rewrite. The 0.1 line was a learning-project parser that panicked on
238
+ almost any real-world HTML; 0.2 is a robust, tiny, zero-dependency library.
239
+
240
+ ### Added
241
+ - Error-tolerant parser that **never panics**; recovery diagnostics via
242
+ `Dom::errors()`.
243
+ - Streaming, allocation-light `Tokenizer` (`domtree::tokenize`) implementing
244
+ `Iterator<Item = Token>`.
245
+ - Arena-based tree with O(1) `parent`/`children`/`descendants`/`ancestors`/
246
+ sibling traversal through a `Copy` `NodeRef` handle.
247
+ - `find_by_tag` / `find_by_id` / `find_by_class` helpers.
248
+ - Built-in, zero-dependency JSON serialization (`Dom::to_json`, `NodeRef::to_json`)
249
+ behind the default `json` feature.
250
+ - Optional `serde::Serialize` (`serde` feature) producing the identical JSON shape.
251
+ - Optional CSS-subset selector engine (`Dom::select`, `query` feature):
252
+ type/`*`/`#id`/`.class`/`[attr]`/`[attr=value]`, descendant and child
253
+ combinators, selector lists.
254
+ - `no_std` + `alloc` support; new `std` feature for std-only conveniences.
255
+ - Coverage for void elements, self-closing tags, comments, doctype, CDATA,
256
+ raw-text `<script>`/`<style>`, escapable raw text, boolean/unquoted attributes,
257
+ hyphenated/namespaced/custom tags, HTML entities (named + numeric), and
258
+ implicit closing for the common cases.
259
+ - Rebuilt WebAssembly bindings (`dom-tree-wasm`): ~19 KB gzipped, zero JS deps.
260
+ - `cargo-fuzz` harness, criterion benchmarks, and a full GitHub Actions CI.
261
+
262
+ ### Changed
263
+ - **Breaking:** the public API is redesigned. The crate is now used as `domtree`;
264
+ `parse` returns an owned `Dom`; `NodeRef::kind()` returns a borrowed `Copy`
265
+ `NodeKind<'a>`. The old owned `Node`/`Props`/`NodeData` types are gone.
266
+ - `serde`/`serde_json` are no longer dependencies of the default build.
267
+
268
+ ### Performance
269
+ - ~250 MiB/s on a synthetic article workload — within ~5–15 % of `tl` and
270
+ roughly on par on large documents — while remaining zero-dependency.
271
+
272
+ ## [0.1.1] — 2020
273
+ - Initial learning-project parser (HTML → debug tree / `serde_json` string).
@@ -0,0 +1,18 @@
1
+ # Code of conduct
2
+
3
+ This project follows the
4
+ [Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/).
5
+
6
+ In short: be welcoming and patient, assume good intent, and critique code and
7
+ ideas rather than people. Harassment, insults, and personal or political
8
+ attacks are not acceptable in issues, pull requests, discussions, or any other
9
+ project space.
10
+
11
+ ## Reporting
12
+
13
+ If someone's behaviour makes you uncomfortable, contact the maintainer
14
+ privately at the email address listed in `Cargo.toml` (`authors`). Reports are
15
+ handled confidentially. The maintainer may remove, edit, or reject comments,
16
+ commits, issues, and other contributions that break this code of conduct, and
17
+ may temporarily or permanently ban a contributor for repeated or serious
18
+ violations, following the Contributor Covenant's enforcement guidelines.
@@ -0,0 +1,69 @@
1
+ # Contributing
2
+
3
+ Thanks for your interest! This is a small, focused crate — contributions that
4
+ keep it small and focused are very welcome.
5
+
6
+ ## Two load-bearing principles
7
+
8
+ 1. **Never panic on input.** `parse` must always return a best-effort tree for
9
+ *any* byte sequence. No `unwrap`/`expect`/`panic`/indexing on input-derived
10
+ data — the lib denies those clippy lints. Use `get(..)` and iterators. New
11
+ code should be exercised by the proptest suite and ideally the fuzz harness.
12
+ 2. **Stay tiny and zero-dependency by default.** The default build has no
13
+ external runtime dependencies. New dependencies must be optional, behind a
14
+ feature, and justified. Prefer hand-rolled, `no_std`-friendly code.
15
+
16
+ ## Scope
17
+
18
+ This is intentionally a *forgiving* parser, not a WHATWG-compliant one. Proposals
19
+ to add the full tree-construction algorithm (adoption agency, foster parenting,
20
+ auto `<html>`/`<head>`/`<body>`) are out of scope — that's what `html5ever` is
21
+ for. Bug fixes, more entity coverage, more implicit-close cases, performance, and
22
+ ergonomics are all in scope.
23
+
24
+ ## Development
25
+
26
+ ```sh
27
+ cargo test --all-features # unit + corpus snapshots + proptest + doctests
28
+ cargo clippy --all-features --all-targets -- -D warnings
29
+ cargo fmt --all
30
+
31
+ # #[derive(FromHtml)]: end-to-end tests, then the macro's compile-fail tests
32
+ # (after changing a diagnostic: TRYBUILD=overwrite cargo test -p domq-derive)
33
+ cargo test --features derive
34
+ cargo test -p domq-derive
35
+
36
+ # CLI (workspace member)
37
+ cargo test -p domq-cli
38
+ cargo run -p domq-cli -- --help
39
+
40
+ # WASM
41
+ cd wasm && wasm-pack build --target web --release && wasm-pack test --node
42
+
43
+ # Playground (after building the wasm into playground/pkg — see playground/README.md)
44
+ python3 -m http.server -d playground 8000
45
+
46
+ # Benchmarks (see benchmarks/README.md); the memory report; the simd feature
47
+ cd benchmarks && cargo bench
48
+ cd benchmarks && cargo run --release --example memory
49
+ cargo test --features simd
50
+
51
+ # Fuzzing (nightly)
52
+ cargo +nightly fuzz run parse
53
+ cargo +nightly fuzz run differential # our tree vs html5ever on the claimed subset
54
+ ```
55
+
56
+ Snapshot tests use [`insta`]: after an intentional output change, run
57
+ `INSTA_UPDATE=always cargo test --test corpus` and review the diff before
58
+ committing the updated `.snap` files.
59
+
60
+ [`insta`]: https://insta.rs/
61
+
62
+ ## Before opening a PR
63
+
64
+ - `cargo fmt`, `cargo clippy` clean, tests pass.
65
+ - Add a test for the behaviour you changed (a corpus fixture is great for
66
+ parsing changes).
67
+ - Keep the MSRV at 1.88 (edition 2024); the CI `msrv` job enforces it. Raise it only in a
68
+ minor release, and never to a Rust release less than about six months old.
69
+ - Add an entry to `CHANGELOG.md` for anything user-visible.
domq-0.4.0/Cargo.toml ADDED
@@ -0,0 +1,89 @@
1
+ [package]
2
+ name = "domq"
3
+ version = "0.4.0"
4
+ edition = "2024"
5
+ rust-version = "1.88"
6
+ authors = ["iamsaquib8 <saquibulhassan6@gmail.com>"]
7
+ description = "Tiny, zero-dependency, forgiving HTML parser: turn messy real-world HTML into a clean DOM tree (and JSON). WASM-first, no_std-friendly."
8
+ repository = "https://github.com/iamsaquib8/domq"
9
+ homepage = "https://github.com/iamsaquib8/domq"
10
+ documentation = "https://docs.rs/domq"
11
+ readme = "README.md"
12
+ license = "MIT"
13
+ keywords = ["html", "parser", "css-selector", "markdown", "wasm"]
14
+ categories = ["parser-implementations", "wasm", "web-programming", "no-std"]
15
+ exclude = ["/.github", "/benchmarks", "/fuzz", "/wasm", "/cli", "/derive", "/python", "/packaging", "/playground", "/docs/launch"]
16
+
17
+ [lib]
18
+ name = "domq"
19
+
20
+ [features]
21
+ default = ["json", "query", "markdown"]
22
+ # Built-in, zero-dependency JSON serializer (`Dom::to_json`).
23
+ json = []
24
+ # Optional `serde::Serialize` impls on the tree (pulls in `serde`).
25
+ serde = ["dep:serde"]
26
+ # Hand-rolled CSS selector engine (`Dom::select`, `Selector`). Zero-dependency.
27
+ query = []
28
+ # Zero-dependency HTML -> Markdown (`Dom::to_markdown`).
29
+ markdown = []
30
+ # The complete WHATWG named character reference table (2,125 names) instead of
31
+ # the curated ~180. Off by default to keep the default build small.
32
+ entities-full = []
33
+ # Enables std-only conveniences (e.g. `std::error::Error` for `ParseError`).
34
+ # The crate is `no_std + alloc` by default.
35
+ std = []
36
+ # SIMD-accelerated scanning of text and attribute values via `memchr` (adds
37
+ # that one dependency, `no_std`). Off by default: the default build stays
38
+ # zero-dependency and produces exactly the same tokens.
39
+ simd = ["dep:memchr"]
40
+ # `#[derive(FromHtml)]`: declarative extraction into structs. Re-exports the
41
+ # proc-macro from the `domq-derive` companion crate (which pulls in `syn`,
42
+ # `quote` and `proc-macro2` at compile time only). The traits it implements
43
+ # (`FromHtml`, `Dom::extract`, …) come with `query` and need no extra crates.
44
+ derive = ["query", "dep:domq-derive"]
45
+
46
+ [[example]]
47
+ name = "to_markdown"
48
+ required-features = ["query", "markdown"]
49
+
50
+ [[test]]
51
+ name = "extract"
52
+ required-features = ["derive", "markdown"]
53
+
54
+ [[test]]
55
+ name = "extract_derive_properties"
56
+ required-features = ["derive"]
57
+
58
+ [dependencies]
59
+ serde = { version = "1", default-features = false, features = ["alloc"], optional = true }
60
+ memchr = { version = "2.7", default-features = false, optional = true }
61
+ domq-derive = { path = "derive", version = "0.4.0", optional = true }
62
+
63
+ [dev-dependencies]
64
+ # Snapshot testing for the corpus suite (dev-only; not a runtime dependency).
65
+ insta = "1"
66
+ # Property tests: "never panics", arena-invariant checks.
67
+ proptest = "1"
68
+ # Drives the `serde` Serialize impl in the JSON-equality roundtrip test.
69
+ serde_json = "1"
70
+
71
+ [package.metadata.docs.rs]
72
+ # Document all features (incl. query + serde) on docs.rs.
73
+ all-features = true
74
+ rustdoc-args = ["--cfg", "docsrs"]
75
+
76
+ [profile.release]
77
+ opt-level = 3
78
+ lto = true
79
+
80
+ [workspace]
81
+ resolver = "2"
82
+ # The `domq` command-line tool and the `#[derive(FromHtml)]` proc-macro
83
+ # crate. They share this crate's lockfile and target dir.
84
+ members = ["cli", "derive"]
85
+ # The WASM bindings, the Python bindings, the comparison benchmarks, and the
86
+ # fuzz harness are independent crates with their own (heavier) dependency
87
+ # graphs and release profiles, so they are kept out of this crate's pristine,
88
+ # minimal-dep graph.
89
+ exclude = ["wasm", "python", "benchmarks", "fuzz"]
domq-0.4.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2020 iamsaquib8
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
domq-0.4.0/LICENSE-MIT ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2020 iamsaquib8
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.