rusteerio 1.0.0 → 1.0.1

package/Cargo.lock

@@ -686,7 +686,7 @@ dependencies = [
 
 [[package]]
 name = "rusteerio"
-version = "1.0.0"
+version = "1.0.1"
 dependencies = [
  "ego-tree",
  "html5ever",

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rusteerio"
-version = "1.0.0"
+version = "1.0.1"
 edition = "2021"
 description = "Rust HTML5 parser and CSS selector engine for the rusteerio Node.js package"
 license = "MIT"

package/THIRD_PARTY_NOTICES.md

@@ -4,7 +4,7 @@ This file preserves third-party license and attribution notices for the Rust nat
 
 Generated by `node scripts/generate-notices.mjs` from `cargo metadata --locked`, crate license files, and the pinned supplemental sources recorded in `scripts/notices/sources.json`. It includes build dependencies and dependencies for all target platforms; a particular binary may use a subset. Identical notice texts are reproduced once and referenced below. The html5ever, markup5ever and html5gum crates contain local observer-hook and opt-in UTF-16 transport patches; their complete patched source and original notices are included in `native/vendor`. Other dependency source files are unmodified.
 
-Cargo.lock SHA-256: `972c2efc036e05a48803d6631cfd4f6026989898043bbd9aaf68bb52449c1716`.
+Cargo.lock SHA-256: `42f353a71e79c84b5415fbfa8581ddd3c1aefa2b8c15f78a876631c8a9be2a68`.
 
 ## Source availability
 
@@ -18,6 +18,12 @@ The vendored compatibility suite and fixtures come from Cheerio 1.2.0, commit `e
 
 - [Cheerio MIT LICENSE](#notice-61c1e21d3a8ff20f) — [upstream source](https://github.com/cheeriojs/cheerio/blob/e3c7aaf9ed64fe3cb9a181e58a41c0fdd6dbfbfc/LICENSE)
 
+## htmlparser2 tokenizer port
+
+The htmlparser2-compatible markup tokenizer in `native/src/markup_tokenizer.rs` ports the state machine of htmlparser2 10.1.0, commit `57ace50bf6eb3bfab0468deafe10d0a8a2f233aa`. Its original MIT notice is reproduced here.
+
+- [htmlparser2 MIT LICENSE](#notice-204cfa747341660e) — [upstream source](https://github.com/fb55/htmlparser2/blob/57ace50bf6eb3bfab0468deafe10d0a8a2f233aa/LICENSE)
+
 ## Rust dependencies (112 locked packages)
 
 ### aho-corasick 1.1.5
@@ -2547,6 +2553,31 @@ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 ```
 
+<a id="notice-204cfa747341660e"></a>
+
+### Notice 204cfa747341660e
+
+```text
+Copyright 2010, 2011, Chris Winberry <chris@winberry.net>. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+ 
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.
+```
+
 <a id="notice-209fbbe0ad52d923"></a>
 
 ### Notice 209fbbe0ad52d923

package/native/src/arena.rs

@@ -372,7 +372,12 @@ impl DomArena {
                             node.extras.get("attribsPrefix"),
                         )
                     });
-                    for (name, value) in &node.attribs {
+                    let attributes = super::js_property_order(
+                        node.attribs
+                            .iter()
+                            .map(|(name, value)| (name.as_str(), value)),
+                    );
+                    for (name, value) in attributes {
                         output.push(' ');
                         match qualifiers {
                             Some((namespaces, prefixes)) => super::push_attribute_name(
@@ -880,481 +885,30 @@ fn escape(value: &str, attribute: bool, xml: bool, utf8: bool, output: &mut Stri
 }
 
 pub(crate) fn parse_xml(source: &str, options: Value) -> DomArena {
-    let mut arena = DomArena::from_records(vec![DomNode::new("root")], options);
-    let mut parents = vec![0u32];
-    parse_xml_segment(source, &mut arena, &mut parents, 0);
-    arena
+    crate::tokenizer::parse_markup(source, options)
 }
 
-fn parse_xml_segment(source: &str, arena: &mut DomArena, parents: &mut Vec<u32>, base: usize) {
-    let xml = arena.xml_mode();
-    let indices = arena.options.get("withStartIndices") == Some(&Value::Bool(true))
-        || arena.options.get("withEndIndices") == Some(&Value::Bool(true));
-    let mut positions = if indices {
-        vec![0usize; source.len() + 1]
-    } else {
-        Vec::new()
-    };
-    let mut utf16 = 0;
-    let codec = arena.options.get("__utf16Codec") == Some(&Value::Bool(true));
-    let mut escaped = false;
-    if indices {
-        for (at, ch) in source.char_indices() {
-            for position in &mut positions[at..at + ch.len_utf8()] {
-                *position = utf16;
-            }
-            if codec && ch == crate::codec::ESCAPE && !escaped {
-                escaped = true;
-            } else {
-                utf16 += ch.len_utf16();
-                escaped = false;
-            }
-            positions[at + ch.len_utf8()] = utf16;
-        }
-    }
-    let source_position = |index: usize| {
-        base + if indices {
-            positions[index.min(source.len())]
-        } else {
-            index
-        }
-    };
-    macro_rules! append_parsed {
-        ($arena:expr,$parent:expr,$node:expr,$start:expr,$end:expr $(,)?) => {
-            append_parsed(
-                $arena,
-                $parent,
-                $node,
-                source_position($start),
-                source_position(($end).saturating_add(1)).saturating_sub(1),
-            )
-        };
-    }
-    let mut index = 0;
-    let bytes = source.as_bytes();
-    let decode = arena.options.get("decodeEntities") != Some(&Value::Bool(false));
-    while index < bytes.len() {
-        let start = index;
-        if !xml {
-            if let Some(name) = parents
-                .last()
-                .and_then(|id| arena.node(*id))
-                .and_then(|node| node.name.as_deref())
-                .filter(|name| matches!(*name, "script" | "style" | "xmp" | "textarea" | "title"))
-                .map(str::to_owned)
-            {
-                let closing = format!("</{name}");
-                let end = source.as_bytes()[index..]
-                    .windows(closing.len())
-                    .enumerate()
-                    .find(|(offset, candidate)| {
-                        candidate.eq_ignore_ascii_case(closing.as_bytes())
-                            && bytes
-                                .get(index + offset + closing.len())
-                                .is_none_or(|ch| *ch == b'>' || ch.is_ascii_whitespace())
-                    })
-                    .map_or(bytes.len(), |(offset, _)| index + offset);
-                if end > index {
-                    let text = &source[index..end];
-                    let mut node = DomNode::new("text");
-                    node.data = Some(if decode && name == "title" {
-                        decode_entities_mode(
-                            text,
-                            false,
-                            false,
-                            arena.options.get("__utf16Codec") == Some(&Value::Bool(true)),
-                        )
-                    } else {
-                        text.into()
-                    });
-                    append_parsed!(
-                        arena,
-                        *parents.last().unwrap(),
-                        node,
-                        index,
-                        end.saturating_sub(1),
-                    );
-                    index = end;
-                    continue;
-                }
-            }
-            if bytes[index] == b'<'
-                && bytes.get(index + 1).is_some_and(|ch| {
-                    !ch.is_ascii_alphabetic() && !matches!(*ch, b'!' | b'?' | b'/')
-                })
-            {
-                let mut node = DomNode::new("text");
-                node.data = Some("<".into());
-                append_parsed!(arena, *parents.last().unwrap(), node, index, index);
-                index += 1;
-                continue;
-            }
-        }
-        if bytes[index] != b'<' {
-            index = source[index..]
-                .find('<')
-                .map_or(bytes.len(), |offset| index + offset);
-            let text = &source[start..index];
-            let mut node = DomNode::new("text");
-            node.data = Some(if decode {
-                decode_entities_mode(
-                    text,
-                    xml,
-                    false,
-                    arena.options.get("__utf16Codec") == Some(&Value::Bool(true)),
-                )
-            } else {
-                text.into()
-            });
-            append_parsed!(
-                arena,
-                *parents.last().unwrap(),
-                node,
-                start,
-                index.saturating_sub(1),
-            );
-            continue;
-        }
-        if source[index..].starts_with("<!--") {
-            let end = source[index + 4..]
-                .find("-->")
-                .map(|offset| index + 4 + offset);
-            let data_end = end.unwrap_or(bytes.len());
-            index = end.map_or(bytes.len(), |end| end + 3);
-            let mut node = DomNode::new("comment");
-            node.data = Some(source[start + 4..data_end].into());
-            append_parsed!(
-                arena,
-                *parents.last().unwrap(),
-                node,
-                start,
-                index.saturating_sub(1),
-            );
-            continue;
-        }
-        if source[index..].starts_with("<![CDATA[") {
-            let end = source[index + 9..]
-                .find("]]>")
-                .map(|offset| index + 9 + offset);
-            let data_end = end.unwrap_or(bytes.len());
-            index = end.map_or(bytes.len(), |end| end + 3);
-            if !xml && arena.options.get("recognizeCDATA") != Some(&Value::Bool(true)) {
-                let mut node = DomNode::new("comment");
-                node.data = Some(format!("[CDATA[{}]]", &source[start + 9..data_end]));
-                append_parsed!(
-                    arena,
-                    *parents.last().unwrap(),
-                    node,
-                    start,
-                    index.saturating_sub(1)
-                );
-                continue;
-            }
-            let id = append_parsed!(
-                arena,
-                *parents.last().unwrap(),
-                DomNode::new("cdata"),
-                start,
-                index.saturating_sub(1),
-            );
-            let mut text = DomNode::new("text");
-            text.data = Some(source[start + 9..data_end].into());
-            append_parsed!(arena, id, text, start + 9, data_end.saturating_sub(1));
-            continue;
-        }
-        let mut cursor = index + 1;
-        let mut quote = None;
-        while cursor < bytes.len() {
-            let ch = bytes[cursor];
-            if let Some(q) = quote {
-                if ch == q {
-                    quote = None;
-                }
-            } else if ch == b'\'' || ch == b'"' {
-                quote = Some(ch);
-            } else if ch == b'>' {
-                break;
-            }
-            cursor += 1;
-        }
-        if cursor >= bytes.len() {
-            let mut text = DomNode::new("text");
-            text.data = Some(source[index..].into());
-            append_parsed!(
-                arena,
-                *parents.last().unwrap(),
-                text,
-                start,
-                bytes.len().saturating_sub(1),
-            );
-            break;
-        }
-        index = cursor + 1;
-        let token = source[start + 1..cursor].trim();
-        if token.starts_with('!') || token.starts_with('?') {
-            let mut node = DomNode::new("directive");
-            node.name = Some(token.split_whitespace().next().unwrap_or("").into());
-            node.data = Some(token.into());
-            append_parsed!(arena, *parents.last().unwrap(), node, start, cursor);
-            continue;
-        }
-        if let Some(close) = token.strip_prefix('/') {
-            let close = close.split_whitespace().next().unwrap_or("");
-            if let Some(position) = parents.iter().rposition(|id| {
-                arena.nodes[*id as usize]
-                    .name
-                    .as_deref()
-                    .is_some_and(|name| {
-                        if xml {
-                            name == close
-                        } else {
-                            name.eq_ignore_ascii_case(close)
-                        }
-                    })
-            }) {
-                for id in parents.drain(position..) {
-                    if arena.options.get("withEndIndices") == Some(&Value::Bool(true)) {
-                        arena.nodes[id as usize]
-                            .extras
-                            .insert("endIndex".into(), json!(source_position(cursor)));
-                    }
-                }
-            }
-            continue;
-        }
-        let slash = token.ends_with('/');
-        let content = token.strip_suffix('/').unwrap_or(token);
-        let name_end = content.find(char::is_whitespace).unwrap_or(content.len());
-        let name = &content[..name_end];
-        if name.is_empty() {
-            continue;
-        }
-        let lower = arena
-            .options
-            .get("lowerCaseTags")
-            .and_then(Value::as_bool)
-            .unwrap_or(!xml);
-        let name = if lower {
-            name.to_ascii_lowercase()
-        } else {
-            name.into()
-        };
-        if !xml {
-            while parents
-                .last()
-                .and_then(|id| arena.node(*id))
-                .and_then(|node| node.name.as_deref())
-                .is_some_and(|open| implied_close(&name, open))
-            {
-                let id = parents.pop().unwrap();
-                if arena.options.get("withEndIndices") == Some(&Value::Bool(true)) {
-                    arena.nodes[id as usize].extras.insert(
-                        "endIndex".into(),
-                        json!(source_position(start.saturating_sub(1))),
-                    );
-                }
-            }
-        }
-        let foreign = matches!(name.as_str(), "svg" | "math")
-            || parents
-                .iter()
-                .rev()
-                .filter_map(|id| arena.node(*id))
-                .filter_map(|node| node.name.as_deref())
-                .find_map(|name| {
-                    if matches!(
-                        name,
-                        "mi" | "mo"
-                            | "mn"
-                            | "ms"
-                            | "mtext"
-                            | "annotation-xml"
-                            | "foreignobject"
-                            | "desc"
-                            | "title"
-                    ) {
-                        Some(false)
-                    } else if matches!(name, "svg" | "math") {
-                        Some(true)
-                    } else {
-                        None
-                    }
-                })
-                .unwrap_or(false);
-        let self_close = slash
-            && (xml
-                || foreign
-                || arena.options.get("recognizeSelfClosing") == Some(&Value::Bool(true)));
-        let mut node = DomNode::new(if !xml && name == "script" {
-            "script"
-        } else if !xml && name == "style" {
-            "style"
-        } else {
-            "tag"
-        });
-        node.name = Some(name.clone());
-        node.attribs = parse_attributes(
-            &content[name_end..],
-            decode,
-            arena
-                .options
-                .get("lowerCaseAttributeNames")
-                .and_then(Value::as_bool)
-                .unwrap_or(!xml),
-            xml,
-            arena.options.get("__utf16Codec") == Some(&Value::Bool(true)),
-        );
-        let id = append_parsed!(arena, *parents.last().unwrap(), node, start, cursor);
-        let void = !xml && html_parser_void(&name);
-        if !self_close && !void {
-            parents.push(id);
-        } else if arena.options.get("withEndIndices") == Some(&Value::Bool(true)) {
-            arena.nodes[id as usize]
-                .extras
-                .insert("endIndex".into(), json!(source_position(cursor)));
-        }
-    }
-}
-
-/// Incremental HTML-parser2/XML state; only incomplete markup/entity tokens are
-/// buffered, while completed input is discarded after native tree construction.
+/// Incremental htmlparser2/XML input. The complete input is tokenized once, so
+/// tags and entities split across chunks parse exactly as one string does.
 pub(crate) struct XmlStream {
     pub arena: DomArena,
-    parents: Vec<u32>,
     pending: String,
-    offset: usize,
 }
 impl XmlStream {
     pub(crate) fn new(options: Value) -> Self {
         Self {
             arena: DomArena::from_records(vec![DomNode::new("root")], options),
-            parents: vec![0],
             pending: String::new(),
-            offset: 0,
         }
     }
     pub(crate) fn feed(&mut self, chunk: &str, finish: bool) {
         self.pending.push_str(chunk);
-        let mut consumed = 0;
-        loop {
-            let input = &self.pending[consumed..];
-            let mut length = if finish {
-                input.len()
-            } else {
-                self.complete_prefix(input)
-            };
-            if !finish
-                && self.arena.options.get("__utf16Codec") == Some(&Value::Bool(true))
-                && input[..length]
-                    .chars()
-                    .rev()
-                    .take_while(|ch| *ch == crate::codec::ESCAPE)
-                    .count()
-                    % 2
-                    == 1
-            {
-                length -= crate::codec::ESCAPE.len_utf8();
-            }
-            if length == 0 {
-                break;
-            }
-            let segment = &input[..length];
-            parse_xml_segment(segment, &mut self.arena, &mut self.parents, self.offset);
-            self.offset += if self.arena.options.get("__utf16Codec") == Some(&Value::Bool(true)) {
-                crate::codec::logical_len(segment)
-            } else {
-                segment.encode_utf16().count()
-            };
-            consumed += length;
-            if consumed == self.pending.len() || finish {
-                break;
-            }
+        if finish {
+            let options = self.arena.options.clone();
+            self.arena = parse_xml(&std::mem::take(&mut self.pending), options);
         }
-        self.pending.drain(..consumed);
     }
-    fn complete_prefix(&self, input: &str) -> usize {
-        let bytes = input.as_bytes();
-        if input.is_empty() {
-            return 0;
-        }
-        if !self.arena.xml_mode() {
-            if let Some(name) = self
-                .parents
-                .last()
-                .and_then(|id| self.arena.node(*id))
-                .and_then(|node| node.name.as_deref())
-                .filter(|name| matches!(*name, "script" | "style" | "xmp" | "textarea" | "title"))
-            {
-                let close = format!("</{name}");
-                let closing = bytes
-                    .windows(close.len())
-                    .enumerate()
-                    .find(|(at, value)| {
-                        value.eq_ignore_ascii_case(close.as_bytes())
-                            && bytes
-                                .get(at + close.len())
-                                .is_none_or(|ch| *ch == b'>' || ch.is_ascii_whitespace())
-                    })
-                    .map(|(at, _)| at);
-                match closing {
-                    Some(0) if bytes.len() == close.len() => return 0,
-                    Some(0) => {}
-                    Some(index) => return safe_entity_prefix(input, index, name == "title"),
-                    None => {
-                        let mut end = input.len().saturating_sub(close.len() + 1);
-                        while !input.is_char_boundary(end) {
-                            end -= 1;
-                        }
-                        return safe_entity_prefix(input, end, name == "title");
-                    }
-                }
-            }
-        }
-        if bytes[0] != b'<' {
-            let end = input.find('<').unwrap_or(input.len());
-            return safe_entity_prefix(
-                input,
-                end,
-                self.arena.options.get("decodeEntities") != Some(&Value::Bool(false)),
-            );
-        }
-        if input.starts_with("<!--") {
-            return input.find("-->").map_or(0, |index| index + 3);
-        }
-        if input.starts_with("<![CDATA[") {
-            return input.find("]]>").map_or(0, |index| index + 3);
-        }
-        let mut quote = None;
-        for (index, ch) in bytes.iter().copied().enumerate().skip(1) {
-            if let Some(q) = quote {
-                if ch == q {
-                    quote = None;
-                }
-            } else if ch == b'\'' || ch == b'"' {
-                quote = Some(ch);
-            } else if ch == b'>' {
-                return index + 1;
-            }
-        }
-        0
-    }
-}
-fn safe_entity_prefix(source: &str, end: usize, decode: bool) -> usize {
-    if decode {
-        if let Some(start) = source[..end].rfind('&') {
-            let tail = &source[start + 1..end];
-            if tail.len() < 64
-                && tail
-                    .chars()
-                    .all(|ch| ch.is_ascii_alphanumeric() || ch == '#')
-            {
-                return start;
-            }
-        }
-    }
-    end
 }
-
 pub(crate) fn append_parsed(
     arena: &mut DomArena,
     parent: u32,
@@ -1424,70 +978,6 @@ pub(crate) fn implied_close(name: &str, open: &str) -> bool {
         _ => false,
     }
 }
-fn parse_attributes(
-    source: &str,
-    decode: bool,
-    lower: bool,
-    xml: bool,
-    codec: bool,
-) -> Map<String, Value> {
-    let bytes = source.as_bytes();
-    let mut index = 0;
-    let mut attrs = Map::new();
-    while index < bytes.len() {
-        while index < bytes.len() && bytes[index].is_ascii_whitespace() {
-            index += 1;
-        }
-        let start = index;
-        while index < bytes.len() && !bytes[index].is_ascii_whitespace() && bytes[index] != b'=' {
-            index += 1;
-        }
-        if start == index {
-            index += 1;
-            continue;
-        }
-        let name = &source[start..index];
-        while index < bytes.len() && bytes[index].is_ascii_whitespace() {
-            index += 1;
-        }
-        let mut value = "";
-        if bytes.get(index) == Some(&b'=') {
-            index += 1;
-            while index < bytes.len() && bytes[index].is_ascii_whitespace() {
-                index += 1;
-            }
-            if matches!(bytes.get(index), Some(b'\'' | b'"')) {
-                let quote = bytes[index];
-                index += 1;
-                let start = index;
-                while index < bytes.len() && bytes[index] != quote {
-                    index += 1;
-                }
-                value = &source[start..index];
-                index += (index < bytes.len()) as usize;
-            } else {
-                let start = index;
-                while index < bytes.len() && !bytes[index].is_ascii_whitespace() {
-                    index += 1;
-                }
-                value = &source[start..index];
-            }
-        }
-        let name = if lower {
-            name.to_ascii_lowercase()
-        } else {
-            name.into()
-        };
-        attrs.entry(name).or_insert_with(|| {
-            Value::String(if decode {
-                decode_entities_mode(value, xml, true, codec)
-            } else {
-                value.into()
-            })
-        });
-    }
-    attrs
-}
 pub(crate) fn decode_entities_mode(
     source: &str,
     xml: bool,