rusteerio 1.0.1 → 1.1.0

package/native/src/snapshot.rs

@@ -0,0 +1,509 @@
+//! A compact copy of every node record that JavaScript decodes lazily. One call
+//! replaces a native round trip per node when a document is read node by node.
+//!
+//! Each node occupies `STRIDE` entries of `nodes`:
+//! 0 type, 1 parent + 1, 2 previous sibling + 1, 3 next sibling + 1,
+//! 4 first entry in `children`, 5 child count, 6 name + 1, 7 data + 1,
+//! 8 first pair in `attributes`, 9 attribute count, 10 flags, 11 start index,
+//! 12 end index, 13 namespace, 14 remaining fields as JSON + 1.
+//! String references index pairs of UTF-16 offsets into `text`.
+//!
+//! JavaScript receives one `Uint32Array`: the lengths of `nodes`, `children`,
+//! `attributes` and `strings`, whether any string holds codec escapes, then
+//! the four lists. String offsets count UTF-16 units of the encoded text, so
+//! JavaScript decodes only the strings it reads. Its memory belongs to V8,
+//! which frees it as soon as the snapshot is collected; an external buffer
+//! would add finalizer bookkeeping and wait for a later event-loop turn.
+use html5ever::{LocalName, Namespace};
+use napi::{Env, JsObject, JsTypedArray, Result, TypedArrayType};
+use scraper::Node;
+use serde_json::{Map, Value};
+
+use crate::arena::DomNode;
+use crate::id_hash::IdMap;
+use crate::NativeDocument;
+
+pub(crate) const STRIDE: usize = 15;
+/// Text longer than this stays out of snapshots: such nodes, mostly inline
+/// scripts and styles, are read individually if JavaScript asks for them.
+const LARGE_TEXT_BYTES: usize = 4096;
+const ABSENT: u32 = u32::MAX;
+const HAS_START_INDEX: u32 = 1;
+const HAS_END_INDEX: u32 = 2;
+const HAS_NAMESPACE: u32 = 4;
+const EMPTY_ATTRIBUTE_MAPS: u32 = 8;
+/// The record cannot be represented here; JavaScript reads it individually.
+const OPAQUE: u32 = 16;
+const RELEASED: u32 = 32;
+
+pub(crate) struct NodeSnapshot {
+    is_encoded: bool,
+    nodes: Vec<u32>,
+    children: Vec<u32>,
+    attributes: Vec<u32>,
+    strings: Vec<u32>,
+    text: String,
+}
+
+impl NodeSnapshot {
+    pub(crate) fn into_js(self, env: &Env) -> Result<JsObject> {
+        let parts = [&self.nodes, &self.children, &self.attributes, &self.strings];
+        let lengths = parts.map(|part| part.len() as u32);
+        let header = [
+            lengths[0],
+            lengths[1],
+            lengths[2],
+            lengths[3],
+            u32::from(self.is_encoded),
+        ];
+        let length = header.len() + parts.iter().map(|part| part.len()).sum::<usize>();
+        let mut buffer = env.create_arraybuffer(length * 4)?;
+        // SAFETY: every bit pattern is a valid `u32`; `align_to_mut` puts any
+        // misaligned bytes in the prefix, which is checked to be empty.
+        let (prefix, words, _) = unsafe { buffer.align_to_mut::<u32>() };
+        if !prefix.is_empty() || words.len() < length {
+            return Err(napi::Error::from_reason("Unaligned snapshot buffer"));
+        }
+        let mut offset = 0;
+        for part in std::iter::once(&header[..]).chain(parts.iter().map(|part| part.as_slice())) {
+            words[offset..offset + part.len()].copy_from_slice(part);
+            offset += part.len();
+        }
+        let data: JsTypedArray =
+            buffer
+                .into_raw()
+                .into_typedarray(TypedArrayType::Uint32, length, 0)?;
+        let mut object = env.create_object()?;
+        object.set_named_property("data", data)?;
+        // V8 decodes UTF-8 slowly; text copies as Latin-1 when every
+        // character fits, which also keeps V8's compact one-byte string.
+        let text = if self.text.is_ascii() {
+            env.create_string_latin1(self.text.as_bytes())?
+        } else if let Some(latin1) = latin1_bytes(&self.text) {
+            env.create_string_latin1(&latin1)?
+        } else {
+            env.create_string_utf16(&utf16_units(&self.text))?
+        };
+        object.set_named_property("text", text)?;
+        Ok(object)
+    }
+}
+
+/// `text` in Latin-1, if every character is below U+0100: UTF-8 encodes
+/// those with lead bytes below 0xC4.
+fn latin1_bytes(text: &str) -> Option<Vec<u8>> {
+    if text.bytes().any(|byte| byte >= 0xC4) {
+        return None;
+    }
+    Some(text.chars().map(|character| character as u8).collect())
+}
+
+/// `text` as UTF-16, widening eight ASCII bytes at a time and decoding other
+/// characters directly from their UTF-8 bytes, which `text` guarantees valid.
+fn utf16_units(text: &str) -> Vec<u16> {
+    const HIGH_BITS: u64 = 0x8080_8080_8080_8080;
+    let bytes = text.as_bytes();
+    let mut units = Vec::with_capacity(bytes.len());
+    let mut index = 0;
+    while index < bytes.len() {
+        if let Some(chunk) = bytes.get(index..index + 8) {
+            let word = u64::from_le_bytes(chunk.try_into().unwrap());
+            if word & HIGH_BITS == 0 {
+                units.extend(chunk.iter().map(|byte| u16::from(*byte)));
+                index += 8;
+                continue;
+            }
+        }
+        let lead = u32::from(bytes[index]);
+        let continuation = |offset: usize| u32::from(bytes[index + offset] & 0x3F);
+        let (scalar, length) = match lead {
+            0x00..=0x7F => (lead, 1),
+            0x80..=0xDF => (((lead & 0x1F) << 6) | continuation(1), 2),
+            0xE0..=0xEF => (
+                ((lead & 0x0F) << 12) | (continuation(1) << 6) | continuation(2),
+                3,
+            ),
+            _ => (
+                ((lead & 0x07) << 18)
+                    | (continuation(1) << 12)
+                    | (continuation(2) << 6)
+                    | continuation(3),
+                4,
+            ),
+        };
+        if scalar < 0x1_0000 {
+            units.push(scalar as u16);
+        } else {
+            let offset = scalar - 0x1_0000;
+            units.push(0xD800 | (offset >> 10) as u16);
+            units.push(0xDC00 | (offset & 0x3FF) as u16);
+        }
+        index += length;
+    }
+    units
+}
+
+/// The UTF-16 length of `text`: one unit per character, two for characters
+/// beyond the basic plane, counted from UTF-8 lead bytes.
+fn utf16_length(text: &str) -> usize {
+    text.bytes()
+        .map(|byte| usize::from(byte & 0xC0 != 0x80) + usize::from(byte >= 0xF0))
+        .sum()
+}
+
+struct Builder {
+    nodes: Vec<u32>,
+    children: Vec<u32>,
+    attributes: Vec<u32>,
+    strings: Vec<u32>,
+    text: String,
+    units: u32,
+    interned: IdMap<String, u32>,
+    /// html5ever names are atoms that hash and compare by identity.
+    local_names: IdMap<LocalName, u32>,
+    namespaces: IdMap<Namespace, u32>,
+    codec: bool,
+    is_encoded: bool,
+}
+
+pub(crate) fn build(document: &NativeDocument) -> NodeSnapshot {
+    let codec = document.options.get("__utf16Codec") == Some(&Value::Bool(true))
+        || document.options.get("__losslessUnicode") == Some(&Value::Bool(true));
+    let count = document.slot_count() as usize;
+    let mut builder = Builder {
+        nodes: Vec::with_capacity(count * STRIDE),
+        children: Vec::with_capacity(count),
+        attributes: Vec::new(),
+        strings: Vec::with_capacity(count * 2),
+        text: String::with_capacity(document.content_bytes.get()),
+        units: 0,
+        interned: IdMap::with_capacity_and_hasher(32, Default::default()),
+        local_names: IdMap::with_capacity_and_hasher(32, Default::default()),
+        namespaces: IdMap::default(),
+        codec,
+        is_encoded: false,
+    };
+    if let Some(arena) = document.arena.borrow().as_ref() {
+        builder.nodes.reserve(arena.nodes.len() * STRIDE);
+        for id in 0..arena.nodes.len() {
+            match arena.node(id as u32) {
+                Some(node) => builder.encode(node),
+                None => builder.released(),
+            }
+        }
+    } else {
+        builder.encode_immutable(document);
+    }
+    NodeSnapshot {
+        is_encoded: builder.is_encoded,
+        nodes: builder.nodes,
+        children: builder.children,
+        attributes: builder.attributes,
+        strings: builder.strings,
+        text: builder.text,
+    }
+}
+
+/// The JSON fields a document root's record carries, as `convert_node` sets them.
+fn quirks_mode_json(mode: html5ever::tree_builder::QuirksMode) -> &'static str {
+    match mode {
+        html5ever::tree_builder::QuirksMode::Quirks => r#"{"x-mode":"quirks"}"#,
+        html5ever::tree_builder::QuirksMode::LimitedQuirks => r#"{"x-mode":"limited-quirks"}"#,
+        html5ever::tree_builder::QuirksMode::NoQuirks => r#"{"x-mode":"no-quirks"}"#,
+    }
+}
+
+/// The JSON fields a doctype's record carries, as `convert_node` sets them.
+fn doctype_fields_json(doctype: &scraper::node::Doctype) -> String {
+    let quoted = |value: &str| serde_json::to_string(value).unwrap_or_else(|_| "\"\"".into());
+    format!(
+        r#"{{"x-name":{},"x-publicId":{},"x-systemId":{}}}"#,
+        quoted(doctype.name()),
+        quoted(doctype.public_id()),
+        quoted(doctype.system_id())
+    )
+}
+
+fn pointer(value: Option<u32>) -> u32 {
+    value.map_or(0, |id| id + 1)
+}
+
+impl Builder {
+    fn string(&mut self, value: &str) -> u32 {
+        let start = self.units;
+        self.text.push_str(value);
+        let units = if value.is_ascii() {
+            value.len()
+        } else {
+            self.is_encoded |= self.codec && crate::codec::has_encoding(value);
+            utf16_length(value)
+        };
+        self.units += units as u32;
+        self.strings.push(start);
+        self.strings.push(self.units);
+        (self.strings.len() / 2 - 1) as u32
+    }
+
+    fn interned(&mut self, value: &str) -> u32 {
+        if let Some(reference) = self.interned.get(value) {
+            return *reference;
+        }
+        let reference = self.string(value);
+        self.interned.insert(value.to_owned(), reference);
+        reference
+    }
+
+    fn local_name(&mut self, name: &LocalName) -> u32 {
+        if let Some(reference) = self.local_names.get(name) {
+            return *reference;
+        }
+        let reference = self.interned(name);
+        self.local_names.insert(name.clone(), reference);
+        reference
+    }
+
+    fn namespace(&mut self, namespace: &Namespace) -> u32 {
+        if let Some(reference) = self.namespaces.get(namespace) {
+            return *reference;
+        }
+        let reference = self.interned(namespace);
+        self.namespaces.insert(namespace.clone(), reference);
+        reference
+    }
+
+    fn released(&mut self) {
+        let mut fields = [0; STRIDE];
+        fields[10] = RELEASED;
+        self.nodes.extend_from_slice(&fields);
+    }
+
+    fn encode(&mut self, node: &DomNode) {
+        let mut fields = [0; STRIDE];
+        fields[0] = self.interned(&node.kind);
+        fields[1] = pointer(node.parent);
+        fields[2] = pointer(node.prev);
+        fields[3] = pointer(node.next);
+        fields[4] = self.children.len() as u32;
+        fields[5] = node.children.len() as u32;
+        self.children.extend_from_slice(&node.children);
+        fields[6] = node
+            .name
+            .as_deref()
+            .map_or(0, |name| self.interned(name) + 1);
+        let mut flags = 0;
+        match node.data.as_deref() {
+            Some(data) if data.len() > LARGE_TEXT_BYTES => flags |= OPAQUE,
+            Some(data) => fields[7] = self.string(data) + 1,
+            None => {}
+        }
+        fields[8] = (self.attributes.len() / 2) as u32;
+        fields[9] = node.attribs.len() as u32;
+        for (name, value) in &node.attribs {
+            let Some(value) = value.as_str() else {
+                flags |= OPAQUE;
+                break;
+            };
+            let name = self.interned(name);
+            let value = self.string(value);
+            self.attributes.extend_from_slice(&[name, value]);
+        }
+        fields[11] = ABSENT;
+        fields[12] = ABSENT;
+        let mut rest = Map::new();
+        if let Some(namespace) = &node.namespace {
+            flags |= HAS_NAMESPACE;
+            fields[13] = self.interned(namespace);
+        }
+        match (&node.attribs_namespace, &node.attribs_prefix) {
+            (Some(namespaces), Some(prefixes)) if namespaces.is_empty() && prefixes.is_empty() => {
+                flags |= EMPTY_ATTRIBUTE_MAPS;
+            }
+            (namespaces, prefixes) => {
+                if let Some(namespaces) = namespaces {
+                    rest.insert(
+                        "attribsNamespace".into(),
+                        Value::Object((**namespaces).clone()),
+                    );
+                }
+                if let Some(prefixes) = prefixes {
+                    rest.insert("attribsPrefix".into(), Value::Object((**prefixes).clone()));
+                }
+            }
+        }
+        for (key, value) in &node.extras {
+            let index = value.as_u64().filter(|index| *index < u64::from(ABSENT));
+            match (key.as_str(), value) {
+                ("startIndex", Value::Null) => flags |= HAS_START_INDEX,
+                ("startIndex", _) if index.is_some() => {
+                    flags |= HAS_START_INDEX;
+                    fields[11] = index.unwrap() as u32;
+                }
+                ("endIndex", Value::Null) => flags |= HAS_END_INDEX,
+                ("endIndex", _) if index.is_some() => {
+                    flags |= HAS_END_INDEX;
+                    fields[12] = index.unwrap() as u32;
+                }
+                _ => {
+                    rest.insert(key.clone(), value.clone());
+                }
+            }
+        }
+        if !rest.is_empty() {
+            match serde_json::to_string(&rest) {
+                Ok(json) => fields[14] = self.interned(&json) + 1,
+                Err(_) => flags |= OPAQUE,
+            }
+        }
+        fields[10] = flags;
+        self.nodes.extend_from_slice(&fields);
+    }
+
+    /// Mirrors `NativeDocument::immutable_dom_node` without building records
+    /// for elements, text and comments; other nodes use that record directly.
+    fn encode_immutable(&mut self, document: &NativeDocument) {
+        let html = document.html.borrow();
+        let ids = document.ids.borrow();
+        let reverse = document.reverse.borrow();
+        let Some(html) = html.as_ref() else {
+            return;
+        };
+        let local = |node: Option<ego_tree::NodeRef<'_, Node>>| {
+            node.and_then(|node| reverse.get(&node.id()).copied())
+        };
+        let text_type = self.interned("text");
+        let comment_type = self.interned("comment");
+        let tag_type = self.interned("tag");
+        let script_type = self.interned("script");
+        let style_type = self.interned("style");
+        for (index, node_id) in ids.iter().enumerate() {
+            let Some(node) = html.tree.get(*node_id) else {
+                self.released();
+                continue;
+            };
+            let is_plain_element = matches!(node.value(), Node::Element(element)
+                if element.name.prefix.is_none()
+                    && element.attrs.iter().all(|(name, _)| name.ns.is_empty() && name.prefix.is_none()));
+            let is_direct = is_plain_element
+                || matches!(
+                    node.value(),
+                    Node::Text(_)
+                        | Node::Comment(_)
+                        | Node::Document
+                        | Node::Fragment
+                        | Node::Doctype(_)
+                );
+            if !is_direct {
+                match document.immutable_dom_node(index as u32) {
+                    Some(record) => self.encode(&record),
+                    None => self.released(),
+                }
+                continue;
+            }
+            let mut fields = [0; STRIDE];
+            fields[1] = pointer(local(node.parent()));
+            fields[2] = pointer(local(node.prev_sibling()));
+            fields[3] = pointer(local(node.next_sibling()));
+            fields[4] = self.children.len() as u32;
+            let first_child = self.children.len();
+            self.children.extend(
+                node.children()
+                    .filter_map(|child| reverse.get(&child.id()).copied()),
+            );
+            fields[5] = (self.children.len() - first_child) as u32;
+            fields[8] = (self.attributes.len() / 2) as u32;
+            fields[11] = ABSENT;
+            fields[12] = ABSENT;
+            let mut flags = HAS_START_INDEX | HAS_END_INDEX;
+            match node.value() {
+                Node::Document => {
+                    fields[0] = self.interned("root");
+                    fields[14] = self.interned(quirks_mode_json(html.quirks_mode)) + 1;
+                    flags = 0;
+                }
+                Node::Fragment => {
+                    fields[0] = self.interned("root");
+                    flags = 0;
+                }
+                Node::Doctype(doctype) => {
+                    fields[0] = self.interned("directive");
+                    fields[6] = self.interned("!doctype") + 1;
+                    fields[7] = self.string(&crate::doctype_content(doctype)) + 1;
+                    fields[14] = self.interned(&doctype_fields_json(doctype)) + 1;
+                    flags = 0;
+                }
+                Node::Text(text) => {
+                    fields[0] = text_type;
+                    if text.len() > LARGE_TEXT_BYTES {
+                        flags |= OPAQUE;
+                    } else {
+                        fields[7] = self.string(text) + 1;
+                    }
+                }
+                Node::Comment(comment) => {
+                    fields[0] = comment_type;
+                    if comment.len() > LARGE_TEXT_BYTES {
+                        flags |= OPAQUE;
+                    } else {
+                        fields[7] = self.string(comment) + 1;
+                    }
+                }
+                Node::Element(element) => {
+                    fields[0] = match element.name() {
+                        "script" => script_type,
+                        "style" => style_type,
+                        _ => tag_type,
+                    };
+                    fields[6] = self.local_name(&element.name.local) + 1;
+                    for (attribute, value) in &element.attrs {
+                        let attribute = self.local_name(&attribute.local);
+                        let value = self.string(value);
+                        self.attributes.extend_from_slice(&[attribute, value]);
+                    }
+                    fields[9] = element.attrs.len() as u32;
+                    fields[13] = self.namespace(&element.name.ns);
+                    flags |= HAS_NAMESPACE | EMPTY_ATTRIBUTE_MAPS;
+                }
+                _ => unreachable!("other node types use the record path"),
+            }
+            fields[10] = flags;
+            self.nodes.extend_from_slice(&fields);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn transcodes_text_like_the_standard_library() {
+        for text in [
+            "",
+            "ascii only",
+            "café ©",
+            "번개장터 “quoted” — dash",
+            "emoji 😀 and \u{e800}",
+            "\u{ff}\u{100}",
+            "a\u{10ffff}b",
+        ] {
+            let expected: Vec<u16> = text.encode_utf16().collect();
+            assert_eq!(utf16_units(text), expected, "{text}");
+            assert_eq!(utf16_length(text), expected.len(), "{text}");
+            let latin1 = latin1_bytes(text);
+            assert_eq!(
+                latin1.is_some(),
+                text.chars().all(|character| (character as u32) < 0x100),
+                "{text}"
+            );
+            if let Some(bytes) = latin1 {
+                assert_eq!(
+                    bytes
+                        .iter()
+                        .map(|byte| u16::from(*byte))
+                        .collect::<Vec<_>>(),
+                    expected
+                );
+            }
+        }
+    }
+}

package/native/src/stream.rs

@@ -211,7 +211,7 @@ impl Task for FinishTask {
 }
 enum Parser {
     Html(Box<html5ever::driver::Parser<sink::CompatibleSink>>),
-    Xml(arena::XmlStream),
+    Xml(Box<arena::XmlStream>),
 }
 fn run(
     receiver: mpsc::Receiver<Message>,
@@ -222,7 +222,7 @@ fn run(
     let parser = if arena::xml_mode(&options)
         || options.get("_useHtmlParser2") == Some(&Value::Bool(true))
     {
-        Parser::Xml(arena::XmlStream::new(options.clone()))
+        Parser::Xml(Box::new(arena::XmlStream::new(options.clone())))
     } else {
         let parse_options = html5ever::driver::ParseOpts {
             tokenizer: html5ever::tokenizer::TokenizerOpts {

package/native/src/tokenizer.rs

@@ -23,6 +23,8 @@ struct Builder<'a> {
     xml: bool,
     lower_tags: bool,
     lower_attrs: bool,
+    with_end_indices: bool,
+    recognize_self_closing: Option<bool>,
 }
 impl<'a> Builder<'a> {
     fn new(source: &'a Source<'a>, options: Value) -> Self {
@@ -35,8 +37,13 @@ impl<'a> Builder<'a> {
             .get("lowerCaseAttributeNames")
             .and_then(Value::as_bool)
             .unwrap_or(!xml);
+        let with_end_indices = options.get("withEndIndices") == Some(&Value::Bool(true));
+        let recognize_self_closing = options.get("recognizeSelfClosing").and_then(Value::as_bool);
+        let mut arena = DomArena::from_records(vec![DomNode::new("root")], options);
+        // Growing the arena one doubling at a time copies every node record again.
+        arena.nodes.reserve(source.units.len() / 48);
         Self {
-            arena: DomArena::from_records(vec![DomNode::new("root")], options),
+            arena,
             source,
             parents: vec![0],
             foreign: vec![xml],
@@ -48,6 +55,8 @@ impl<'a> Builder<'a> {
             xml,
             lower_tags,
             lower_attrs,
+            with_end_indices,
+            recognize_self_closing,
         }
     }
     fn slice(&self, start: usize, end: usize) -> String {
@@ -63,7 +72,7 @@ impl<'a> Builder<'a> {
     }
     fn close(&mut self) {
         if let Some(id) = (self.parents.len() > 1).then(|| self.parents.pop().unwrap()) {
-            if self.arena.options.get("withEndIndices") == Some(&Value::Bool(true)) {
+            if self.with_end_indices {
                 self.arena.nodes[id as usize]
                     .extras
                     .insert("endIndex".into(), json!(self.end));
@@ -109,12 +118,14 @@ impl<'a> Builder<'a> {
     fn end_open(&mut self, self_closing: bool) {
         if let Some(pending) = self.pending.take() {
             let is_void = !self.xml && html_parser_void(&pending.name);
-            let kind = if !self.xml && matches!(pending.name.as_str(), "script" | "style") {
-                pending.name.as_str()
-            } else {
-                "tag"
+            let kind = match pending.name.as_str() {
+                "script" if !self.xml => "script",
+                "style" if !self.xml => "style",
+                _ => "tag",
             };
             let mut node = DomNode::new(kind);
+            let lowercase_name = (pending.name.bytes().any(|byte| byte.is_ascii_uppercase()))
+                .then(|| pending.name.to_ascii_lowercase());
             node.name = Some(pending.name);
             node.attribs = pending.attrs;
             let id = append_parsed(
@@ -124,6 +135,17 @@ impl<'a> Builder<'a> {
                 pending.start,
                 self.end,
             );
+            let name = lowercase_name
+                .as_deref()
+                .or(self.arena.nodes[id as usize].name.as_deref())
+                .unwrap_or("");
+            let index = self.arena.name_index.get_or_insert_with(Default::default);
+            match index.get_mut(name) {
+                Some(ids) => ids.push(id),
+                None => {
+                    index.insert(name.to_owned(), vec![id]);
+                }
+            }
             if !is_void && !self_closing {
                 self.parents.push(id);
             }
@@ -370,12 +392,7 @@ impl Builder<'_> {
     }
     fn on_self_closing_tag(&mut self, end: usize) {
         self.end = end;
-        let self_closing = self
-            .arena
-            .options
-            .get("recognizeSelfClosing")
-            .and_then(Value::as_bool)
-            .unwrap_or(self.xml)
+        let self_closing = self.recognize_self_closing.unwrap_or(self.xml)
             || *self.foreign.last().unwrap_or(&self.xml);
         self.end_open(self_closing);
         self.start = end + 1;