rusteerio 1.1.3 → 1.1.4

package/Cargo.lock

@@ -720,7 +720,7 @@ dependencies = [
 
 [[package]]
 name = "rusteerio"
-version = "1.1.3"
+version = "1.1.4"
 dependencies = [
  "ego-tree",
  "html5ever",

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rusteerio"
-version = "1.1.3"
+version = "1.1.4"
 edition = "2021"
 description = "Rust HTML5 parser and CSS selector engine for the rusteerio Node.js package"
 license = "MIT"

package/THIRD_PARTY_NOTICES.md

@@ -4,7 +4,7 @@ This file preserves third-party license and attribution notices for the Rust nat
 
 Generated by `node scripts/generate-notices.mjs` from `cargo metadata --locked`, crate license files, and the pinned supplemental sources recorded in `scripts/notices/sources.json`. It includes build dependencies and dependencies for all target platforms; a particular binary may use a subset. Identical notice texts are reproduced once and referenced below. The html5ever, markup5ever and html5gum crates contain local observer-hook and opt-in UTF-16 transport patches; their complete patched source and original notices are included in `native/vendor`. Other dependency source files are unmodified.
 
-Cargo.lock SHA-256: `b4e1749e9f6c31b8ecd8cb108285e93f365a4906f4ef04e8239959f549b7de1a`.
+Cargo.lock SHA-256: `a2ff4645db9ecfd1110ed1ae3af80a43ffaf87ab3c192ce24bee23065684331c`.
 
 ## Source availability
 

package/native/src/snapshot.rs

@@ -49,41 +49,128 @@ const KINDS: [&str; 8] = [
 const SHORT_NAME_SLOTS: usize = 256;
 
 pub(crate) struct NodeSnapshot {
+    /// Consecutive ranges of nodes, encoded separately (in parallel for
+    /// large trees) with references local to each chunk.
+    chunks: Vec<Chunk>,
+}
+
+struct Chunk {
     is_encoded: bool,
     nodes: Vec<u32>,
     children: Vec<u32>,
     attributes: Vec<u32>,
     strings: Vec<u32>,
     text: String,
+    units: u32,
 }
 
 impl NodeSnapshot {
     pub(crate) fn into_js(self, env: &Env) -> Result<JsObject> {
-        let parts = [&self.nodes, &self.children, &self.attributes, &self.strings];
-        let lengths = parts.map(|part| part.len() as u32);
+        let total = |part: fn(&Chunk) -> usize| self.chunks.iter().map(part).sum::<usize>();
+        let lengths = [
+            total(|chunk| chunk.nodes.len()),
+            total(|chunk| chunk.children.len()),
+            total(|chunk| chunk.attributes.len()),
+            total(|chunk| chunk.strings.len()),
+        ];
         let header = [
-            lengths[0],
-            lengths[1],
-            lengths[2],
-            lengths[3],
-            u32::from(self.is_encoded),
+            lengths[0] as u32,
+            lengths[1] as u32,
+            lengths[2] as u32,
+            lengths[3] as u32,
+            u32::from(self.chunks.iter().any(|chunk| chunk.is_encoded)),
         ];
-        let length = header.len() + parts.iter().map(|part| part.len()).sum::<usize>();
+        let length = header.len() + lengths.iter().sum::<usize>();
         let data = uninitialized_words(env, length, |words| {
-            let mut offset = 0;
-            for part in std::iter::once(&header[..]).chain(parts.iter().map(|part| part.as_slice()))
-            {
-                words[offset..offset + part.len()].copy_from_slice(part);
-                offset += part.len();
+            words[..header.len()].copy_from_slice(&header);
+            let (nodes, rest) = words[header.len()..].split_at_mut(lengths[0]);
+            let (children, rest) = rest.split_at_mut(lengths[1]);
+            let (attributes, strings) = rest.split_at_mut(lengths[2]);
+            let mut bases = Bases::default();
+            for chunk in &self.chunks {
+                bases.copy(chunk, nodes, children, attributes, strings);
             }
         })?;
+        let text = if self.chunks.len() == 1 {
+            crate::codec::js_string(env, &self.chunks[0].text)?
+        } else {
+            let mut text =
+                String::with_capacity(self.chunks.iter().map(|chunk| chunk.text.len()).sum());
+            for chunk in &self.chunks {
+                text.push_str(&chunk.text);
+            }
+            crate::codec::js_string(env, &text)?
+        };
         let mut object = env.create_object()?;
         object.set_named_property("data", data)?;
-        object.set_named_property("text", crate::codec::js_string(env, &self.text)?)?;
+        object.set_named_property("text", text)?;
         Ok(object)
     }
 }
 
+/// Where the next chunk's entries start in each combined list.
+#[derive(Default)]
+struct Bases {
+    nodes: usize,
+    children: usize,
+    attributes: usize,
+    strings: usize,
+    units: u32,
+}
+
+impl Bases {
+    /// Copies `chunk` after the chunks before it, shifting its references to
+    /// children, attribute pairs, strings and text by theirs.
+    fn copy(
+        &mut self,
+        chunk: &Chunk,
+        nodes: &mut [u32],
+        children: &mut [u32],
+        attributes: &mut [u32],
+        strings: &mut [u32],
+    ) {
+        let string_base = (self.strings / 2) as u32;
+        let child_base = self.children as u32;
+        let pair_base = (self.attributes / 2) as u32;
+        let target = &mut nodes[self.nodes..self.nodes + chunk.nodes.len()];
+        target.copy_from_slice(&chunk.nodes);
+        if self.nodes != 0 {
+            for fields in target.as_chunks_mut::<STRIDE>().0 {
+                let flags = fields[10];
+                if flags & RELEASED != 0 {
+                    continue;
+                }
+                fields[0] += string_base;
+                fields[4] += child_base;
+                fields[8] += pair_base;
+                for field in [6, 7, 14] {
+                    if fields[field] != 0 {
+                        fields[field] += string_base;
+                    }
+                }
+                if flags & HAS_NAMESPACE != 0 {
+                    fields[13] += string_base;
+                }
+            }
+        }
+        children[self.children..self.children + chunk.children.len()]
+            .copy_from_slice(&chunk.children);
+        let target = &mut attributes[self.attributes..self.attributes + chunk.attributes.len()];
+        for (slot, reference) in target.iter_mut().zip(&chunk.attributes) {
+            *slot = reference + string_base;
+        }
+        let target = &mut strings[self.strings..self.strings + chunk.strings.len()];
+        for (slot, offset) in target.iter_mut().zip(&chunk.strings) {
+            *slot = offset + self.units;
+        }
+        self.nodes += chunk.nodes.len();
+        self.children += chunk.children.len();
+        self.attributes += chunk.attributes.len();
+        self.strings += chunk.strings.len();
+        self.units += chunk.units;
+    }
+}
+
 /// A `Uint32Array` of `length` words that `fill` writes. A Node buffer is
 /// used for its storage because, unlike an `ArrayBuffer`, it is not
 /// zero-filled first; `fill` writes every word before JavaScript can read it.
@@ -167,43 +254,34 @@ struct Builder {
     is_encoded: bool,
 }
 
+/// Nodes per chunk worth encoding on another thread.
+const PARALLEL_CHUNK_NODES: usize = 8192;
+
 pub(crate) fn build(document: &NativeDocument) -> NodeSnapshot {
     let codec = document.options.get("__utf16Codec") == Some(&Value::Bool(true))
         || document.options.get("__losslessUnicode") == Some(&Value::Bool(true));
-    let count = document.slot_count() as usize;
-    let mut builder = Builder {
-        nodes: Vec::with_capacity(count * STRIDE),
-        children: Vec::with_capacity(count),
-        attributes: Vec::new(),
-        strings: Vec::with_capacity(count * 2),
-        text: String::with_capacity(document.content_bytes.get()),
-        units: 0,
-        interned: IdMap::with_capacity_and_hasher(32, Default::default()),
-        kinds: [None; KINDS.len()],
-        short_names: Box::new([(0, 0, 0); SHORT_NAME_SLOTS]),
-        local_names: IdMap::with_capacity_and_hasher(32, Default::default()),
-        namespaces: IdMap::default(),
-        codec,
-        is_encoded: false,
-    };
     if let Some(arena) = document.arena.borrow().as_ref() {
-        builder.nodes.reserve(arena.nodes.len() * STRIDE);
-        for id in 0..arena.nodes.len() {
-            match arena.node(id as u32) {
-                Some(node) => builder.encode(node),
-                None => builder.released(),
-            }
-        }
-    } else {
-        builder.encode_immutable(document);
+        let chunks =
+            crate::runtime::chunked_map(arena.nodes.len(), PARALLEL_CHUNK_NODES, |range| {
+                let mut builder = Builder::new(range.len(), 0, codec);
+                for id in range {
+                    match arena.node(id as u32) {
+                        Some(node) => builder.encode(node),
+                        None => builder.released(),
+                    }
+                }
+                vec![builder.finish()]
+            });
+        return NodeSnapshot { chunks };
     }
+    let mut builder = Builder::new(
+        document.slot_count() as usize,
+        document.content_bytes.get(),
+        codec,
+    );
+    builder.encode_immutable(document);
     NodeSnapshot {
-        is_encoded: builder.is_encoded,
-        nodes: builder.nodes,
-        children: builder.children,
-        attributes: builder.attributes,
-        strings: builder.strings,
-        text: builder.text,
+        chunks: vec![builder.finish()],
     }
 }
 
@@ -232,6 +310,36 @@ fn pointer(value: Option<u32>) -> u32 {
 }
 
 impl Builder {
+    fn new(count: usize, text_bytes: usize, codec: bool) -> Self {
+        Self {
+            nodes: Vec::with_capacity(count * STRIDE),
+            children: Vec::with_capacity(count),
+            attributes: Vec::new(),
+            strings: Vec::with_capacity(count * 2),
+            text: String::with_capacity(text_bytes),
+            units: 0,
+            interned: IdMap::with_capacity_and_hasher(32, Default::default()),
+            kinds: [None; KINDS.len()],
+            short_names: Box::new([(0, 0, 0); SHORT_NAME_SLOTS]),
+            local_names: IdMap::with_capacity_and_hasher(32, Default::default()),
+            namespaces: IdMap::default(),
+            codec,
+            is_encoded: false,
+        }
+    }
+
+    fn finish(self) -> Chunk {
+        Chunk {
+            is_encoded: self.is_encoded,
+            nodes: self.nodes,
+            children: self.children,
+            attributes: self.attributes,
+            strings: self.strings,
+            text: self.text,
+            units: self.units,
+        }
+    }
+
     fn string(&mut self, value: &str) -> u32 {
         let start = self.units;
         self.text.push_str(value);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rusteerio",
-  "version": "1.1.3",
+  "version": "1.1.4",
   "description": "Cheerio-compatible HTML processing with a Rust engine and parallel, non-blocking extraction",
   "license": "MIT",
   "repository": {