rusteerio 1.0.0 → 1.0.1

package/native/src/tokenizer.rs

@@ -1,6 +1,8 @@
-//! Consume events from an explicitly supplied JavaScript htmlparser2 Tokenizer.
-//! No markup is retokenized here: user event edits directly determine the DOM.
+//! Build htmlparser2-compatible trees from tokenizer events. Events come from
+//! the native tokenizer port or from an explicitly supplied JavaScript
+//! Tokenizer, whose recorded events directly determine the DOM.
 use crate::arena::{append_parsed, html_parser_void, implied_close, DomArena, DomNode};
+use crate::markup_tokenizer::Token;
 use serde_json::{json, Map, Value};
 
 struct Pending {
@@ -8,9 +10,9 @@ struct Pending {
     attrs: Map<String, Value>,
     start: usize,
 }
-struct Builder {
+struct Builder<'a> {
     arena: DomArena,
-    source: Vec<u16>,
+    source: &'a Source<'a>,
     parents: Vec<u32>,
     foreign: Vec<bool>,
     pending: Option<Pending>,
@@ -20,22 +22,41 @@ struct Builder {
     end: usize,
     xml: bool,
     lower_tags: bool,
+    lower_attrs: bool,
 }
-impl Builder {
+impl<'a> Builder<'a> {
+    fn new(source: &'a Source<'a>, options: Value) -> Self {
+        let xml = crate::arena::xml_mode(&options);
+        let lower_tags = options
+            .get("lowerCaseTags")
+            .and_then(Value::as_bool)
+            .unwrap_or(!xml);
+        let lower_attrs = options
+            .get("lowerCaseAttributeNames")
+            .and_then(Value::as_bool)
+            .unwrap_or(!xml);
+        Self {
+            arena: DomArena::from_records(vec![DomNode::new("root")], options),
+            source,
+            parents: vec![0],
+            foreign: vec![xml],
+            pending: None,
+            attribute_name: String::new(),
+            attribute_value: String::new(),
+            start: 0,
+            end: 0,
+            xml,
+            lower_tags,
+            lower_attrs,
+        }
+    }
     fn slice(&self, start: usize, end: usize) -> String {
-        crate::codec::from_units(
-            self.source
-                .get(
-                    start.min(self.source.len())
-                        ..end.min(self.source.len()).max(start.min(self.source.len())),
-                )
-                .unwrap_or_default(),
-        )
+        self.source.slice(start, end)
     }
     fn name(&self, start: usize, end: usize) -> String {
         let name = self.slice(start, end);
         if self.lower_tags {
-            name.to_lowercase()
+            lowercase(name)
         } else {
             name
         }
@@ -61,7 +82,8 @@ impl Builder {
                 self.close();
             }
         }
-        if !(!self.xml && html_parser_void(&name)) {
+        // htmlparser2 tracks foreign content only in HTML mode.
+        if !self.xml && !html_parser_void(&name) {
             if matches!(name.as_str(), "svg" | "math") {
                 self.foreign.push(true);
             } else if matches!(
@@ -145,34 +167,107 @@ impl Builder {
     }
 }
 
+/// Sources from JavaScript, or that contain codec frames, use the lossless
+/// UTF-16 representation; others are plain Unicode.
+fn is_codec_encoded(options: &Value) -> bool {
+    options.get("__utf16Codec") == Some(&Value::Bool(true))
+        || options.get("__losslessUnicode") == Some(&Value::Bool(true))
+}
+
+fn lowercase(mut name: String) -> String {
+    if name.is_ascii() {
+        name.make_ascii_lowercase();
+        name
+    } else {
+        name.to_lowercase()
+    }
+}
+
+/// The source as UTF-16 units, with each unit's byte offset in the original
+/// string so spans can be copied from it instead of re-encoded.
+struct Source<'a> {
+    text: &'a str,
+    units: Vec<u16>,
+    /// Byte offset per unit index, plus one for the end; `u32::MAX` marks an
+    /// index inside a character. `None` for ASCII, where offsets are indices.
+    offsets: Option<Vec<u32>>,
+    codec: bool,
+}
+
+struct IndexedUnits {
+    units: Vec<u16>,
+    offsets: Vec<u32>,
+}
+
+impl crate::codec::UnitSink for IndexedUnits {
+    fn ascii(&mut self, start: usize, run: &[u8]) {
+        self.units.extend(run.iter().map(|value| u16::from(*value)));
+        self.offsets
+            .extend((start..start + run.len()).map(|offset| offset as u32));
+    }
+    fn unit(&mut self, offset: Option<usize>, unit: u16) {
+        self.units.push(unit);
+        self.offsets
+            .push(offset.map_or(u32::MAX, |offset| offset as u32));
+    }
+}
+
+impl<'a> Source<'a> {
+    fn new(text: &'a str, options: &Value) -> Self {
+        let codec = is_codec_encoded(options);
+        if text.is_ascii() {
+            return Self {
+                text,
+                units: text.bytes().map(u16::from).collect(),
+                offsets: None,
+                codec,
+            };
+        }
+        let mut indexed = IndexedUnits {
+            units: Vec::with_capacity(text.len()),
+            offsets: Vec::with_capacity(text.len() + 1),
+        };
+        let framed = codec && crate::codec::has_encoding(text);
+        crate::codec::visit_units(text, framed, &mut indexed);
+        indexed.offsets.push(text.len() as u32);
+        Self {
+            text,
+            units: indexed.units,
+            offsets: Some(indexed.offsets),
+            codec,
+        }
+    }
+    fn slice(&self, start: usize, end: usize) -> String {
+        let start = start.min(self.units.len());
+        let end = end.clamp(start, self.units.len());
+        let Some(offsets) = self.offsets.as_ref() else {
+            return self.text[start..end].to_owned();
+        };
+        match (offsets[start], offsets[end]) {
+            (from, to) if from != u32::MAX && to != u32::MAX => {
+                self.text[from as usize..to as usize].to_owned()
+            }
+            _ if self.codec => crate::codec::from_units(&self.units[start..end]),
+            _ => String::from_utf16_lossy(&self.units[start..end]),
+        }
+    }
+}
+
+/// Parses markup with the native port of htmlparser2's Tokenizer.
+pub(crate) fn parse_markup(source: &str, options: Value) -> DomArena {
+    let source = Source::new(source, &options);
+    let decode = options.get("decodeEntities") != Some(&Value::Bool(false));
+    let mut builder = Builder::new(&source, options);
+    let xml = builder.xml;
+    crate::markup_tokenizer::tokenize(&source.units, xml, decode, |token| builder.handle(token));
+    builder.arena
+}
+
+/// Replays events recorded from a JavaScript htmlparser2 Tokenizer.
 pub(crate) fn parse(source: &str, events: &Value, options: Value) -> DomArena {
-    let xml = crate::arena::xml_mode(&options);
-    let utf16_codec = options.get("__utf16Codec") == Some(&Value::Bool(true));
-    let lower_tags = options
-        .get("lowerCaseTags")
-        .and_then(Value::as_bool)
-        .unwrap_or(!xml);
-    let lower_attrs = options
-        .get("lowerCaseAttributeNames")
-        .and_then(Value::as_bool)
-        .unwrap_or(!xml);
-    let mut builder = Builder {
-        arena: DomArena::from_records(vec![DomNode::new("root")], options),
-        source: if utf16_codec {
-            crate::codec::to_units(source)
-        } else {
-            source.encode_utf16().collect()
-        },
-        parents: vec![0],
-        foreign: vec![xml],
-        pending: None,
-        attribute_name: String::new(),
-        attribute_value: String::new(),
-        start: 0,
-        end: 0,
-        xml,
-        lower_tags,
-    };
+    let source = Source::new(source, &options);
+    let units = source.units.len() as i64;
+    let mut builder = Builder::new(&source, options);
     for event in events.as_array().into_iter().flatten() {
         let Some(tuple) = event.as_array() else {
             continue;
@@ -180,176 +275,237 @@ pub(crate) fn parse(source: &str, events: &Value, options: Value) -> DomArena {
         let Some(event) = tuple.first().and_then(Value::as_str) else {
             continue;
         };
-        let arg = |index: usize| tuple.get(index + 1).and_then(Value::as_u64).unwrap_or(0) as usize;
+        // htmlparser2 reports an unset section start as -1, which its
+        // `slice` resolves from the end of the input.
+        let arg = |index: usize| {
+            let value = tuple.get(index + 1).and_then(Value::as_i64).unwrap_or(0);
+            if value < 0 {
+                (units + value).max(0) as usize
+            } else {
+                value as usize
+            }
+        };
         let (a, b, c) = (arg(0), arg(1), arg(2));
         match event {
-            "ontext" => {
-                let text = builder.slice(a, b);
-                builder.text(text, b);
-            }
+            "ontext" => builder.on_text(a, b, false),
             "ontextentity" => {
-                let text = if a <= 0xffff {
-                    crate::codec::from_units(&[a as u16])
-                } else {
-                    crate::codec::encode_scalar(char::from_u32(a as u32).unwrap_or('\u{fffd}'))
-                };
+                let text = builder.code_point(a);
                 builder.text(text, b);
             }
-            "onopentagname" => {
-                builder.end = b;
-                let name = builder.name(a, b);
-                builder.open(name);
-            }
-            "onopentagend" => {
-                builder.end = a;
-                builder.end_open(false);
-                builder.start = a + 1;
-            }
-            "onselfclosingtag" => {
-                builder.end = a;
-                let self_closing = builder
-                    .arena
-                    .options
-                    .get("recognizeSelfClosing")
-                    .and_then(Value::as_bool)
-                    .unwrap_or(xml)
-                    || *builder.foreign.last().unwrap_or(&xml);
-                builder.end_open(self_closing);
-                builder.start = a + 1;
-            }
-            "onattribname" => {
-                builder.start = a;
-                let name = builder.slice(a, b);
-                builder.attribute_name = if lower_attrs {
-                    name.to_lowercase()
-                } else {
-                    name
-                };
-            }
-            "onattribdata" => {
-                let data = builder.slice(a, b);
-                builder.attribute_value.push_str(&data);
-            }
+            "onopentagname" => builder.on_open_tag_name(a, b),
+            "onopentagend" => builder.on_open_tag_end(a),
+            "onselfclosingtag" => builder.on_self_closing_tag(a),
+            "onattribname" => builder.on_attribute_name(a, b),
+            "onattribdata" => builder.on_attribute_data(a, b, false),
             "onattribentity" => {
-                builder.attribute_value.push_str(&if a <= 0xffff {
-                    crate::codec::from_units(&[a as u16])
-                } else {
-                    crate::codec::encode_scalar(char::from_u32(a as u32).unwrap_or('\u{fffd}'))
-                });
-            }
-            "onattribend" => {
-                builder.end = b;
-                if let Some(pending) = builder.pending.as_mut() {
-                    pending
-                        .attrs
-                        .entry(builder.attribute_name.clone())
-                        .or_insert_with(|| Value::String(builder.attribute_value.clone()));
-                }
-                builder.attribute_value.clear();
-            }
-            "onclosetag" => {
-                builder.end = b;
-                let name = builder.name(a, b);
-                if !xml
-                    && matches!(
-                        name.as_str(),
-                        "svg"
-                            | "math"
-                            | "mi"
-                            | "mo"
-                            | "mn"
-                            | "ms"
-                            | "mtext"
-                            | "annotation-xml"
-                            | "foreignobject"
-                            | "desc"
-                            | "title"
-                    )
-                {
-                    builder.foreign.pop();
-                }
-                if !(!xml && html_parser_void(&name)) {
-                    if let Some(position) = builder.parents.iter().rposition(|id| {
-                        builder
-                            .arena
-                            .node(*id)
-                            .and_then(|node| node.name.as_deref())
-                            == Some(name.as_str())
-                    }) {
-                        while builder.parents.len() > position {
-                            builder.close();
-                        }
-                    } else if !xml && name == "p" {
-                        builder.open(name);
-                        builder.end_open(true);
-                    }
-                } else if !xml && name == "br" {
-                    builder.open(name);
-                    builder.end_open(true);
-                }
-                builder.start = b + 1;
+                let text = builder.code_point(a);
+                builder.attribute_value.push_str(&text);
             }
+            "onattribend" => builder.on_attribute_end(b),
+            "onclosetag" => builder.on_close_tag(a, b),
             "ondeclaration" => builder.instruction(a, b, '!'),
             "onprocessinginstruction" => builder.instruction(a, b, '?'),
-            "oncomment" => {
-                let mut node = DomNode::new("comment");
-                node.data = Some(builder.slice(a, b.saturating_sub(c)));
-                append_parsed(
-                    &mut builder.arena,
-                    *builder.parents.last().unwrap(),
-                    node,
-                    builder.start,
-                    b,
-                );
-                builder.end = b;
-                builder.start = b + 1;
+            "oncomment" => builder.on_comment(a, b, c),
+            "oncdata" => builder.on_cdata(a, b, c),
+            "onend" => builder.on_end(),
+            _ => {}
+        }
+    }
+    builder.arena
+}
+
+impl Builder<'_> {
+    fn handle(&mut self, token: Token) {
+        match token {
+            Token::Text { start, end, decode } => self.on_text(start, end, decode),
+            Token::OpenTagName { start, end } => self.on_open_tag_name(start, end),
+            Token::OpenTagEnd { end } => self.on_open_tag_end(end),
+            Token::SelfClosingTag { end } => self.on_self_closing_tag(end),
+            Token::AttributeName { start, end } => self.on_attribute_name(start, end),
+            Token::AttributeData { start, end, decode } => {
+                self.on_attribute_data(start, end, decode)
             }
-            "oncdata" => {
-                let data = builder.slice(a, b.saturating_sub(c));
-                if xml || builder.arena.options.get("recognizeCDATA") == Some(&Value::Bool(true)) {
-                    let id = append_parsed(
-                        &mut builder.arena,
-                        *builder.parents.last().unwrap(),
-                        DomNode::new("cdata"),
-                        builder.start,
-                        b,
-                    );
-                    let mut text = DomNode::new("text");
-                    text.data = Some(data);
-                    text.parent = Some(id);
-                    text.extras.insert("startIndex".into(), Value::Null);
-                    text.extras.insert(
-                        "endIndex".into(),
-                        if builder.arena.options.get("withEndIndices") == Some(&Value::Bool(true)) {
-                            json!(b)
-                        } else {
-                            Value::Null
-                        },
-                    );
-                    let child = builder.arena.push(text);
-                    builder.arena.nodes[id as usize].children.push(child);
-                } else {
-                    let mut node = DomNode::new("comment");
-                    node.data = Some(format!("[CDATA[{data}]]"));
-                    append_parsed(
-                        &mut builder.arena,
-                        *builder.parents.last().unwrap(),
-                        node,
-                        builder.start,
-                        b,
-                    );
-                }
-                builder.end = b;
-                builder.start = b + 1;
+            Token::AttributeEnd { end } => self.on_attribute_end(end),
+            Token::CloseTag { start, end } => self.on_close_tag(start, end),
+            Token::Comment {
+                start,
+                end,
+                end_offset,
+            } => self.on_comment(start, end, end_offset),
+            Token::Cdata {
+                start,
+                end,
+                end_offset,
+            } => self.on_cdata(start, end, end_offset),
+            Token::Declaration { start, end } => self.instruction(start, end, '!'),
+            Token::ProcessingInstruction { start, end } => self.instruction(start, end, '?'),
+            Token::End => self.on_end(),
+        }
+    }
+    fn code_point(&self, code_point: usize) -> String {
+        if code_point <= 0xffff {
+            crate::codec::from_units(&[code_point as u16])
+        } else {
+            crate::codec::encode_scalar(char::from_u32(code_point as u32).unwrap_or('\u{fffd}'))
+        }
+    }
+    fn on_text(&mut self, start: usize, end: usize, decode: bool) {
+        let raw = self.slice(start, end);
+        let text = if decode && raw.contains('&') {
+            crate::arena::decode_entities_mode(&raw, self.xml, false, self.source.codec)
+        } else {
+            raw
+        };
+        self.text(text, end);
+    }
+    fn on_open_tag_name(&mut self, start: usize, end: usize) {
+        self.end = end;
+        let name = self.name(start, end);
+        self.open(name);
+    }
+    fn on_open_tag_end(&mut self, end: usize) {
+        self.end = end;
+        self.end_open(false);
+        self.start = end + 1;
+    }
+    fn on_self_closing_tag(&mut self, end: usize) {
+        self.end = end;
+        let self_closing = self
+            .arena
+            .options
+            .get("recognizeSelfClosing")
+            .and_then(Value::as_bool)
+            .unwrap_or(self.xml)
+            || *self.foreign.last().unwrap_or(&self.xml);
+        self.end_open(self_closing);
+        self.start = end + 1;
+    }
+    fn on_attribute_name(&mut self, start: usize, end: usize) {
+        self.start = start;
+        let name = self.slice(start, end);
+        self.attribute_name = if self.lower_attrs {
+            lowercase(name)
+        } else {
+            name
+        };
+    }
+    fn on_attribute_data(&mut self, start: usize, end: usize, decode: bool) {
+        let raw = self.slice(start, end);
+        let data = if decode && raw.contains('&') {
+            crate::arena::decode_entities_mode(&raw, self.xml, true, self.source.codec)
+        } else {
+            raw
+        };
+        if self.attribute_value.is_empty() {
+            self.attribute_value = data;
+        } else {
+            self.attribute_value.push_str(&data);
+        }
+    }
+    fn on_attribute_end(&mut self, end: usize) {
+        self.end = end;
+        let value = std::mem::take(&mut self.attribute_value);
+        if let Some(pending) = self.pending.as_mut() {
+            if !pending.attrs.contains_key(&self.attribute_name) {
+                pending
+                    .attrs
+                    .insert(self.attribute_name.clone(), Value::String(value));
             }
-            "onend" => {
-                builder.end = builder.start;
-                while builder.parents.len() > 1 {
-                    builder.close();
+        }
+    }
+    fn on_close_tag(&mut self, start: usize, end: usize) {
+        self.end = end;
+        let name = self.name(start, end);
+        if !self.xml
+            && matches!(
+                name.as_str(),
+                "svg"
+                    | "math"
+                    | "mi"
+                    | "mo"
+                    | "mn"
+                    | "ms"
+                    | "mtext"
+                    | "annotation-xml"
+                    | "foreignobject"
+                    | "desc"
+                    | "title"
+            )
+        {
+            self.foreign.pop();
+        }
+        if !(!self.xml && html_parser_void(&name)) {
+            if let Some(position) = self.parents.iter().rposition(|id| {
+                self.arena.node(*id).and_then(|node| node.name.as_deref()) == Some(name.as_str())
+            }) {
+                while self.parents.len() > position {
+                    self.close();
                 }
+            } else if !self.xml && name == "p" {
+                self.open(name);
+                self.end_open(true);
             }
-            _ => {}
+        } else if !self.xml && name == "br" {
+            self.open(name);
+            self.end_open(true);
+        }
+        self.start = end + 1;
+    }
+    fn on_comment(&mut self, start: usize, end: usize, end_offset: usize) {
+        let mut node = DomNode::new("comment");
+        node.data = Some(self.slice(start, end.saturating_sub(end_offset)));
+        append_parsed(
+            &mut self.arena,
+            *self.parents.last().unwrap(),
+            node,
+            self.start,
+            end,
+        );
+        self.end = end;
+        self.start = end + 1;
+    }
+    fn on_cdata(&mut self, start: usize, end: usize, end_offset: usize) {
+        let data = self.slice(start, end.saturating_sub(end_offset));
+        if self.xml || self.arena.options.get("recognizeCDATA") == Some(&Value::Bool(true)) {
+            let id = append_parsed(
+                &mut self.arena,
+                *self.parents.last().unwrap(),
+                DomNode::new("cdata"),
+                self.start,
+                end,
+            );
+            let mut text = DomNode::new("text");
+            text.data = Some(data);
+            text.parent = Some(id);
+            text.extras.insert("startIndex".into(), Value::Null);
+            text.extras.insert(
+                "endIndex".into(),
+                if self.arena.options.get("withEndIndices") == Some(&Value::Bool(true)) {
+                    json!(end)
+                } else {
+                    Value::Null
+                },
+            );
+            let child = self.arena.push(text);
+            self.arena.nodes[id as usize].children.push(child);
+        } else {
+            let mut node = DomNode::new("comment");
+            node.data = Some(format!("[CDATA[{data}]]"));
+            append_parsed(
+                &mut self.arena,
+                *self.parents.last().unwrap(),
+                node,
+                self.start,
+                end,
+            );
+        }
+        self.end = end;
+        self.start = end + 1;
+    }
+    fn on_end(&mut self) {
+        self.end = self.start;
+        while self.parents.len() > 1 {
+            self.close();
         }
     }
-    builder.arena
 }

package/native/vendor/README.md

@@ -13,8 +13,9 @@ observe the same construction process used by the Rust HTML5 tree builder.
 Ordinary parsing uses the default observer methods.
 
 An opt-in tokenizer flag also escapes private-use characters produced by
-character references. This preserves the distinction between literal Unicode
-and Rusteerio's internal representation of lone JavaScript UTF-16 surrogates.
+character references, emitting each escape and its character as one text token.
+This preserves the distinction between literal Unicode and Rusteerio's internal
+representation of lone JavaScript UTF-16 surrogates.
 The flag is disabled for upstream users of these crates. The original HTML5
 character-reference decoding rules remain in place.
 

package/native/vendor/html5ever-0.35.0/src/tokenizer/mod.rs

@@ -1698,19 +1698,29 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
 
         for i in 0..num_chars {
             let c = chars[i as usize];
-            let prefix = (self.opts.rusteerio_codec && ('\u{e000}'..='\u{e800}').contains(&c))
-                .then_some('\u{e800}');
-            for c in prefix.into_iter().chain(std::iter::once(c)) {
-                match self.state.get() {
-                    states::Data | states::RawData(states::Rcdata) => self.emit_char(c),
-
-                    states::AttributeValue(_) => go!(self: push_value c),
-
-                    _ => panic!(
-                        "state {:?} should not be reachable in process_char_ref",
-                        self.state.get()
-                    ),
-                }
+            let is_escaped =
+                self.opts.rusteerio_codec && ('\u{e000}'..='\u{e800}').contains(&c);
+            match self.state.get() {
+                // The escape and its character form one text token, so a sink
+                // that forwards each token never receives them separately.
+                states::Data | states::RawData(states::Rcdata) if is_escaped => {
+                    let mut text = StrTendril::from_char('\u{e800}');
+                    text.push_char(c);
+                    self.emit_chars(text);
+                },
+                states::Data | states::RawData(states::Rcdata) => self.emit_char(c),
+
+                states::AttributeValue(_) => {
+                    if is_escaped {
+                        go!(self: push_value '\u{e800}');
+                    }
+                    go!(self: push_value c);
+                },
+
+                _ => panic!(
+                    "state {:?} should not be reachable in process_char_ref",
+                    self.state.get()
+                ),
             }
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rusteerio",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Cheerio-compatible HTML processing with a Rust engine and parallel, non-blocking extraction",
   "license": "MIT",
   "repository": {