rusteerio 0.0.0-stage → 0.1.0

package/native/src/tokenizer.rs

@@ -0,0 +1,355 @@
+//! Consume events from an explicitly supplied JavaScript htmlparser2 Tokenizer.
+//! No markup is retokenized here: user event edits directly determine the DOM.
+use crate::arena::{append_parsed, html_parser_void, implied_close, DomArena, DomNode};
+use serde_json::{json, Map, Value};
+
+struct Pending {
+    name: String,
+    attrs: Map<String, Value>,
+    start: usize,
+}
+struct Builder {
+    arena: DomArena,
+    source: Vec<u16>,
+    parents: Vec<u32>,
+    foreign: Vec<bool>,
+    pending: Option<Pending>,
+    attribute_name: String,
+    attribute_value: String,
+    start: usize,
+    end: usize,
+    xml: bool,
+    lower_tags: bool,
+}
+impl Builder {
+    fn slice(&self, start: usize, end: usize) -> String {
+        crate::codec::from_units(
+            self.source
+                .get(
+                    start.min(self.source.len())
+                        ..end.min(self.source.len()).max(start.min(self.source.len())),
+                )
+                .unwrap_or_default(),
+        )
+    }
+    fn name(&self, start: usize, end: usize) -> String {
+        let name = self.slice(start, end);
+        if self.lower_tags {
+            name.to_lowercase()
+        } else {
+            name
+        }
+    }
+    fn close(&mut self) {
+        if let Some(id) = (self.parents.len() > 1).then(|| self.parents.pop().unwrap()) {
+            if self.arena.options.get("withEndIndices") == Some(&Value::Bool(true)) {
+                self.arena.nodes[id as usize]
+                    .extras
+                    .insert("endIndex".into(), json!(self.end));
+            }
+        }
+    }
+    fn open(&mut self, name: String) {
+        if !self.xml {
+            while self
+                .parents
+                .last()
+                .and_then(|id| self.arena.node(*id))
+                .and_then(|node| node.name.as_deref())
+                .is_some_and(|open| implied_close(&name, open))
+            {
+                self.close();
+            }
+        }
+        if !(!self.xml && html_parser_void(&name)) {
+            if matches!(name.as_str(), "svg" | "math") {
+                self.foreign.push(true);
+            } else if matches!(
+                name.as_str(),
+                "mi" | "mo"
+                    | "mn"
+                    | "ms"
+                    | "mtext"
+                    | "annotation-xml"
+                    | "foreignobject"
+                    | "desc"
+                    | "title"
+            ) {
+                self.foreign.push(false);
+            }
+        }
+        self.pending = Some(Pending {
+            name,
+            attrs: Map::new(),
+            start: self.start,
+        });
+    }
+    fn end_open(&mut self, self_closing: bool) {
+        if let Some(pending) = self.pending.take() {
+            let is_void = !self.xml && html_parser_void(&pending.name);
+            let kind = if !self.xml && matches!(pending.name.as_str(), "script" | "style") {
+                pending.name.as_str()
+            } else {
+                "tag"
+            };
+            let mut node = DomNode::new(kind);
+            node.name = Some(pending.name);
+            node.attribs = pending.attrs;
+            let id = append_parsed(
+                &mut self.arena,
+                *self.parents.last().unwrap(),
+                node,
+                pending.start,
+                self.end,
+            );
+            if !is_void && !self_closing {
+                self.parents.push(id);
+            }
+        }
+    }
+    fn text(&mut self, data: String, end: usize) {
+        self.end = end.saturating_sub(1);
+        let mut node = DomNode::new("text");
+        node.data = Some(data);
+        append_parsed(
+            &mut self.arena,
+            *self.parents.last().unwrap(),
+            node,
+            self.start,
+            self.end,
+        );
+        self.start = end;
+    }
+    fn instruction(&mut self, start: usize, end: usize, prefix: char) {
+        let value = self.slice(start, end);
+        let boundary = value
+            .find(|ch: char| ch.is_whitespace() || ch == '/')
+            .unwrap_or(value.len());
+        let name = if self.lower_tags {
+            value[..boundary].to_lowercase()
+        } else {
+            value[..boundary].to_owned()
+        };
+        let mut node = DomNode::new("directive");
+        node.name = Some(format!("{prefix}{name}"));
+        node.data = Some(format!("{prefix}{value}"));
+        append_parsed(
+            &mut self.arena,
+            *self.parents.last().unwrap(),
+            node,
+            self.start,
+            end,
+        );
+        self.end = end;
+        self.start = end + 1;
+    }
+}
+
+pub(crate) fn parse(source: &str, events: &Value, options: Value) -> DomArena {
+    let xml = crate::arena::xml_mode(&options);
+    let utf16_codec = options.get("__utf16Codec") == Some(&Value::Bool(true));
+    let lower_tags = options
+        .get("lowerCaseTags")
+        .and_then(Value::as_bool)
+        .unwrap_or(!xml);
+    let lower_attrs = options
+        .get("lowerCaseAttributeNames")
+        .and_then(Value::as_bool)
+        .unwrap_or(!xml);
+    let mut builder = Builder {
+        arena: DomArena::from_records(vec![DomNode::new("root")], options),
+        source: if utf16_codec {
+            crate::codec::to_units(source)
+        } else {
+            source.encode_utf16().collect()
+        },
+        parents: vec![0],
+        foreign: vec![xml],
+        pending: None,
+        attribute_name: String::new(),
+        attribute_value: String::new(),
+        start: 0,
+        end: 0,
+        xml,
+        lower_tags,
+    };
+    for event in events.as_array().into_iter().flatten() {
+        let Some(tuple) = event.as_array() else {
+            continue;
+        };
+        let Some(event) = tuple.first().and_then(Value::as_str) else {
+            continue;
+        };
+        let arg = |index: usize| tuple.get(index + 1).and_then(Value::as_u64).unwrap_or(0) as usize;
+        let (a, b, c) = (arg(0), arg(1), arg(2));
+        match event {
+            "ontext" => {
+                let text = builder.slice(a, b);
+                builder.text(text, b);
+            }
+            "ontextentity" => {
+                let text = if a <= 0xffff {
+                    crate::codec::from_units(&[a as u16])
+                } else {
+                    crate::codec::encode_scalar(char::from_u32(a as u32).unwrap_or('\u{fffd}'))
+                };
+                builder.text(text, b);
+            }
+            "onopentagname" => {
+                builder.end = b;
+                let name = builder.name(a, b);
+                builder.open(name);
+            }
+            "onopentagend" => {
+                builder.end = a;
+                builder.end_open(false);
+                builder.start = a + 1;
+            }
+            "onselfclosingtag" => {
+                builder.end = a;
+                let self_closing = builder
+                    .arena
+                    .options
+                    .get("recognizeSelfClosing")
+                    .and_then(Value::as_bool)
+                    .unwrap_or(xml)
+                    || *builder.foreign.last().unwrap_or(&xml);
+                builder.end_open(self_closing);
+                builder.start = a + 1;
+            }
+            "onattribname" => {
+                builder.start = a;
+                let name = builder.slice(a, b);
+                builder.attribute_name = if lower_attrs {
+                    name.to_lowercase()
+                } else {
+                    name
+                };
+            }
+            "onattribdata" => {
+                let data = builder.slice(a, b);
+                builder.attribute_value.push_str(&data);
+            }
+            "onattribentity" => {
+                builder.attribute_value.push_str(&if a <= 0xffff {
+                    crate::codec::from_units(&[a as u16])
+                } else {
+                    crate::codec::encode_scalar(char::from_u32(a as u32).unwrap_or('\u{fffd}'))
+                });
+            }
+            "onattribend" => {
+                builder.end = b;
+                if let Some(pending) = builder.pending.as_mut() {
+                    pending
+                        .attrs
+                        .entry(builder.attribute_name.clone())
+                        .or_insert_with(|| Value::String(builder.attribute_value.clone()));
+                }
+                builder.attribute_value.clear();
+            }
+            "onclosetag" => {
+                builder.end = b;
+                let name = builder.name(a, b);
+                if !xml
+                    && matches!(
+                        name.as_str(),
+                        "svg"
+                            | "math"
+                            | "mi"
+                            | "mo"
+                            | "mn"
+                            | "ms"
+                            | "mtext"
+                            | "annotation-xml"
+                            | "foreignobject"
+                            | "desc"
+                            | "title"
+                    )
+                {
+                    builder.foreign.pop();
+                }
+                if !(!xml && html_parser_void(&name)) {
+                    if let Some(position) = builder.parents.iter().rposition(|id| {
+                        builder
+                            .arena
+                            .node(*id)
+                            .and_then(|node| node.name.as_deref())
+                            == Some(name.as_str())
+                    }) {
+                        while builder.parents.len() > position {
+                            builder.close();
+                        }
+                    } else if !xml && name == "p" {
+                        builder.open(name);
+                        builder.end_open(true);
+                    }
+                } else if !xml && name == "br" {
+                    builder.open(name);
+                    builder.end_open(true);
+                }
+                builder.start = b + 1;
+            }
+            "ondeclaration" => builder.instruction(a, b, '!'),
+            "onprocessinginstruction" => builder.instruction(a, b, '?'),
+            "oncomment" => {
+                let mut node = DomNode::new("comment");
+                node.data = Some(builder.slice(a, b.saturating_sub(c)));
+                append_parsed(
+                    &mut builder.arena,
+                    *builder.parents.last().unwrap(),
+                    node,
+                    builder.start,
+                    b,
+                );
+                builder.end = b;
+                builder.start = b + 1;
+            }
+            "oncdata" => {
+                let data = builder.slice(a, b.saturating_sub(c));
+                if xml || builder.arena.options.get("recognizeCDATA") == Some(&Value::Bool(true)) {
+                    let id = append_parsed(
+                        &mut builder.arena,
+                        *builder.parents.last().unwrap(),
+                        DomNode::new("cdata"),
+                        builder.start,
+                        b,
+                    );
+                    let mut text = DomNode::new("text");
+                    text.data = Some(data);
+                    text.parent = Some(id);
+                    text.extras.insert("startIndex".into(), Value::Null);
+                    text.extras.insert(
+                        "endIndex".into(),
+                        if builder.arena.options.get("withEndIndices") == Some(&Value::Bool(true)) {
+                            json!(b)
+                        } else {
+                            Value::Null
+                        },
+                    );
+                    let child = builder.arena.push(text);
+                    builder.arena.nodes[id as usize].children.push(child);
+                } else {
+                    let mut node = DomNode::new("comment");
+                    node.data = Some(format!("[CDATA[{data}]]"));
+                    append_parsed(
+                        &mut builder.arena,
+                        *builder.parents.last().unwrap(),
+                        node,
+                        builder.start,
+                        b,
+                    );
+                }
+                builder.end = b;
+                builder.start = b + 1;
+            }
+            "onend" => {
+                builder.end = builder.start;
+                while builder.parents.len() > 1 {
+                    builder.close();
+                }
+            }
+            _ => {}
+        }
+    }
+    builder.arena
+}

package/native/src/tracker.rs

@@ -0,0 +1,232 @@
+//! Attach source spans to actual HTML5 sink operations. Scanner metadata is
+//! retained only as a fallback for deferred table text and parser-only closures.
+use crate::diagnostics::DiagnosticContext;
+use serde_json::{Map, Value};
+use std::cell::RefCell;
+use std::collections::{HashMap, HashSet};
+use std::hash::Hash;
+use std::rc::Rc;
+
+pub(crate) struct Locations<K> {
+    context: Rc<DiagnosticContext>,
+    records: RefCell<HashMap<K, Value>>,
+    names: RefCell<HashMap<K, String>>,
+    closed: RefCell<HashSet<K>>,
+}
+impl<K: Copy + Eq + Hash> Locations<K> {
+    pub(crate) fn new(context: Rc<DiagnosticContext>) -> Self {
+        Self {
+            context,
+            records: RefCell::new(HashMap::new()),
+            names: RefCell::new(HashMap::new()),
+            closed: RefCell::new(HashSet::new()),
+        }
+    }
+    pub(crate) fn created(&self, id: K, name: &str) {
+        self.names.borrow_mut().insert(id, name.to_owned());
+        self.closed.borrow_mut().remove(&id);
+        let token = self.context.current_token();
+        let value = if token.kind == "startTag"
+            && token
+                .name
+                .as_deref()
+                .is_some_and(|token| token.eq_ignore_ascii_case(name))
+        {
+            let mut range = self.context.location(token.span.0, token.span.1);
+            if !token.attributes.is_empty() {
+                let attrs: Map<String, Value> = token
+                    .attributes
+                    .into_iter()
+                    .map(|(name, start, _end)| {
+                        let raw = self.context.source_between(start, token.span.1);
+                        let bytes = raw.as_bytes();
+                        let mut end = 0;
+                        while end < bytes.len()
+                            && !bytes[end].is_ascii_whitespace()
+                            && !matches!(bytes[end], b'=' | b'>' | b'/')
+                        {
+                            end += 1;
+                        }
+                        let name_end = end;
+                        while end < bytes.len() && bytes[end].is_ascii_whitespace() {
+                            end += 1;
+                        }
+                        if bytes.get(end) == Some(&b'=') {
+                            end += 1;
+                            while end < bytes.len() && bytes[end].is_ascii_whitespace() {
+                                end += 1;
+                            }
+                            if let Some(q @ (b'\'' | b'"')) = bytes.get(end).copied() {
+                                end += 1;
+                                while end < bytes.len() && bytes[end] != q {
+                                    end += 1;
+                                }
+                                end += (end < bytes.len()) as usize;
+                            } else {
+                                while end < bytes.len()
+                                    && !bytes[end].is_ascii_whitespace()
+                                    && bytes[end] != b'>'
+                                {
+                                    end += 1;
+                                }
+                            }
+                        } else {
+                            end = name_end;
+                        }
+                        (name, self.context.location(start, start + end))
+                    })
+                    .collect();
+                range["attrs"] = Value::Object(attrs);
+            }
+            range["startTag"] = range.clone();
+            if crate::void_name(name) {
+                self.closed.borrow_mut().insert(id);
+            }
+            range
+        } else if matches!(
+            name,
+            "a" | "b"
+                | "big"
+                | "code"
+                | "em"
+                | "font"
+                | "i"
+                | "nobr"
+                | "s"
+                | "small"
+                | "strike"
+                | "strong"
+                | "tt"
+                | "u"
+        ) {
+            let names = self.names.borrow();
+            let records = self.records.borrow();
+            records
+                .iter()
+                .filter(|(other, value)| {
+                    names.get(other).is_some_and(|other| other == name)
+                        && value.get("startTag").is_some()
+                })
+                .max_by_key(|(_, value)| value["startOffset"].as_u64().unwrap_or(0))
+                .map(|(_, value)| {
+                    let mut value = value.clone();
+                    if let Some(object) = value.as_object_mut() {
+                        object.remove("endTag");
+                    }
+                    let tag = value["startTag"].clone();
+                    for field in ["endLine", "endCol", "endOffset"] {
+                        value[field] = tag[field].clone();
+                    }
+                    value
+                })
+                .unwrap_or(Value::Null)
+        } else {
+            Value::Null
+        };
+        self.records.borrow_mut().insert(id, value);
+    }
+    pub(crate) fn closed(&self, id: K) {
+        let token = self.context.current_token();
+        if !matches!(token.kind, "startTag" | "endTag" | "eof") {
+            return;
+        }
+        if self.closed.borrow().contains(&id) {
+            return;
+        }
+        let names = self.names.borrow();
+        let Some(name) = names.get(&id) else {
+            return;
+        };
+        let mut records = self.records.borrow_mut();
+        let Some(Value::Object(record)) = records.get_mut(&id) else {
+            return;
+        };
+        if record.contains_key("endTag") {
+            return;
+        }
+        self.closed.borrow_mut().insert(id);
+        let explicit = token.kind == "endTag"
+            && token
+                .name
+                .as_deref()
+                .is_some_and(|token| token.eq_ignore_ascii_case(name));
+        let end = if explicit { token.span.1 } else { token.span.0 };
+        let point = self.context.location(end, end);
+        for field in ["endLine", "endCol", "endOffset"] {
+            record.insert(field.into(), point[field].clone());
+        }
+        if explicit {
+            record.insert(
+                "endTag".into(),
+                self.context.location(token.span.0, token.span.1),
+            );
+        }
+    }
+    pub(crate) fn closed_as(&self, id: K, name: &str) {
+        let previous = self.names.borrow_mut().insert(id, name.to_owned());
+        self.closed(id);
+        if let Some(previous) = previous {
+            self.names.borrow_mut().insert(id, previous);
+        }
+    }
+    pub(crate) fn token(&self, id: K, kind: &str) {
+        let token = self.context.current_token();
+        if token.kind == kind {
+            self.records
+                .borrow_mut()
+                .insert(id, self.context.location(token.span.0, token.span.1));
+        }
+    }
+    pub(crate) fn text(&self, id: K, data: &str) {
+        let token = self.context.current_token();
+        if token.kind != "text" {
+            return;
+        }
+        let (mut start, end) = self.context.current_span();
+        let raw = self.context.source_between(start, end);
+        let normalized = raw
+            .replace("\r\n", "\n")
+            .replace('\r', "\n")
+            .replace('\0', "\u{fffd}");
+        let decoded = crate::arena::decode_entities_mode(
+            &normalized,
+            false,
+            false,
+            self.context.codec_enabled(),
+        );
+        if normalized != data && decoded != data {
+            if normalized.starts_with('\n')
+                && (&normalized[1..] == data
+                    || crate::arena::decode_entities_mode(
+                        &normalized[1..],
+                        false,
+                        false,
+                        self.context.codec_enabled(),
+                    ) == data)
+            {
+                start += if raw.starts_with("\r\n") { 2 } else { 1 };
+            } else {
+                return;
+            }
+        }
+        let range = self.context.location(start, end);
+        let mut records = self.records.borrow_mut();
+        if let Some(Value::Object(record)) = records.get_mut(&id) {
+            for field in ["endLine", "endCol", "endOffset"] {
+                record.insert(field.into(), range[field].clone());
+            }
+        } else {
+            records.insert(id, range);
+        }
+    }
+    pub(crate) fn get(&self, id: K) -> Option<Value> {
+        self.records.borrow().get(&id).cloned()
+    }
+    pub(crate) fn take(&self) -> HashMap<K, Value> {
+        let ids: Vec<_> = self.names.borrow().keys().copied().collect();
+        for id in ids {
+            self.closed(id);
+        }
+        std::mem::take(&mut *self.records.borrow_mut())
+    }
+}

package/native/vendor/README.md

@@ -0,0 +1,25 @@
+# Vendored HTML5 integration hooks
+
+These directories contain html5ever 0.35.0, markup5ever 0.35.0 and html5gum 0.8.4,
+with their original license files (MIT OR Apache-2.0 for html5ever/markup5ever;
+MIT for html5gum). Their exact patched sources ship in
+the npm package and are selected by the root Cargo manifest's
+`[patch.crates-io]` entries.
+
+Rusteerio adds optional TreeSink observers for the parser's actual open-element
+stack. Stack push, pop, replacement and truncation operations notify the sink
+as they happen. These hooks allow a caller's parse5-compatible tree adapter to
+observe the same construction process used by the Rust HTML5 tree builder.
+Ordinary parsing uses the default observer methods.
+
+An opt-in tokenizer flag also escapes private-use characters produced by
+character references. This preserves the distinction between literal Unicode
+and Rusteerio's internal representation of lone JavaScript UTF-16 surrogates.
+The flag is disabled for upstream users of these crates. The original HTML5
+character-reference decoding rules remain in place.
+
+The patches preserve the HTML5 tree-building algorithms. Review changes against
+the original [html5ever archive](https://crates.io/api/v1/crates/html5ever/0.35.0/download)
+and [markup5ever archive](https://crates.io/api/v1/crates/markup5ever/0.35.0/download),
+and the [html5gum archive](https://crates.io/api/v1/crates/html5gum/0.8.4/download).
+Do not remove or edit the original third-party notices when updating these sources.

package/native/vendor/html5ever-0.35.0/.cargo_vcs_info.json

@@ -0,0 +1,6 @@
+{
+  "git": {
+    "sha1": "6db7bbf1c4dfaecd28731950d427e40c8b911bef"
+  },
+  "path_in_vcs": "html5ever"
+}