rusteerio 0.0.0-stage → 0.1.0

package/native/vendor/html5gum-0.8.4/src/emitters/default.rs

@@ -0,0 +1,207 @@
+//! The default emitter is what powers the simple SAX-like API that you see in the README.
+use std::collections::btree_map::Entry;
+use std::collections::BTreeMap;
+use std::mem::take;
+
+use crate::{Error, HtmlString, Span, SpanBound, Spanned};
+
+use crate::emitters::callback::{Callback, CallbackEmitter, CallbackEvent};
+
+use super::{Emitter, ForwardingEmitter};
+
+#[derive(Debug, Default)]
+struct OurCallback<S: SpanBound> {
+    tag_name: Vec<u8>,
+    tag_start_span: S,
+    attribute_name: Spanned<HtmlString, S>,
+    attribute_map: BTreeMap<HtmlString, Spanned<HtmlString, S>>,
+}
+
+impl<S: SpanBound> Callback<Token<S>, S> for OurCallback<S> {
+    fn handle_event(&mut self, event: CallbackEvent<'_>, span: Span<S>) -> Option<Token<S>> {
+        crate::utils::trace_log!("event: {:?}", event);
+        match event {
+            CallbackEvent::OpenStartTag { name } => {
+                self.tag_name.clear();
+                self.tag_name.extend(name);
+                self.tag_start_span = span.start;
+                None
+            }
+            CallbackEvent::AttributeName { name } => {
+                self.attribute_name.clear();
+                match self.attribute_map.entry(name.to_owned().into()) {
+                    Entry::Occupied(_) => Some(Token::Error(Spanned {
+                        value: Error::DuplicateAttribute,
+                        span,
+                    })),
+                    Entry::Vacant(vacant) => {
+                        self.attribute_name.extend(name);
+                        vacant.insert(Spanned {
+                            value: Default::default(),
+                            span,
+                        });
+                        None
+                    }
+                }
+            }
+            CallbackEvent::AttributeValue { value } => {
+                if !self.attribute_name.is_empty() {
+                    let attr = self.attribute_map.get_mut(&*self.attribute_name).unwrap();
+                    attr.extend(value);
+                    attr.span.end = span.end.offset(1);
+                }
+                None
+            }
+            CallbackEvent::CloseStartTag { self_closing } => Some(Token::StartTag(StartTag {
+                self_closing,
+                name: take(&mut self.tag_name).into(),
+                span: Span {
+                    start: self.tag_start_span,
+                    end: span.end,
+                },
+                attributes: take(&mut self.attribute_map),
+            })),
+            CallbackEvent::EndTag { name } => {
+                self.attribute_map.clear();
+                Some(Token::EndTag(EndTag {
+                    name: name.to_owned().into(),
+                    span,
+                }))
+            }
+            CallbackEvent::String { value } => Some(Token::String(Spanned {
+                value: value.to_owned().into(),
+                span,
+            })),
+            CallbackEvent::Comment { value } => Some(Token::Comment(Spanned {
+                value: value.to_owned().into(),
+                span,
+            })),
+            CallbackEvent::Doctype {
+                name,
+                public_identifier,
+                system_identifier,
+                force_quirks,
+            } => Some(Token::Doctype(Spanned {
+                value: Doctype {
+                    force_quirks,
+                    name: name.to_owned().into(),
+                    public_identifier: public_identifier.map(|x| x.to_owned().into()),
+                    system_identifier: system_identifier.map(|x| x.to_owned().into()),
+                },
+                span,
+            })),
+            CallbackEvent::Error(error) => Some(Token::Error(Spanned { value: error, span })),
+        }
+    }
+}
+
+/// This is the emitter you implicitly use with [crate::Tokenizer::new]. Refer to the [crate
+/// docs](crate) for how usage looks like.
+#[derive(Debug)]
+pub struct DefaultEmitter<S: SpanBound = ()> {
+    inner: CallbackEmitter<OurCallback<S>, Token<S>, S>,
+}
+
+impl Default for DefaultEmitter<()> {
+    fn default() -> Self {
+        Self {
+            inner: Default::default(),
+        }
+    }
+}
+
+impl<S: SpanBound> DefaultEmitter<S> {
+    /// Create a new [`DefaultEmitter`] for a certain [`Span`]type which you can pass to
+    /// [`crate::Tokenizer::new_with_emitter`].
+    #[must_use]
+    pub fn new_with_span() -> Self {
+        Self {
+            inner: Default::default(),
+        }
+    }
+
+    /// Whether to use [crate::naive_next_state] to switch states automatically.
+    ///
+    /// The default is off.
+    pub fn naively_switch_states(&mut self, yes: bool) {
+        self.inner.naively_switch_states(yes)
+    }
+}
+
+impl<S: SpanBound> ForwardingEmitter for DefaultEmitter<S> {
+    type Token = Token<S>;
+
+    fn inner(&mut self) -> &mut impl Emitter<Token = Self::Token> {
+        &mut self.inner
+    }
+}
+
+/// A HTML end/close tag, such as `<p>` or `<a>`.
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct StartTag<S: SpanBound> {
+    /// Whether this tag is self-closing. If it is self-closing, no following [EndTag] should be
+    /// expected.
+    pub self_closing: bool,
+
+    /// The start tag's name, such as `"p"` or `"a"`.
+    pub name: HtmlString,
+
+    /// A mapping for any HTML attributes this start tag may have.
+    ///
+    /// Duplicate attributes are ignored after the first one as per WHATWG spec. Implement your own
+    /// [crate::Emitter] to tweak this behavior.
+    pub attributes: BTreeMap<HtmlString, Spanned<HtmlString, S>>,
+    /// The span of the start tag. Includes exactly the `<p attr="value">`.
+    pub span: Span<S>,
+}
+
+/// A HTML end/close tag, such as `</p>` or `</a>`.
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct EndTag<S: SpanBound> {
+    /// The ending tag's name, such as `"p"` or `"a"`.
+    pub name: HtmlString,
+    /// The span of the end tag. Includes exactly the `</p>`.
+    pub span: Span<S>,
+}
+
+/// A doctype. Some examples:
+///
+/// * `<!DOCTYPE {name}>`
+/// * `<!DOCTYPE {name} PUBLIC '{public_identifier}'>`
+/// * `<!DOCTYPE {name} SYSTEM '{system_identifier}'>`
+/// * `<!DOCTYPE {name} PUBLIC '{public_identifier}' '{system_identifier}'>`
+#[derive(Debug, Eq, PartialEq, Clone)]
+pub struct Doctype {
+    /// The ["force quirks"](https://html.spec.whatwg.org/#force-quirks-flag) flag.
+    pub force_quirks: bool,
+
+    /// The doctype's name. For HTML documents this is "html".
+    pub name: HtmlString,
+
+    /// The doctype's public identifier.
+    pub public_identifier: Option<HtmlString>,
+
+    /// The doctype's system identifier.
+    pub system_identifier: Option<HtmlString>,
+}
+
+/// The token type used by default. You can define your own token type by implementing the
+/// [`crate::Emitter`] trait and using [`crate::Tokenizer::new_with_emitter`].
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum Token<S: SpanBound = ()> {
+    /// A HTML start tag.
+    StartTag(StartTag<S>),
+    /// A HTML end tag.
+    EndTag(EndTag<S>),
+    /// A literal string.
+    String(Spanned<HtmlString, S>),
+    /// A HTML comment.
+    Comment(Spanned<HtmlString, S>),
+    /// A HTML doctype declaration.
+    Doctype(Spanned<Doctype, S>),
+    /// A HTML parsing error.
+    ///
+    /// Can be skipped over, the tokenizer is supposed to recover from the error and continues with
+    /// more tokens afterward.
+    Error(Spanned<Error, S>),
+}

package/native/vendor/html5gum-0.8.4/src/emitters/emitter.rs

@@ -0,0 +1,344 @@
+use crate::{Error, State};
+
+macro_rules! create_emitter {
+    (
+        $(#[$trait_outer:meta])+
+        pub trait $trait_name:ident {
+            $(#[$token:meta])*
+            type Token;
+            $(
+                $(#[$outer:meta])*
+                fn $name:ident($($args:tt)*) $(-> $ret:ty)?
+                $($body:block)?
+            )+
+        }
+
+        $(#[$forward_outer:meta])*
+        pub trait $forward_name:ident {
+        }
+    ) => {
+        $(#[$trait_outer])*
+        pub trait $trait_name {
+            $(#[$token])*
+            type Token;
+
+            $(
+                create_emitter!{ @fun $(#[$outer])* fn $name($($args)*) $(-> $ret)? $($body)? }
+            )+
+        }
+
+        $(#[$forward_outer])*
+        pub trait $forward_name {
+            $(#[$token])*
+            type Token;
+            /// Get a mutable reference to the inner [Emitter].
+            fn inner(&mut self) -> &mut impl Emitter<Token = Self::Token>;
+
+            $(
+                create_emitter!{ @default_forward $(#[$outer])* fn $name($($args)*) $(-> $ret)? }
+            )+
+        }
+
+        impl<T: $forward_name> $trait_name for T {
+            type Token = <Self as $forward_name>::Token;
+
+            $(
+                create_emitter!{ @to_forward $forward_name $(#[$outer])* fn $name($($args)*) $(-> $ret)? }
+            )+
+        }
+    };
+    (@fun $(#[$outer:meta])* fn $name:ident($($args:tt)*) $(-> $ret:ty)? $body:block) => {
+        $(#[$outer])*
+        fn $name($($args)*) $(-> $ret)?
+        $body
+    };
+    (@fun $(#[$outer:meta])* fn $name:ident($($args:tt)*) $(-> $ret:ty)?) => {
+        $(#[$outer])*
+        fn $name($($args)*) $(-> $ret)?;
+    };
+    (@default_forward $(#[$outer:meta])* fn $name:ident(&mut self $(,$($arg_name:ident: $arg_ty:ty),* $(,)?)?) $(-> $ret:ty)?) => {
+        #[allow(unused_attributes)]
+        $(#[$outer])*
+        #[inline]
+        fn $name(&mut self $(, $($arg_name: $arg_ty),*)?) $(-> $ret)? {
+            self.inner().$name($($($arg_name),*)?)
+        }
+    };
+    (@to_forward $forward_name:ident $(#[$outer:meta])* fn $name:ident(&mut self $(,$($arg_name:ident: $arg_ty:ty),* $(,)?)?) $(-> $ret:tt $(<$ret2:tt $(:: $ret3:tt)?>)?)?) => {
+        #[allow(unused_attributes)]
+        $(#[$outer])*
+        #[inline]
+        fn $name(&mut self $(, $($arg_name: $arg_ty),*)?) $(-> $ret $(<$ret2 $(:: $ret3)?>)?)? {
+            <Self as $forward_name>::$name(self $(, $($arg_name),*)?)
+        }
+    };
+
+}
+
+create_emitter! {
+/// An emitter is an object providing methods to the tokenizer to produce tokens.
+///
+/// If you want to mostly wrap another embedder, consider implementing [ForwardingEmitter].
+///
+/// Domain-specific applications of the HTML tokenizer can manually implement this trait to
+/// customize per-token allocations, or avoid them altogether.
+///
+/// An emitter is assumed to have these internal states:
+///
+/// * _last start tag_: The most recently emitted start tag's name
+/// * _current token_: Can be a tag, doctype or comment token. There's only one current token.
+/// * _current attribute_: The currently processed HTML attribute, consisting of two strings for name and value.
+///
+/// The following methods are describing what kind of behavior the WHATWG spec expects, but that
+/// doesn't mean you need to follow it. For example:
+///
+/// * If your usage of the tokenizer will ignore all errors, none of the error handling and
+///   validation requirements apply to you. You can implement `emit_error` as noop and omit all
+///   checks that would emit errors.
+///
+/// * If you don't care about attributes at all, you can make all related methods a noop.
+///
+/// The state machine needs to have a functional implementation of
+/// `current_is_appropriate_end_tag_token` to do correct transitions, however.
+pub trait Emitter {
+    /// The token type emitted by this emitter. This controls what type of values the [`crate::Tokenizer`]
+    /// yields when used as an iterator.
+    type Token;
+
+    /// Set the name of the _last start tag_.
+    ///
+    /// This is primarily for testing purposes. This is *not* supposed to override the tag name of
+    /// the current tag.
+    fn set_last_start_tag(&mut self, last_start_tag: Option<&[u8]>)
+
+    /// The state machine has reached the end of the file. It will soon call `pop_token` for the
+    /// last time.
+    fn emit_eof(&mut self)
+
+    /// A (probably recoverable) parsing error has occured.
+    fn emit_error(&mut self, error: Error)
+
+    /// Whether this emitter cares about errors at all.
+    ///
+    /// If your implementation of `emit_error` is a noop, you can override this function to return
+    /// `false` and decorate it with `#[inline]` to aid the compiler in optimizing out more dead
+    /// code.
+    ///
+    /// This method should return the same value at all times. Returning different values on each
+    /// call might cause bogus errors to be emitted.
+    #[inline]
+    #[must_use]
+    fn should_emit_errors(&mut self) -> bool {
+        // should_emit_errors takes self so that users can implement it as a runtime option in
+        // their program. It takes &mut for no particular reason other than _potential_
+        // convenience, and just because we can provide that guarantee to the emitter.
+        true
+    }
+
+    /// Opt in to rusteerio's internal lossless UTF-16 source protocol.
+    fn rusteerio_codec(&mut self) -> bool { false }
+
+    /// After every state change, the tokenizer calls this method to retrieve a new token that can
+    /// be returned via the tokenizer's iterator interface.
+    fn pop_token(&mut self) -> Option<Self::Token>
+
+    /// Start a new string.
+    fn init_string(&mut self) {
+    }
+
+    /// Emit a bunch of plain characters as character tokens.
+    fn emit_string(&mut self, c: &[u8])
+
+    /// Emit decoded character-reference bytes. The default is identical to
+    /// upstream emission; rusteerio overrides this boundary to escape literal
+    /// private-use scalars in its lossless UTF-16 transport.
+    fn emit_char_ref(&mut self, value: &[u8], is_attribute: bool) {
+        if is_attribute { self.push_attribute_value(value); }
+        else { self.emit_string(value); }
+    }
+
+    /// Set the _current token_ to a start tag.
+    fn init_start_tag(&mut self)
+
+    /// Set the _current token_ to an end tag.
+    fn init_end_tag(&mut self)
+
+    /// Set the _current token_ to a comment.
+    fn init_comment(&mut self)
+
+    /// Emit the _current token_, assuming it is a tag.
+    ///
+    /// Also get the current attribute and append it to the to-be-emitted tag. See docstring for
+    /// [`Emitter::init_attribute`] for how duplicates should be handled.
+    ///
+    /// If a start tag is emitted, update the _last start tag_.
+    ///
+    /// If the current token is not a start/end tag, this method may panic.
+    ///
+    /// The return value is used to switch the tokenizer to a new state. Used in tree building.
+    ///
+    /// If this method always returns `None`, states are never switched, which leads to artifacts
+    /// like contents of `<script>` tags being incorrectly interpreted as HTML.
+    ///
+    /// It's not possible to implement this method correctly in line with the spec without
+    /// implementing a full-blown tree builder as per [tree
+    /// construction](https://html.spec.whatwg.org/#tree-construction), which this crate does not
+    /// offer.
+    ///
+    /// You can approximate correct behavior using [`naive_next_state`], but the caveats of doing
+    /// so are not well-understood.
+    ///
+    /// See the `tokenize_with_state_switches` cargo example for a practical example where this
+    /// matters.
+    #[must_use]
+    fn emit_current_tag(&mut self) -> Option<State>
+
+    /// Emit the _current token_, assuming it is a comment.
+    ///
+    /// If the current token is not a comment, this method may panic.
+    fn emit_current_comment(&mut self)
+
+    /// Emit the _current token_, assuming it is a doctype.
+    ///
+    /// If the current token is not a doctype, this method may panic.
+    fn emit_current_doctype(&mut self)
+
+    /// Assuming the _current token_ is a start tag, set the self-closing flag.
+    ///
+    /// If the current token is not a start or end tag, this method may panic.
+    ///
+    /// If the current token is an end tag, the emitter should emit the
+    /// [`crate::Error::EndTagWithTrailingSolidus`] error.
+    fn set_self_closing(&mut self)
+
+    /// Assuming the _current token_ is a doctype, set its "force quirks" flag to true.
+    ///
+    /// If the current token is not a doctype, this method pay panic.
+    fn set_force_quirks(&mut self)
+
+    /// Assuming the _current token_ is a start/end tag, append a string to the current tag's name.
+    ///
+    /// If the current token is not a start or end tag, this method may panic.
+    fn push_tag_name(&mut self, s: &[u8])
+
+    /// Assuming the _current token_ is a comment, append a string to the comment's contents.
+    ///
+    /// If the current token is not a comment, this method may panic.
+    fn push_comment(&mut self, s: &[u8])
+
+    /// Assuming the _current token_ is a doctype, append a string to the doctype's name.
+    ///
+    /// If the current token is not a doctype, this method may panic.
+    fn push_doctype_name(&mut self, s: &[u8])
+
+    /// Set the _current token_ to a new doctype token:
+    ///
+    /// * the name should be empty
+    /// * the "public identifier" should be null (different from empty)
+    /// * the "system identifier" should be null (different from empty)
+    /// * the "force quirks" flag should be `false`
+    fn init_doctype(&mut self)
+
+    /// Set the _current attribute_ to a new one, starting with empty name and value strings.
+    ///
+    /// The old attribute, if any, should be put on the _current token_. If an attribute with that
+    /// name already exists, WHATWG says the new one should be ignored and a
+    /// [`crate::Error::DuplicateAttribute`] error should be emitted.
+    ///
+    /// If the current token is an end tag token, a [`crate::Error::EndTagWithAttributes`] error should be
+    /// emitted.
+    ///
+    /// If the current token is no tag at all, this method may panic.
+    fn init_attribute(&mut self)
+
+    /// Start with the attribute value after the aatribute's name.
+    ///
+    /// If there is no current attribute, this method may panic.
+    fn init_attribute_value(&mut self) {
+    }
+
+    /// Append a string to the current attribute's name.
+    ///
+    /// If there is no current attribute, this method may panic.
+    fn push_attribute_name(&mut self, s: &[u8])
+
+    /// Append a string to the current attribute's value.
+    ///
+    /// If there is no current attribute, this method may panic.
+    fn push_attribute_value(&mut self, s: &[u8])
+
+    /// Assuming the _current token_ is a doctype, set its "public identifier" to the given string.
+    ///
+    /// If the current token is not a doctype, this method may panic.
+    fn set_doctype_public_identifier(&mut self, value: &[u8])
+
+    /// Assuming the _current token_ is a doctype, set its "system identifier" to the given string.
+    ///
+    /// If the current token is not a doctype, this method may panic.
+    fn set_doctype_system_identifier(&mut self, value: &[u8])
+
+    /// Assuming the _current token_ is a doctype, append a string to its "public identifier" to the given string.
+    ///
+    /// If the current token is not a doctype, this method may panic.
+    fn push_doctype_public_identifier(&mut self, s: &[u8])
+
+    /// Assuming the _current token_ is a doctype, append a string to its "system identifier" to the given string.
+    ///
+    /// If the current token is not a doctype, this method may panic.
+    fn push_doctype_system_identifier(&mut self, s: &[u8])
+
+    /// Start a new tag/comment or something starting with `<`.
+    fn start_open_tag(&mut self) {
+    }
+
+    /// Return true if all of these hold. Return false otherwise.
+    ///
+    /// * the _current token_ is an end tag
+    /// * the _last start tag_ exists
+    /// * the current end tag token's name equals to the last start tag's name.
+    ///
+    /// See also [WHATWG's definition of "appropriate end tag
+    /// token"](https://html.spec.whatwg.org/#appropriate-end-tag-token).
+    fn current_is_appropriate_end_tag_token(&mut self) -> bool
+
+    /// By default, this always returns false and thus
+    /// all CDATA sections are tokenized as bogus comments.
+    ///
+    /// See [markup declaration open
+    /// state](https://html.spec.whatwg.org/multipage/#markup-declaration-open-state).
+    fn adjusted_current_node_present_but_not_in_html_namespace(&mut self) -> bool {
+        false
+    }
+
+    /// Move the reader position by the given amount.
+    ///
+    /// Useful for [crate::Span]s.
+    #[inline]
+    fn move_position(&mut self, _offset: isize) {}
+}
+
+/// An [Emitter] which forwards most methods to an inner emitter.
+///
+/// If you want to modify most methods, consider implementing [Emitter] directly
+pub trait ForwardingEmitter {}
+}
+
+/// Take an educated guess at the next state using the name of a just-now emitted start tag.
+///
+/// This can be used to implement [`Emitter::emit_current_tag`] for most HTML scraping applications,
+/// but is unsuitable for implementing a browser.
+///
+/// The mapping was inspired by `lol-html` which has additional safeguards to detect ambiguous
+/// parsing state: <https://github.com/cloudflare/lol-html/blob/f40a9f767c41caf07851548d7470649a6019548c/src/parser/tree_builder_simulator/mod.rs#L73-L86>
+#[must_use]
+pub fn naive_next_state(tag_name: &[u8]) -> Option<State> {
+    match tag_name {
+        b"textarea" | b"title" => Some(State::RcData),
+        b"plaintext" => Some(State::PlainText),
+        b"script" => Some(State::ScriptData),
+        b"style" | b"iframe" | b"xmp" | b"noembed" | b"noframe" | b"noscript" => {
+            Some(State::RawText)
+        }
+        _ => None,
+    }
+}