frostwork 0.2.0__tar.gz → 0.2.2__tar.gz

{frostwork-0.2.0 → frostwork-0.2.2}/CHANGELOG.md

@@ -3,6 +3,28 @@
 All notable user-facing changes will be recorded here. Frostwork will follow semantic versioning after its
 first public release.
 
+## 0.2.2 (2026-11-05)
+
+### Performance
+
+- `navigation()` no longer takes quadratic time on a document with many unclosed `<meta` tags, e.g.
+  `"<meta " * 80000`, which took over 30 seconds in 0.2.1.
+
+## 0.2.1 (2026-11-05)
+
+### Correctness
+
+- `navigation()` no longer reads a `<base>` or meta refresh written after a self-closed element whose
+  content browsers read as text, e.g. `<script/><base href="…">`: unless it is an SVG or MathML
+  element, browsers ignore the `/` of `<script/>`, `<style/>`, `<title/>`, `<textarea/>`,
+  `<noscript/>` (with scripting enabled), … and read the markup up to the matching end tag as text.
+
+### Performance
+
+- `navigation()` no longer tokenizes a document that cannot hold what was asked for: a `<base>` for
+  the base URL, a `<meta>` with an `http-equiv` attribute for the meta refresh.
+- `frostwork.encoding.bom_encoding()`, `codec_name()` and `decode()` have less call overhead.
+
 ## 0.2.0 (2026-11-05)
 
 ### Backward-incompatible changes

{frostwork-0.2.0 → frostwork-0.2.2}/Cargo.lock

@@ -50,7 +50,7 @@ dependencies = [
 
 [[package]]
 name = "frostwork"
-version = "0.2.0"
+version = "0.2.2"
 dependencies = [
  "chardetng",
  "encoding_rs",

{frostwork-0.2.0 → frostwork-0.2.2}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "frostwork"
-version = "0.2.0"
+version = "0.2.2"
 edition = "2021"
 description = "Treeless, one-pass HTML extraction — CSS/XPath selectors matched on a streaming scan, no DOM."
 license = "Apache-2.0"

{frostwork-0.2.0 → frostwork-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: frostwork
-Version: 0.2.0
+Version: 0.2.2
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3 :: Only

{frostwork-0.2.0 → frostwork-0.2.2}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "maturin"
 
 [project]
 name = "frostwork"
-version = "0.2.0"
+version = "0.2.2"
 description = "Treeless, one-pass HTML extraction — CSS/XPath selectors matched on a streaming scan, no DOM."
 readme = "README.md"
 license = "Apache-2.0"
@@ -89,7 +89,7 @@ module-name = "frostwork._frostwork"
 python-source = "python"
 
 [tool.bumpversion]
-current_version = "0.2.0"
+current_version = "0.2.2"
 commit = true
 tag = true
 tag_name = "{new_version}"

{frostwork-0.2.0 → frostwork-0.2.2}/python/frostwork/encoding.py

@@ -77,7 +77,7 @@ def codec_name(label: str) -> Optional[str]:
     :func:`decode` to decode with it. The name of ``ISO-8859-8-I`` resolves back to ``ISO-8859-8``,
     which decodes the same.
     """
-    name = resolve_label(label)
+    name = _cached_resolve_label(label)
     return None if name is None else _CODEC_NAMES[name]
 
 
@@ -100,7 +100,7 @@ def bom_encoding(body: Buffer) -> Optional[Tuple[str, int]]:
     A UTF-16 document that starts with an XML declaration and no BOM is recognized too, with length
     0. A UTF-32LE BOM reads as a UTF-16LE one, as in browsers.
     """
-    return _bom_encoding(bytes(body))
+    return _bom_encoding(body if type(body) is bytes else bytes(body))
 
 
 def declared_encoding(body: Buffer) -> Optional[str]:
@@ -125,10 +125,11 @@ def decode(body: Buffer, label: str) -> str:
     A BOM is decoded like any other bytes; strip the length :func:`bom_encoding` returns first. Raises
     :exc:`LookupError` if :func:`resolve_label` cannot resolve *label*.
     """
-    name = resolve_label(label)
+    name = _cached_resolve_label(label)
     if name is None:
         raise LookupError(f"unknown encoding: {label}")
-    body = bytes(body)
+    if type(body) is not bytes:
+        body = bytes(body)
     if name == "UTF-8":
         return body.decode("utf-8", "replace")
     if name in _SINGLE_BYTE:

{frostwork-0.2.0 → frostwork-0.2.2}/src/lib.rs

@@ -733,12 +733,10 @@ impl Plan {
     ) -> (FlatColumns, Vec<GroupRows>, Navigation) {
         let doc_url = url::Url::parse(url).ok();
         let (bytes, value_enc, doc_enc) = prepare_bytes(html, encoding, Some(url));
-        // With no fields to extract, a document that never spells a tag that can change the answer
-        // needs no scan. Checked on the prepared bytes, where NUL deletion can have joined a tag name.
-        if self.schema.is_empty()
-            && !has_start_tag(&bytes, b"base")
-            && !(options.meta_refresh && has_start_tag(&bytes, b"meta"))
-        {
+        // With no fields to extract, a document that never spells what can change the answer needs no
+        // scan. Checked on the prepared bytes, where NUL deletion can have joined a name, and which are
+        // ASCII-compatible. A `<base>` only matters to the refresh when there is one.
+        if self.schema.is_empty() && !may_change(&bytes, options.base_url, options.meta_refresh) {
             let base_url = options.base_url.then(|| matcher::document_url(doc_url.as_ref(), url));
             let nav = Navigation { base_url, meta_refresh: None };
             return (Vec::new(), Vec::new(), nav);
@@ -750,11 +748,26 @@ impl Plan {
     }
 }
 
-/// Whether `<name` occurs in `bytes`, ASCII-case-insensitively: a superset of the start tags named `name`.
-fn has_start_tag(bytes: &[u8], name: &[u8]) -> bool {
-    memchr::memchr_iter(b'<', bytes).any(|i| {
-        bytes.get(i + 1..i + 1 + name.len()).is_some_and(|candidate| candidate.eq_ignore_ascii_case(name))
-    })
+/// Whether `bytes` can hold a `<base>`, when `base`, or a `<meta>` with an `http-equiv` attribute,
+/// which every meta refresh needs, when `refresh`. Tags are matched ASCII-case-insensitively wherever
+/// `<name` is followed by the end of a tag name, a superset of the start tags named `name`, and
+/// `http-equiv` anywhere after the first `<meta`, a superset of its attributes. Linear in `bytes`.
+fn may_change(bytes: &[u8], base: bool, refresh: bool) -> bool {
+    let tag_at = |i: usize, name: &[u8]| {
+        let end = i + 1 + name.len();
+        bytes.get(i + 1..end).is_some_and(|candidate| candidate.eq_ignore_ascii_case(name))
+            && bytes.get(end).is_some_and(|&c| !tokenizer::is_name_char(c))
+    };
+    let tags = || memchr::memchr_iter(b'<', bytes);
+    (base && tags().any(|i| tag_at(i, b"base")))
+        || (refresh
+            && tags().find(|&i| tag_at(i, b"meta")).is_some_and(|i| {
+                // `q` is the rarest byte of `http-equiv` in markup, so few candidates reach the comparison.
+                let rest = &bytes[i..];
+                memchr::memchr2_iter(b'q', b'Q', rest).any(|q| {
+                    q >= 6 && rest.get(q - 6..q + 4).is_some_and(|w| w.eq_ignore_ascii_case(b"http-equiv"))
+                })
+            }))
 }
 
 /// The base URL and meta refresh of `html`, as a browser reads them; see

{frostwork-0.2.0 → frostwork-0.2.2}/src/matcher/mod.rs

@@ -2636,6 +2636,9 @@ impl<'a> TokenSink<'a> for Matcher<'a> {
             self.materialize_attrs::<false>(raw_attrs)
         };
         if let Some(nav) = self.nav.as_mut() {
+            if self_closing {
+                nav.self_closed(name, &self.stack, span_start, open_end, self.input.bytes());
+            }
             if nav.wants(name) {
                 nav.start_tag(name, raw_attrs, &self.stack, span_start, self.input.encoding());
             }

{frostwork-0.2.0 → frostwork-0.2.2}/src/matcher/navigation.rs

@@ -17,6 +17,7 @@ use encoding_rs::Encoding;
 use url::Url;
 
 use super::OpenElem;
+use crate::tokenizer::DataMode;
 
 /// What [`crate::navigation`] and [`crate::Plan::extract_with_navigation`] collect.
 #[derive(Clone, Copy, Debug)]
@@ -55,8 +56,11 @@ pub(crate) struct NavCollector {
     /// The refresh to follow, with `None` for the document itself.
     refresh: Option<(u64, Option<Url>)>,
     /// The open elements that start or end foreign content for a browser, outermost first, tracked
-    /// until the base URL is settled.
+    /// until the answer is settled.
     scopes: Vec<OpenScope>,
+    /// Where the text a browser reads after a self-closed `<script/>`, `<title/>`… ends; the tags
+    /// before it are part of that text.
+    text_until: usize,
 }
 
 /// What a browser parses the content of an element as. Inside SVG or MathML, a `<base>` is a foreign
@@ -90,19 +94,23 @@ impl NavCollector {
             base: None,
             refresh: None,
             scopes: Vec::new(),
+            text_until: 0,
         }
     }
 
     /// Whether a start tag named `name` can change the answer. Checked before anything else, so outside
     /// foreign content every other element costs a few comparisons of its name length.
     pub(crate) fn wants(&self, name: &[u8]) -> bool {
-        (!self.base_seen && (!self.scopes.is_empty() || is_any(name, &[b"base", b"svg", b"math"])))
-            || (self.opts.meta_refresh && name.eq_ignore_ascii_case(b"meta"))
+        !self.done()
+            && (!self.scopes.is_empty()
+                || is_any(name, &[b"svg", b"math"])
+                || (!self.base_seen && name.eq_ignore_ascii_case(b"base"))
+                || (self.opts.meta_refresh && name.eq_ignore_ascii_case(b"meta")))
     }
 
     /// An end tag, which in foreign content breaks out of it if it is `</p>` or `</br>`.
     pub(crate) fn end_tag(&mut self, name: &[u8], stack: &[OpenElem]) {
-        if self.base_seen || self.scopes.is_empty() || !is_any(name, &[b"p", b"br"]) {
+        if self.done() || self.scopes.is_empty() || !is_any(name, &[b"p", b"br"]) {
             return;
         }
         self.prune(stack);
@@ -189,6 +197,41 @@ impl NavCollector {
         self.scopes.push(OpenScope { depth, start, scope, broken: false });
     }
 
+    /// A start tag `name` spanning `start..open_end` of `bytes` and ending in `/>`, below the open
+    /// elements `stack`. Called before [`Self::start_tag`].
+    ///
+    /// Outside foreign content, a browser ignores the slash, so the markup up to the element's end tag is
+    /// its text. libxml2 honours the slash, and its raw-text elements are the browser's, less
+    /// `<noscript>`, which is raw text to a browser running scripts.
+    pub(crate) fn self_closed(
+        &mut self,
+        name: &[u8],
+        stack: &[OpenElem],
+        start: usize,
+        open_end: usize,
+        bytes: &[u8],
+    ) {
+        if self.done() || start < self.text_until || self.in_noscript_text(stack) {
+            return;
+        }
+        let mode = match crate::implied_close::data_mode_of(name) {
+            DataMode::Normal if self.opts.scripting && name.eq_ignore_ascii_case(b"noscript") => {
+                DataMode::Rawtext
+            }
+            DataMode::Normal => return,
+            mode => mode,
+        };
+        self.prune(stack);
+        if !self.in_foreign_content() {
+            self.text_until = crate::tokenizer::raw_text_end(bytes, open_end, name, mode, &mut Vec::new()).0;
+        }
+    }
+
+    /// Whether `stack` has a `<noscript>`, whose content is text to a browser running scripts.
+    fn in_noscript_text(&self, stack: &[OpenElem]) -> bool {
+        self.opts.scripting && stack.iter().any(|e| e.tag.eq_ignore_ascii_case(b"noscript"))
+    }
+
     /// Whether the rest of the document can no longer change the answer.
     pub(crate) fn done(&self) -> bool {
         !self.opts.meta_refresh && self.base_seen
@@ -210,21 +253,21 @@ impl NavCollector {
                 .find(|(n, _)| n.eq_ignore_ascii_case(wanted))
                 .map(|&(_, v)| v.map_or(Cow::Borrowed(""), |v| super::decode::decode_attr(v, value_encoding)))
         };
-        if !self.base_seen {
-            self.prune(stack);
-            self.track_scope(name, attr, stack.len(), start);
+        if start < self.text_until {
+            return;
         }
+        self.prune(stack);
+        self.track_scope(name, attr, stack.len(), start);
         let is_base = name.eq_ignore_ascii_case(b"base");
         if !is_base && !name.eq_ignore_ascii_case(b"meta") {
             return;
         }
         // `<template>` content is not part of the document, and a `<noscript>` is text to a browser
         // running scripts. A `<meta>` breaks out of foreign content and stays an HTML element.
-        let ignored = |e: &OpenElem| {
-            e.tag.eq_ignore_ascii_case(b"template")
-                || (self.opts.scripting && e.tag.eq_ignore_ascii_case(b"noscript"))
-        };
-        if stack.iter().any(ignored) || (is_base && self.in_foreign_content()) {
+        if stack.iter().any(|e| e.tag.eq_ignore_ascii_case(b"template"))
+            || self.in_noscript_text(stack)
+            || (is_base && self.in_foreign_content())
+        {
             return;
         }
         if is_base {
@@ -469,6 +512,45 @@ mod tests {
         }
     }
 
+    /// Whether Chromium 143 and Firefox 144 read a `<base>` after a self-closed element whose content
+    /// is text as the base URL (2026-11-05). They agree on every case.
+    #[test]
+    fn navigation_browser_table_self_closed_text() {
+        const B: &str = "<base href='/s/'>";
+        let names = ["script", "style", "title", "textarea", "xmp", "iframe", "noembed", "noframes", "noscript"];
+        for name in names.iter().chain(&["plaintext"]) {
+            let ends = *name != "plaintext";
+            let cases: &[(&str, String, bool)] = &[
+                ("head", page(&format!("<{name}/>{B}"), ""), false),
+                ("body", page("", &format!("<{name}/>{B}")), false),
+                ("closed", page("", &format!("<{name}/>x</{name}>{B}")), ends),
+                ("closed, upper case", page("", &format!("<{}/>x</{name}>{B}", name.to_uppercase())), ends),
+                ("svg", page("", &format!("<svg><{name}/></svg>{B}")), true),
+                ("foreignObject", page("", &format!("<svg><foreignObject><{name}/>{B}")), false),
+                ("mi", page("", &format!("<math><mi><{name}/></mi></math>{B}")), false),
+            ];
+            for (case, html, counts) in cases {
+                let expected = if *counts { "http://h/s/" } else { DOC };
+                assert_eq!(nav(html.as_bytes(), None, true).base_url.as_deref(), Some(expected), "{name}, {case}");
+            }
+        }
+        let cases: &[(&str, &str, bool, bool)] = &[
+            ("noscript, no scripting", "<noscript/>{B}", false, true),
+            ("script in noscript", "<noscript><script/></noscript>{B}", true, true),
+            ("script in noscript, no scripting", "<noscript><script/></noscript>{B}", false, false),
+            ("svg in the text", "<title/><svg></title>{B}", true, true),
+            ("escaped script", "<script/><!--<script></script>{B}</script>", true, false),
+            ("template", "<template><script/></template>{B}", true, false),
+        ];
+        for (case, body, scripting, counts) in cases {
+            let html = page("", &body.replace("{B}", B));
+            let expected = if *counts { "http://h/s/" } else { DOC };
+            assert_eq!(nav(html.as_bytes(), None, *scripting).base_url.as_deref(), Some(expected), "{case}");
+        }
+        let html = page("<script/><meta http-equiv=refresh content='5;url=/r'>", "");
+        assert_eq!(nav(html.as_bytes(), None, true).meta_refresh, None);
+    }
+
     /// The query of a `<base href>` is encoded in the document encoding, as is that of every link
     /// resolved against it (measured on the links; the base URL itself is not observable).
     #[test]
@@ -609,6 +691,35 @@ mod tests {
         assert_eq!(reload.meta_refresh, Some((5, "http://h/".into())));
     }
 
+    /// A document is not scanned when a byte search rules out what was asked for, so the spellings such a
+    /// search could miss still count, and the near misses it must not take for a match answer the same.
+    #[test]
+    fn what_the_skipped_scan_could_miss_still_counts() {
+        let refresh = |html: &[u8]| nav(html, None, true).meta_refresh;
+        let to = |delay, url: &str| Some((delay, url.to_owned()));
+        assert_eq!(refresh(b"<META HTTP-EQUIV=REFRESH CONTENT=1>"), to(1, DOC));
+        assert_eq!(refresh(b"<meta http-equiv=r&#101;fresh content=2>"), to(2, DOC));
+        let utf16: Vec<u8> = "\u{feff}<meta http-equiv=refresh content=3>".encode_utf16().flat_map(u16::to_le_bytes).collect();
+        assert_eq!(refresh(&utf16), to(3, DOC));
+        assert_eq!(refresh(br#"<meta a=">" http-equiv=refresh content=5>"#), to(5, DOC));
+        assert_eq!(refresh(b"<script>'<meta a=\"</script><meta http-equiv=refresh content=6>\"'</script>"), to(6, DOC));
+        assert_eq!(refresh(b"<meta name=refresh content=4><p http-equiv=refresh>http-equiv</p>"), None);
+        let base = |html: &[u8]| nav(html, None, true).base_url;
+        assert_eq!(base(b"<BASE\thref=/a/>"), Some("http://h/a/".into()));
+        assert_eq!(base(b"<base/href=/b/>"), Some("http://h/b/".into()));
+        assert_eq!(base(b"<basefont href=/c/>"), Some(DOC.into()));
+    }
+
+    #[test]
+    fn many_unclosed_meta_tags_take_linear_time() {
+        let evil = "<meta ".repeat(80000);
+        let start = std::time::Instant::now();
+        assert_eq!(nav(evil.as_bytes(), None, true).meta_refresh, None);
+        let good = evil + r#"<meta http-equiv="refresh" content="3;url=/next">"#;
+        assert_eq!(nav(good.as_bytes(), None, true).meta_refresh, Some((3, "http://h/next".into())));
+        assert!(start.elapsed().as_secs() < 2, "{:?}", start.elapsed());
+    }
+
     #[test]
     fn delay_is_whole_seconds() {
         let delay = |content: &str| nav(refresh(content).as_bytes(), None, true).meta_refresh.map(|(d, _)| d);

{frostwork-0.2.0 → frostwork-0.2.2}/src/tokenizer.rs

@@ -113,7 +113,7 @@ fn is_ws(c: u8) -> bool {
 ///
 /// `find_raw_end` ends a rawtext close tag on exactly this set, so anything narrower here leaves the
 /// tokenizer's two name scanners disagreeing with each other as well as with the oracle.
-fn is_name_char(c: u8) -> bool {
+pub(crate) fn is_name_char(c: u8) -> bool {
     !cls::is(c, cls::TAG_NAME_END)
 }
 
@@ -235,6 +235,26 @@ fn find_raw_end<'a>(
     }
 }
 
+/// Where the character data of a `name` element in `mode`, starting at `from`, ends. Returns
+/// (text_end, after_end_tag).
+pub(crate) fn raw_text_end<'a>(
+    b: &'a [u8],
+    from: usize,
+    name: &[u8],
+    mode: DataMode,
+    attr_buf: &mut Vec<(&'a [u8], Option<&'a [u8]>)>,
+) -> (usize, usize) {
+    match mode {
+        // PLAINTEXT has no end: the rest of the document is its text, and libxml2 emits no end tag
+        // (the element is closed by end-of-document, like any element left open).
+        DataMode::Plaintext => (b.len(), b.len()),
+        // `script` needs the escaped/double-escaped states on top of "find the end tag"; every
+        // other mode ends at the first matching end tag.
+        _ if name.eq_ignore_ascii_case(b"script") => find_script_end(b, from, attr_buf),
+        _ => find_raw_end(b, from, name, attr_buf),
+    }
+}
+
 /// Case-insensitive tag-name match at `p`, including the delimiter required after the name.
 fn tag_name_at(b: &[u8], p: usize, prefix: &[u8], name: &[u8]) -> Option<usize> {
     if p + prefix.len() + name.len() > b.len()
@@ -575,15 +595,7 @@ fn handle_start<'a, S: TokenSink<'a>>(
     sink.start_tag(name, attr_buf, self_closing, p, i);
 
     if mode != DataMode::Normal && !self_closing {
-        // PLAINTEXT has no end: the rest of the document is its text, and libxml2 emits no end tag
-        // (the element is closed by end-of-document, like any element left open).
-        let (text_end, after) = match mode {
-            DataMode::Plaintext => (n, n),
-            // `script` needs the escaped/double-escaped states on top of "find the end tag"; every
-            // other mode ends at the first matching end tag.
-            _ if name.eq_ignore_ascii_case(b"script") => find_script_end(b, i, attr_buf),
-            _ => find_raw_end(b, i, name, attr_buf),
-        };
+        let (text_end, after) = raw_text_end(b, i, name, mode, attr_buf);
         if text_end > i {
             sink.text(&b[i..text_end], mode == DataMode::Rcdata, i);
         }