html-to-markdown 2.4.0__tar.gz → 2.4.2__tar.gz

{html_to_markdown-2.4.0 → html_to_markdown-2.4.2}/Cargo.lock

@@ -249,9 +249,9 @@ checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
 
 [[package]]
 name = "clap_mangen"
-version = "0.2.30"
+version = "0.2.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "263c8214a8e0cb8129f3c62036c50e9c6e15c7bd364c42e0437c492b9293f778"
+checksum = "439ea63a92086df93893164221ad4f24142086d535b3a0957b9b9bea2dc86301"
 dependencies = [
  "clap",
  "roff",
@@ -386,9 +386,9 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67773048316103656a637612c4a62477603b777d91d9c62ff2290f9cde178fdb"
+checksum = "59c9b8bdf64ee849747c1b12eb861d21aa47fa161564f48332f1afe2373bf899"
 dependencies = [
  "ctor-proc-macro",
  "dtor",
@@ -396,9 +396,9 @@ dependencies = [
 
 [[package]]
 name = "ctor-proc-macro"
-version = "0.0.6"
+version = "0.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2931af7e13dc045d8e9d26afccc6fa115d64e115c9c84b1166288b46f6782c2"
+checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
 
 [[package]]
 name = "cty"
@@ -507,9 +507,9 @@ checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"
 
 [[package]]
 name = "flate2"
-version = "1.1.4"
+version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9"
+checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
@@ -595,7 +595,7 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-cli"
-version = "2.4.0"
+version = "2.4.2"
 dependencies = [
  "assert_cmd",
  "clap",
@@ -609,7 +609,7 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-node"
-version = "2.4.0"
+version = "2.4.2"
 dependencies = [
  "html-to-markdown-rs",
  "mimalloc-rust",
@@ -620,7 +620,7 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-py"
-version = "2.4.0"
+version = "2.4.2"
 dependencies = [
  "base64",
  "html-to-markdown-rs",
@@ -630,7 +630,7 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-rs"
-version = "2.4.0"
+version = "2.4.2"
 dependencies = [
  "ammonia",
  "base64",
@@ -647,7 +647,7 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-wasm"
-version = "2.4.0"
+version = "2.4.2"
 dependencies = [
  "console_error_panic_hook",
  "getrandom",
@@ -989,9 +989,9 @@ dependencies = [
 
 [[package]]
 name = "napi"
-version = "3.3.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1b74e3dce5230795bb4d2821b941706dee733c7308752507254b0497f39cad7"
+checksum = "c3a1135cfe16ca43ac82ac05858554fc39c037d8e4592f2b4a83d7ef8e822f43"
 dependencies = [
  "bitflags",
  "ctor",
@@ -1003,15 +1003,15 @@ dependencies = [
 
 [[package]]
 name = "napi-build"
-version = "2.2.3"
+version = "2.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcae8ad5609d14afb3a3b91dee88c757016261b151e9dcecabf1b2a31a6cab14"
+checksum = "3ae82775d1b06f3f07efd0666e59bbc175da8383bc372051031d7a447e94fbea"
 
 [[package]]
 name = "napi-derive"
-version = "3.2.5"
+version = "3.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7552d5a579b834614bbd496db5109f1b9f1c758f08224b0dee1e408333adf0d0"
+checksum = "78665d6bdf10e9a4e6b38123efb0f66962e6197c1aea2f07cff3f159a374696d"
 dependencies = [
  "convert_case",
  "ctor",
@@ -1023,9 +1023,9 @@ dependencies = [
 
 [[package]]
 name = "napi-derive-backend"
-version = "2.2.0"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f6a81ac7486b70f2532a289603340862c06eea5a1e650c1ffeda2ce1238516a"
+checksum = "42d55d01423e7264de3acc13b258fa48ca7cf38a4d25db848908ec3c1304a85a"
 dependencies = [
  "convert_case",
  "proc-macro2",
@@ -1036,9 +1036,9 @@ dependencies = [
 
 [[package]]
 name = "napi-sys"
-version = "3.0.0"
+version = "3.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e4e7135a8f97aa0f1509cce21a8a1f9dcec1b50d8dee006b48a5adb69a9d64d"
+checksum = "1ed8f0e23a62a3ce0fbb6527cdc056e9282ddd9916b068c46f8923e18eed5ee6"
 dependencies = [
  "libloading",
 ]
@@ -1263,9 +1263,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.101"
+version = "1.0.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
+checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8"
 dependencies = [
  "unicode-ident",
 ]
@@ -1609,9 +1609,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.107"
+version = "2.0.108"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b"
+checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
 dependencies = [
  "proc-macro2",
  "quote",

{html_to_markdown-2.4.0 → html_to_markdown-2.4.2}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/html-to-markdown-py"]
 
 [workspace.package]
-version = "2.4.0"
+version = "2.4.2"
 edition = "2021"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
 license = "MIT"
@@ -15,7 +15,7 @@ rust-version = "1.80"
 
 [workspace.dependencies]
 # Core library
-html-to-markdown-rs = { version = "2.4.0", path = "crates/html-to-markdown" }
+html-to-markdown-rs = { version = "2.4.2", path = "crates/html-to-markdown" }
 
 # HTML parsing and sanitization
 tl = "0.7"

{html_to_markdown-2.4.0 → html_to_markdown-2.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: html-to-markdown
-Version: 2.4.0
+Version: 2.4.2
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -162,9 +162,12 @@ Key fields (see docstring for full matrix):
 
 ### `PreprocessingOptions`
 
-- `enabled`: enable HTML sanitisation
-- `preset`: `"minimal" | "standard" | "aggressive"`
-- `remove_navigation`, `remove_forms`
+- `enabled`: enable HTML sanitisation (default: `True` since v2.4.2 for robust malformed HTML handling)
+- `preset`: `"minimal" | "standard" | "aggressive"` (default: `"standard"`)
+- `remove_navigation`: remove navigation elements (default: `True`)
+- `remove_forms`: remove form elements (default: `True`)
+
+**Note:** As of v2.4.2, preprocessing is enabled by default to ensure robust handling of malformed HTML (e.g., bare angle brackets like `1<2` in content). Set `enabled=False` if you need minimal preprocessing.
 
 ### `InlineImageConfig`
 

{html_to_markdown-2.4.0 → html_to_markdown-2.4.2}/README_PYPI.md

@@ -129,9 +129,12 @@ Key fields (see docstring for full matrix):
 
 ### `PreprocessingOptions`
 
-- `enabled`: enable HTML sanitisation
-- `preset`: `"minimal" | "standard" | "aggressive"`
-- `remove_navigation`, `remove_forms`
+- `enabled`: enable HTML sanitisation (default: `True` since v2.4.2 for robust malformed HTML handling)
+- `preset`: `"minimal" | "standard" | "aggressive"` (default: `"standard"`)
+- `remove_navigation`: remove navigation elements (default: `True`)
+- `remove_forms`: remove form elements (default: `True`)
+
+**Note:** As of v2.4.2, preprocessing is enabled by default to ensure robust handling of malformed HTML (e.g., bare angle brackets like `1<2` in content). Set `enabled=False` if you need minimal preprocessing.
 
 ### `InlineImageConfig`
 

{html_to_markdown-2.4.0 → html_to_markdown-2.4.2}/crates/html-to-markdown/src/converter.rs

@@ -47,6 +47,9 @@ use std::collections::BTreeMap;
 #[cfg(feature = "inline-images")]
 use std::rc::Rc;
 
+use std::borrow::Cow;
+use std::str;
+
 use crate::error::Result;
 #[cfg(feature = "inline-images")]
 use crate::inline_images::{InlineImageCollector, InlineImageFormat, InlineImageSource};
@@ -971,7 +974,12 @@ fn convert_html_impl(
         .replace("<hr/>", "<hr>")
         .replace("<img/>", "<img>");
 
-    let dom = tl::parse(&html, tl::ParserOptions::default())
+    // Escape malformed angle brackets in text content to prevent parser failures
+    let html = escape_malformed_angle_brackets(&html);
+
+    let html = strip_script_and_style_sections(&html);
+
+    let dom = tl::parse(html.as_ref(), tl::ParserOptions::default())
         .map_err(|_| crate::error::ConversionError::ParseError("Failed to parse HTML".to_string()))?;
 
     let parser = dom.parser();
@@ -1075,6 +1083,237 @@ fn convert_html_impl(
     }
 }
 
+/// Escape malformed angle brackets in HTML that are not part of valid tags.
+///
+/// This function ensures robust parsing by escaping bare `<` and `>` characters
+/// that appear in text content and are not part of HTML tags. This prevents
+/// parser failures on malformed HTML like "1<2" or comparisons in text.
+///
+/// # Examples
+///
+/// - `1<2` becomes `1&lt;2`
+/// - `<div>1<2</div>` becomes `<div>1&lt;2</div>`
+/// - `<script>1 < 2</script>` remains unchanged (handled by script stripping)
+fn escape_malformed_angle_brackets(input: &str) -> Cow<'_, str> {
+    let bytes = input.as_bytes();
+    let len = bytes.len();
+    let mut idx = 0;
+    let mut last = 0;
+    let mut output: Option<String> = None;
+
+    while idx < len {
+        if bytes[idx] == b'<' {
+            // Check if this is a valid tag start
+            if idx + 1 < len {
+                let next = bytes[idx + 1];
+
+                // Valid tag patterns: <tagname, </tagname, <!doctype, <!--
+                let is_valid_tag = match next {
+                    b'!' => {
+                        // DOCTYPE or comment
+                        idx + 2 < len
+                            && (bytes[idx + 2] == b'-'
+                                || bytes[idx + 2].is_ascii_alphabetic()
+                                || bytes[idx + 2].is_ascii_uppercase())
+                    }
+                    b'/' => {
+                        // Closing tag
+                        idx + 2 < len && (bytes[idx + 2].is_ascii_alphabetic() || bytes[idx + 2].is_ascii_uppercase())
+                    }
+                    b'?' => {
+                        // XML declaration
+                        true
+                    }
+                    c if c.is_ascii_alphabetic() || c.is_ascii_uppercase() => {
+                        // Opening tag
+                        true
+                    }
+                    _ => false,
+                };
+
+                if !is_valid_tag {
+                    // This is a bare `<` that should be escaped
+                    let out = output.get_or_insert_with(|| String::with_capacity(input.len() + 4));
+                    out.push_str(&input[last..idx]);
+                    out.push_str("&lt;");
+                    last = idx + 1;
+                }
+            } else {
+                // `<` at end of string - escape it
+                let out = output.get_or_insert_with(|| String::with_capacity(input.len() + 4));
+                out.push_str(&input[last..idx]);
+                out.push_str("&lt;");
+                last = idx + 1;
+            }
+        }
+        idx += 1;
+    }
+
+    if let Some(mut out) = output {
+        if last < input.len() {
+            out.push_str(&input[last..]);
+        }
+        Cow::Owned(out)
+    } else {
+        Cow::Borrowed(input)
+    }
+}
+
+fn strip_script_and_style_sections(input: &str) -> Cow<'_, str> {
+    const TAGS: [&[u8]; 2] = [b"script", b"style"];
+    const SVG: &[u8] = b"svg";
+
+    let bytes = input.as_bytes();
+    let len = bytes.len();
+    let mut idx = 0;
+    let mut last = 0;
+    let mut output: Option<String> = None;
+    let mut svg_depth = 0usize;
+
+    while idx < len {
+        if bytes[idx] == b'<' {
+            if matches_tag_start(bytes, idx + 1, SVG) {
+                if let Some(open_end) = find_tag_end(bytes, idx + 1 + SVG.len()) {
+                    svg_depth += 1;
+                    idx = open_end;
+                    continue;
+                }
+            } else if matches_end_tag_start(bytes, idx + 1, SVG) {
+                if let Some(close_end) = find_tag_end(bytes, idx + 2 + SVG.len()) {
+                    if svg_depth > 0 {
+                        svg_depth = svg_depth.saturating_sub(1);
+                    }
+                    idx = close_end;
+                    continue;
+                }
+            }
+
+            if svg_depth == 0 {
+                let mut handled = false;
+                for tag in TAGS {
+                    if matches_tag_start(bytes, idx + 1, tag) {
+                        if let Some(open_end) = find_tag_end(bytes, idx + 1 + tag.len()) {
+                            let remove_end = find_closing_tag(bytes, open_end, tag).unwrap_or(len);
+                            let out = output.get_or_insert_with(|| String::with_capacity(input.len()));
+                            out.push_str(&input[last..idx]);
+                            out.push_str(&input[idx..open_end]);
+                            out.push_str("</");
+                            out.push_str(str::from_utf8(tag).unwrap());
+                            out.push('>');
+
+                            last = remove_end;
+                            idx = remove_end;
+                            handled = true;
+                        }
+                    }
+
+                    if handled {
+                        break;
+                    }
+                }
+
+                if handled {
+                    continue;
+                }
+            }
+        }
+
+        idx += 1;
+    }
+
+    if let Some(mut out) = output {
+        if last < input.len() {
+            out.push_str(&input[last..]);
+        }
+        Cow::Owned(out)
+    } else {
+        Cow::Borrowed(input)
+    }
+}
+
+fn matches_tag_start(bytes: &[u8], mut start: usize, tag: &[u8]) -> bool {
+    if start >= bytes.len() {
+        return false;
+    }
+
+    if start + tag.len() > bytes.len() {
+        return false;
+    }
+
+    if !bytes[start..start + tag.len()].eq_ignore_ascii_case(tag) {
+        return false;
+    }
+
+    start += tag.len();
+
+    match bytes.get(start) {
+        Some(b'>' | b'/' | b' ' | b'\t' | b'\n' | b'\r') => true,
+        Some(_) => false,
+        None => true,
+    }
+}
+
+fn find_tag_end(bytes: &[u8], mut idx: usize) -> Option<usize> {
+    let len = bytes.len();
+    let mut in_quote: Option<u8> = None;
+
+    while idx < len {
+        match bytes[idx] {
+            b'"' | b'\'' => {
+                if let Some(current) = in_quote {
+                    if current == bytes[idx] {
+                        in_quote = None;
+                    }
+                } else {
+                    in_quote = Some(bytes[idx]);
+                }
+            }
+            b'>' if in_quote.is_none() => return Some(idx + 1),
+            _ => {}
+        }
+        idx += 1;
+    }
+
+    None
+}
+
+fn find_closing_tag(bytes: &[u8], mut idx: usize, tag: &[u8]) -> Option<usize> {
+    let len = bytes.len();
+    let mut depth = 1usize;
+
+    while idx < len {
+        if bytes[idx] == b'<' {
+            if matches_tag_start(bytes, idx + 1, tag) {
+                if let Some(next) = find_tag_end(bytes, idx + 1 + tag.len()) {
+                    depth += 1;
+                    idx = next;
+                    continue;
+                }
+            } else if matches_end_tag_start(bytes, idx + 1, tag) {
+                if let Some(close) = find_tag_end(bytes, idx + 2 + tag.len()) {
+                    depth -= 1;
+                    if depth == 0 {
+                        return Some(close);
+                    }
+                    idx = close;
+                    continue;
+                }
+            }
+        }
+
+        idx += 1;
+    }
+
+    None
+}
+
+fn matches_end_tag_start(bytes: &[u8], start: usize, tag: &[u8]) -> bool {
+    if start >= bytes.len() || bytes[start] != b'/' {
+        return false;
+    }
+    matches_tag_start(bytes, start + 1, tag)
+}
+
 /// Check if an element is inline (not block-level).
 fn is_inline_element(tag_name: &str) -> bool {
     matches!(
@@ -4002,6 +4241,22 @@ mod tests {
         assert_eq!(calculate_list_continuation_indent(4), 7);
     }
 
+    #[test]
+    fn strips_script_sections_without_removing_following_content() {
+        let input = "<div>before</div><script>1 < 2</script><p>after</p>";
+        let stripped = strip_script_and_style_sections(input);
+        assert_eq!(stripped, "<div>before</div><script></script><p>after</p>");
+    }
+
+    #[test]
+    fn strips_multiline_script_sections() {
+        let input = "<html>\n<script>1 < 2</script>\nContent\n</html>";
+        let stripped = strip_script_and_style_sections(input);
+        assert!(stripped.contains("Content"));
+        assert!(stripped.contains("<script"));
+        assert!(!stripped.contains("1 < 2"));
+    }
+
     #[test]
     fn test_add_list_continuation_indent_blank_line() {
         let opts = ConversionOptions::default();
@@ -4049,4 +4304,98 @@ mod tests {
         add_list_continuation_indent(&mut output, 1, false, &opts);
         assert_eq!(output, "* First\n  ");
     }
+
+    #[test]
+    fn test_escape_malformed_angle_brackets_bare() {
+        let input = "1<2";
+        let escaped = escape_malformed_angle_brackets(input);
+        assert_eq!(escaped, "1&lt;2");
+    }
+
+    #[test]
+    fn test_escape_malformed_angle_brackets_in_text() {
+        let input = "<html>1<2 Content</html>";
+        let escaped = escape_malformed_angle_brackets(input);
+        assert_eq!(escaped, "<html>1&lt;2 Content</html>");
+    }
+
+    #[test]
+    fn test_escape_malformed_angle_brackets_multiple() {
+        let input = "1 < 2 < 3";
+        let escaped = escape_malformed_angle_brackets(input);
+        assert_eq!(escaped, "1 &lt; 2 &lt; 3");
+    }
+
+    #[test]
+    fn test_escape_malformed_angle_brackets_preserves_valid_tags() {
+        let input = "<div>content</div>";
+        let escaped = escape_malformed_angle_brackets(input);
+        assert_eq!(escaped, "<div>content</div>");
+    }
+
+    #[test]
+    fn test_escape_malformed_angle_brackets_mixed() {
+        let input = "<div>1<2</div><p>3<4</p>";
+        let escaped = escape_malformed_angle_brackets(input);
+        assert_eq!(escaped, "<div>1&lt;2</div><p>3&lt;4</p>");
+    }
+
+    #[test]
+    fn test_escape_malformed_angle_brackets_at_end() {
+        let input = "test<";
+        let escaped = escape_malformed_angle_brackets(input);
+        assert_eq!(escaped, "test&lt;");
+    }
+
+    #[test]
+    fn test_escape_malformed_angle_brackets_preserves_comments() {
+        let input = "<!-- comment -->1<2";
+        let escaped = escape_malformed_angle_brackets(input);
+        assert_eq!(escaped, "<!-- comment -->1&lt;2");
+    }
+
+    #[test]
+    fn test_escape_malformed_angle_brackets_preserves_doctype() {
+        let input = "<!DOCTYPE html>1<2";
+        let escaped = escape_malformed_angle_brackets(input);
+        assert_eq!(escaped, "<!DOCTYPE html>1&lt;2");
+    }
+
+    #[test]
+    fn test_convert_with_malformed_angle_brackets() {
+        // Test the full conversion pipeline (issue #94)
+        let html = "<html>1<2\nContent</html>";
+        let result = convert_html(html, &ConversionOptions::default()).unwrap();
+        assert!(
+            result.contains("Content"),
+            "Result should contain 'Content': {:?}",
+            result
+        );
+        assert!(
+            result.contains("1<2") || result.contains("1&lt;2"),
+            "Result should contain escaped or unescaped comparison"
+        );
+    }
+
+    #[test]
+    fn test_convert_with_malformed_angle_brackets_in_div() {
+        let html = "<html><div>1<2</div><div>Content</div></html>";
+        let result = convert_html(html, &ConversionOptions::default()).unwrap();
+        assert!(
+            result.contains("Content"),
+            "Result should contain 'Content': {:?}",
+            result
+        );
+    }
+
+    #[test]
+    fn test_convert_with_multiple_malformed_angle_brackets() {
+        let html = "<html>1 < 2 < 3<p>Content</p></html>";
+        let result = convert_html(html, &ConversionOptions::default()).unwrap();
+        assert!(
+            result.contains("Content"),
+            "Result should contain 'Content': {:?}",
+            result
+        );
+    }
 }

{html_to_markdown-2.4.0 → html_to_markdown-2.4.2}/crates/html-to-markdown/src/options.rs

@@ -258,7 +258,7 @@ pub struct PreprocessingOptions {
 impl Default for PreprocessingOptions {
     fn default() -> Self {
         Self {
-            enabled: false,
+            enabled: true,
             preset: PreprocessingPreset::default(),
             remove_navigation: true,
             remove_forms: true,

{html_to_markdown-2.4.0 → html_to_markdown-2.4.2}/html_to_markdown/__init__.py

@@ -49,4 +49,4 @@ __all__ = [
     "markdownify",
 ]
 
-__version__ = "2.4.0"
+__version__ = "2.4.2"

{html_to_markdown-2.4.0 → html_to_markdown-2.4.2}/html_to_markdown/options.py

@@ -128,8 +128,8 @@ class PreprocessingOptions:
         ... )
     """
 
-    enabled: bool = False
-    """Whether to enable HTML preprocessing (disabled by default for minimal transformation)."""
+    enabled: bool = True
+    """Whether to enable HTML preprocessing (enabled by default for robust handling of malformed HTML)."""
 
     preset: Literal["minimal", "standard", "aggressive"] = "standard"
     """Preprocessing aggressiveness level."""

{html_to_markdown-2.4.0 → html_to_markdown-2.4.2}/pyproject.toml

@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "html-to-markdown"
-version = "2.4.0"
+version = "2.4.2"
 description = "High-performance HTML to Markdown converter powered by Rust with a clean Python API"
 readme = "README_PYPI.md"
 keywords = [