justhtml 0.6.0__py3-none-any.whl → 0.33.0__py3-none-any.whl

justhtml/entities.py

@@ -4,27 +4,33 @@ Implements HTML5 character reference (entity) decoding per WHATWG spec §13.2.5.
 Supports both named entities (&,  ) and numeric references (<, <).
 """
 
+from __future__ import annotations
+
 import html.entities
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 # Use Python's complete HTML5 entity list (2231 entities)
 # Keys include the trailing semicolon (e.g., "amp;", "lang;")
 # We'll strip semicolons when looking up to match both forms
-_HTML5_ENTITIES = html.entities.html5
+_HTML5_ENTITIES: dict[str, str] = html.entities.html5
 
 # Build a normalized lookup without semicolons for easier access
-NAMED_ENTITIES = {}
-for key, value in _HTML5_ENTITIES.items():
+NAMED_ENTITIES: dict[str, str] = {}
+for _key, _value in _HTML5_ENTITIES.items():
     # Remove trailing semicolon for lookup
-    if key.endswith(";"):
-        NAMED_ENTITIES[key[:-1]] = value
+    if _key.endswith(";"):
+        NAMED_ENTITIES[_key[:-1]] = _value
     else:
-        NAMED_ENTITIES[key] = value
+        NAMED_ENTITIES[_key] = _value
 
 # Legacy named character references that can be used without semicolons
 # Per HTML5 spec, these are primarily ISO-8859-1 (Latin-1) entities from HTML4
 # Modern entities like "prod", "notin" etc. require semicolons
 # Note: Some have both uppercase and lowercase versions (e.g., COPY/copy, GT/gt)
-LEGACY_ENTITIES = {
+LEGACY_ENTITIES: set[str] = {
     "gt",
     "lt",
     "amp",
@@ -134,7 +140,7 @@ LEGACY_ENTITIES = {
 }
 
 # HTML5 numeric character reference replacements (§13.2.5.73)
-NUMERIC_REPLACEMENTS = {
+NUMERIC_REPLACEMENTS: dict[int, str] = {
     0x00: "\ufffd",  # NULL
     0x80: "\u20ac",  # EURO SIGN
     0x82: "\u201a",  # SINGLE LOW-9 QUOTATION MARK
@@ -166,7 +172,23 @@ NUMERIC_REPLACEMENTS = {
 }
 
 
-def decode_numeric_entity(text, is_hex=False):
+def _is_control_character(codepoint: int) -> bool:
+    # C0 controls and C1 controls
+    return (0x00 <= codepoint <= 0x1F) or (0x7F <= codepoint <= 0x9F)
+
+
+def _is_noncharacter(codepoint: int) -> bool:
+    if 0xFDD0 <= codepoint <= 0xFDEF:
+        return True
+    last = codepoint & 0xFFFF
+    return last == 0xFFFE or last == 0xFFFF
+
+
+def decode_numeric_entity(
+    text: str,
+    is_hex: bool = False,
+    report_error: Callable[[str], None] | None = None,
+) -> str:
     """Decode a numeric character reference like &#60; or &#x3C;.
 
     Args:
@@ -179,20 +201,30 @@ def decode_numeric_entity(text, is_hex=False):
     base = 16 if is_hex else 10
     codepoint = int(text, base)
 
-    # Apply HTML5 replacements for certain ranges
-    if codepoint in NUMERIC_REPLACEMENTS:
-        return NUMERIC_REPLACEMENTS[codepoint]
-
     # Invalid ranges per HTML5 spec
     if codepoint > 0x10FFFF:
         return "\ufffd"  # REPLACEMENT CHARACTER
     if 0xD800 <= codepoint <= 0xDFFF:  # Surrogate range
         return "\ufffd"
 
+    if report_error is not None:
+        if _is_control_character(codepoint):
+            report_error("control-character-reference")
+        if _is_noncharacter(codepoint):
+            report_error("noncharacter-character-reference")
+
+    # Apply HTML5 replacements for certain ranges
+    if codepoint in NUMERIC_REPLACEMENTS:
+        return NUMERIC_REPLACEMENTS[codepoint]
+
     return chr(codepoint)
 
 
-def decode_entities_in_text(text, in_attribute=False):
+def decode_entities_in_text(
+    text: str,
+    in_attribute: bool = False,
+    report_error: Callable[[str], None] | None = None,
+) -> str:
     """Decode all HTML entities in text.
 
     This is a simple implementation that handles:
@@ -207,7 +239,7 @@ def decode_entities_in_text(text, in_attribute=False):
     Returns:
         Text with entities decoded
     """
-    result = []
+    result: list[str] = []
     i = 0
     length = len(text)
     while i < length:
@@ -245,7 +277,9 @@ def decode_entities_in_text(text, in_attribute=False):
             digit_text = text[digit_start:j]
 
             if digit_text:
-                result.append(decode_numeric_entity(digit_text, is_hex=is_hex))
+                if report_error is not None and not has_semicolon:
+                    report_error("missing-semicolon-after-character-reference")
+                result.append(decode_numeric_entity(digit_text, is_hex=is_hex, report_error=report_error))
                 i = j + 1 if has_semicolon else j
                 continue
 
@@ -274,7 +308,7 @@ def decode_entities_in_text(text, in_attribute=False):
             continue
         # If semicolon present but no exact match, allow legacy prefix match in text
         if has_semicolon and not in_attribute:
-            best_match = None
+            best_match: str | None = None
             best_match_len = 0
             for k in range(len(entity_name), 0, -1):
                 prefix = entity_name[:k]
@@ -283,6 +317,8 @@ def decode_entities_in_text(text, in_attribute=False):
                     best_match_len = k
                     break
             if best_match:
+                if report_error is not None:
+                    report_error("missing-semicolon-after-character-reference")
                 result.append(best_match)
                 i = i + 1 + best_match_len
                 continue
@@ -300,6 +336,8 @@ def decode_entities_in_text(text, in_attribute=False):
                 continue
 
             # Decode legacy entity
+            if report_error is not None and not has_semicolon:
+                report_error("missing-semicolon-after-character-reference")
             result.append(NAMED_ENTITIES[entity_name])
             i = j
             continue
@@ -327,6 +365,8 @@ def decode_entities_in_text(text, in_attribute=False):
                 i += 1
                 continue
 
+            if report_error is not None:
+                report_error("missing-semicolon-after-character-reference")
             result.append(best_match)
             i = i + 1 + best_match_len
             continue

justhtml/errors.py

@@ -1,11 +1,14 @@
-"""Centralized error message definitions and helpers for HTML parsing errors.
+"""Centralized error message definitions and helpers for JustHTML errors.
 
-This module provides human-readable error messages for all parse error codes
-emitted by both the tokenizer and tree builder during HTML parsing.
+This module provides human-readable error messages for parse error codes
+emitted by the tokenizer and tree builder during HTML parsing, plus selected
+security findings emitted by the sanitizer.
 """
 
+from __future__ import annotations
 
-def generate_error_message(code, tag_name=None):
+
+def generate_error_message(code: str, tag_name: str | None = None) -> str:
     """Generate human-readable error message from error code.
 
     Args:
@@ -73,6 +76,8 @@ def generate_error_message(code, tag_name=None):
         "illegal-codepoint-for-numeric-entity": "Invalid codepoint in numeric character reference",
         "missing-semicolon-after-character-reference": "Missing semicolon after character reference",
         "named-entity-without-semicolon": "Named entity used without semicolon",
+        "noncharacter-character-reference": "Noncharacter in character reference",
+        "noncharacter-in-input-stream": "Noncharacter in input stream",
         # ================================================================
         # TREE BUILDER ERRORS
         # ================================================================
@@ -105,8 +110,11 @@ def generate_error_message(code, tag_name=None):
         # Foster parenting / table errors
         "foster-parenting-character": "Text content in table requires foster parenting",
         "foster-parenting-start-tag": "Start tag in table requires foster parenting",
+        "unexpected-character-implies-table-voodoo": "Unexpected character in table triggers foster parenting",
         "unexpected-start-tag-implies-table-voodoo": f"<{tag_name}> start tag in table triggers foster parenting",
         "unexpected-end-tag-implies-table-voodoo": f"</{tag_name}> end tag in table triggers foster parenting",
+        "unexpected-implied-end-tag-in-table-view": "Unexpected implied end tag while closing table",
+        "eof-in-table": "Unexpected end of file in table",
         "unexpected-cell-in-table-body": "Unexpected table cell outside of table row",
         "unexpected-form-in-table": "Form element not allowed in table context",
         "unexpected-hidden-input-in-table": "Hidden input in table triggers foster parenting",
@@ -132,6 +140,14 @@ def generate_error_message(code, tag_name=None):
         "adoption-agency-1.3": "Misnested tags require adoption agency algorithm",
         "non-void-html-element-start-tag-with-trailing-solidus": f"<{tag_name}/> self-closing syntax on non-void element",
         "image-start-tag": f"Deprecated <{tag_name}> tag (use <img> instead)",
+        # Select insertion mode (context-specific taxonomy)
+        "unexpected-start-tag-in-select": f"Unexpected <{tag_name}> start tag in <select>",
+        "unexpected-end-tag-in-select": f"Unexpected </{tag_name}> end tag in <select>",
+        "unexpected-select-in-select": "Unexpected nested <select> in <select>",
+        # ================================================================
+        # SECURITY ERRORS
+        # ================================================================
+        "unsafe-html": "Unsafe HTML detected by sanitization policy",
     }
 
     # Return message or fall back to the code itself if not found

justhtml/linkify.py

@@ -0,0 +1,438 @@
+"""Text linkification scanner.
+
+This module finds URL/email-like substrings in plain text.
+
+It is intentionally HTML-agnostic: in JustHTML it is applied to DOM text nodes,
+not to raw HTML strings.
+
+The behavior is driven by vendored compliance fixtures from the upstream
+`linkify-it` project (MIT licensed). See `tests/linkify-it/README.md`.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Final
+
+
+@dataclass(frozen=True, slots=True)
+class LinkMatch:
+    start: int
+    end: int
+    text: str
+    href: str
+    kind: str  # "url" | "email"
+
+
+DEFAULT_TLDS: Final[frozenset[str]] = frozenset(
+    {
+        # Keep this aligned with linkify-it's default list.
+        # See: https://github.com/markdown-it/linkify-it/blob/master/index.mjs
+        "biz",
+        "com",
+        "edu",
+        "gov",
+        "net",
+        "org",
+        "pro",
+        "web",
+        "xxx",
+        "aero",
+        "asia",
+        "coop",
+        "info",
+        "museum",
+        "name",
+        "shop",
+        "рф",
+    }
+)
+
+
+# A pragmatic Unicode-aware domain label pattern.
+#
+# Use `\w` for Unicode letters/digits (and underscore), and reject underscores
+# during validation. This is intentionally stricter than allowing all non-ASCII
+# codepoints, and matches the fixture behavior around delimiter punctuation.
+_LABEL_RE: Final[str] = (
+    r"[0-9A-Za-z\w\u2600-\u27bf]"
+    r"(?:[0-9A-Za-z\w\u2600-\u27bf-]{0,61}[0-9A-Za-z\w\u2600-\u27bf])?"
+)
+
+# A fast-ish candidate matcher. We do real validation after we find a candidate.
+_CANDIDATE_PATTERN: Final[str] = "".join(
+    [
+        r"(?i)([^0-9A-Za-z_])",  # left boundary (avoid matching after underscore)
+        r"(",  # candidate group
+        r"(?:https?|ftp)://[^\s<>\uFF5C]+",  # absolute URL
+        r"|mailto:[^\s<>\uFF5C]+",  # mailto
+        r"|//[^\s<>\uFF5C]+",  # protocol-relative
+        r"|(?:www\.)[^\s<>\uFF5C]+",  # www.
+        rf"|[0-9A-Za-z.!#$%&'*+/=?^_`{{|}}~\-\"]+@(?:{_LABEL_RE}\.)+{_LABEL_RE}",  # email
+        r"|(?:\d{1,3}\.){3}\d{1,3}(?:/[^\s<>\uFF5C]*)?",  # IPv4
+        rf"|(?:{_LABEL_RE}\.)+{_LABEL_RE}(?:/[^\s<>\uFF5C]*)?",  # fuzzy domain/path
+        r")",
+    ]
+)
+
+_CANDIDATE_RE: Final[re.Pattern[str]] = re.compile(_CANDIDATE_PATTERN, re.UNICODE)
+
+_TRAILING_PUNCT: Final[str] = ".,;:!?"
+
+# RE pattern for 2-character TLDs, copied from linkify-it (MIT licensed).
+_CC_TLD_RE: Final[re.Pattern[str]] = re.compile(
+    r"^(?:a[cdefgilmnoqrstuwxz]|b[abdefghijmnorstvwyz]|c[acdfghiklmnoruvwxyz]|d[ejkmoz]|e[cegrstu]|f[ijkmor]|g[abdefghilmnpqrstuwy]|h[kmnrtu]|i[delmnoqrst]|j[emop]|k[eghimnprwyz]|l[abcikrstuvy]|m[acdeghklmnopqrstuvwxyz]|n[acefgilopruz]|om|p[aefghklmnrstwy]|qa|r[eosuw]|s[abcdeghijklmnortuvxyz]|t[cdfghjklmnortvwz]|u[agksyz]|v[aceginu]|w[fs]|y[et]|z[amw])$",
+    re.IGNORECASE,
+)
+
+
+def _is_valid_tld(tld: str, *, extra_tlds: frozenset[str]) -> bool:
+    t = (tld or "").lower()
+    if not t:
+        return False
+    # Only valid 2-letter ccTLDs (avoid false positives like `.js`).
+    if len(t) == 2 and _CC_TLD_RE.match(t) is not None:
+        return True
+    # Any punycode root.
+    if t.startswith("xn--"):
+        return True
+    return t in DEFAULT_TLDS or t in extra_tlds
+
+
+def _split_domain_for_tld(host: str) -> tuple[str, str] | None:
+    # Return (domain_without_tld, tld).
+    h = (host or "").strip().strip(".")
+    if not h:
+        return None
+    if h.lower() == "localhost":
+        return ("localhost", "")
+    if "." not in h:
+        return None
+    base, tld = h.rsplit(".", 1)
+    return (base, tld)
+
+
+@dataclass(frozen=True, slots=True)
+class LinkifyConfig:
+    fuzzy_ip: bool = False
+    extra_tlds: frozenset[str] = frozenset()
+
+    @staticmethod
+    def with_extra_tlds(extra_tlds: list[str] | tuple[str, ...] | set[str] | frozenset[str]) -> LinkifyConfig:
+        return LinkifyConfig(extra_tlds=frozenset(str(t).lower() for t in extra_tlds))
+
+
+def _is_valid_ipv4(host: str) -> bool:
+    parts = host.split(".")
+    if len(parts) != 4:
+        return False
+    for p in parts:
+        if not p or len(p) > 3:
+            return False
+        if not p.isdigit():
+            return False
+        v = int(p)
+        if v < 0 or v > 255:
+            return False
+    return True
+
+
+def _punycode_host(host: str) -> str:
+    # Safety default: normalize Unicode domains to punycode for href.
+    try:
+        return host.encode("idna").decode("ascii")
+    except UnicodeError:
+        return host
+
+
+def _split_host_and_rest(raw: str) -> tuple[str, str]:
+    # raw is after an optional scheme prefix (or for fuzzy domains, the whole).
+    # Extract host[:port] and the rest (path/query/fragment).
+    for i, ch in enumerate(raw):
+        if ch in "/?#":
+            return raw[:i], raw[i:]
+    return raw, ""
+
+
+def _strip_wrapping(raw: str) -> tuple[str, int, int]:
+    # Trim common wrappers like <...> or quotes, but report how many chars were removed
+    # from start/end so we can compute accurate offsets.
+    start_trim = 0
+    end_trim = 0
+
+    if raw and raw[0] in "<\"'([{" and raw[-1] in ">\"')]}":
+        # Angle brackets are common for autolinks.
+        # Quotes/brackets: we strip them only if they wrap the candidate.
+        raw = raw[1:-1]
+        start_trim = 1
+        end_trim = 1
+
+    return raw, start_trim, end_trim
+
+
+def _trim_trailing(candidate: str) -> str:
+    # Remove trailing punctuation and unbalanced closing brackets.
+    if not candidate:
+        return candidate
+
+    # First strip sentence punctuation.
+    while candidate and candidate[-1] in _TRAILING_PUNCT:
+        candidate = candidate[:-1]
+
+    # Then strip quoting terminators when unbalanced (treat quotes as wrappers).
+    while candidate and candidate[-1] in "\"'":
+        q = candidate[-1]
+        if candidate.count(q) % 2 == 1:
+            candidate = candidate[:-1]
+            continue
+        break
+
+    # Then strip unmatched closing brackets.
+    # We treat ) ] } > as potentially closable.
+    pairs = {")": "(", "]": "[", "}": "{", ">": "<"}
+    while candidate and candidate[-1] in pairs:
+        close = candidate[-1]
+        open_ch = pairs[close]
+        if candidate.count(close) > candidate.count(open_ch):
+            candidate = candidate[:-1]
+            continue
+        break
+
+    return candidate
+
+
+def _href_for(text: str) -> tuple[str, str]:
+    lower = text.lower()
+
+    if lower.startswith("mailto:"):
+        return text, "email"
+
+    if "@" in text and not lower.startswith(("http://", "https://", "ftp://", "//", "www.")):
+        return f"mailto:{text}", "email"
+
+    if lower.startswith(("http://", "https://", "ftp://", "//")):
+        return text, "url"
+
+    # www. and fuzzy domains default to http://
+    return f"http://{text}", "url"
+
+
+def _punycode_href(href: str) -> str:
+    # Convert the host portion to punycode (IDNA), keeping the rest intact.
+    lower = href.lower()
+    prefix = ""
+    rest = href
+
+    if lower.startswith("mailto:"):
+        return href
+
+    if lower.startswith("http://"):
+        prefix = href[:7]
+        rest = href[7:]
+    elif lower.startswith("https://"):
+        prefix = href[:8]
+        rest = href[8:]
+    elif lower.startswith("ftp://"):
+        prefix = href[:6]
+        rest = href[6:]
+    elif lower.startswith("//"):
+        prefix = href[:2]
+        rest = href[2:]
+    else:
+        # Shouldn't happen; fuzzy hrefs are normalized before calling.
+        prefix = ""
+        rest = href
+
+    hostport, tail = _split_host_and_rest(rest)
+
+    # Handle userinfo (user:pass@host)
+    userinfo = ""
+    hostport2 = hostport
+    if "@" in hostport:
+        userinfo, hostport2 = hostport.rsplit("@", 1)
+
+    host = hostport2
+    port = ""
+    if hostport2.startswith("["):
+        # IPv6-ish, don't punycode.
+        return href
+    if ":" in hostport2:
+        host, port = hostport2.split(":", 1)
+
+    host_pc = _punycode_host(host)
+    rebuilt = host_pc
+    if port:
+        rebuilt = f"{rebuilt}:{port}"
+    if userinfo:
+        rebuilt = f"{userinfo}@{rebuilt}"
+
+    return f"{prefix}{rebuilt}{tail}"
+
+
+def find_links(text: str) -> list[LinkMatch]:
+    return find_links_with_config(text, LinkifyConfig())
+
+
+def find_links_with_config(text: str, config: LinkifyConfig) -> list[LinkMatch]:
+    if not text:
+        return []
+
+    # Mirror linkify-it behavior: always scan with a leading boundary character.
+    scan_text = "\n" + text
+
+    out: list[LinkMatch] = []
+
+    for m in _CANDIDATE_RE.finditer(scan_text):
+        raw = m.group(2)
+
+        # Compute absolute offsets (exclude the boundary prefix char).
+        start = m.start(2) - 1
+        end = m.end(2) - 1
+
+        stripped, s_trim, e_trim = _strip_wrapping(raw)
+        start += s_trim
+        end -= e_trim
+
+        cand = _trim_trailing(stripped)
+        if not cand:
+            continue
+
+        # Markdown-style termination: `(...URL...)[...]` should stop at the `)`.
+        lower = cand.lower()
+        if lower.startswith(("http://", "https://", "ftp://")) and ")[" in cand:
+            cand = cand.split(")[", 1)[0]
+            cand = _trim_trailing(cand)
+            if not cand:
+                continue
+
+        # Treat leading quotes as wrappers/delimiters, not part of the URL/email.
+        if cand and cand[0] in "\"'" and 0 <= start < len(text) and text[start] == cand[0]:
+            cand = cand[1:]
+            start += 1
+            if not cand:
+                continue
+
+        # Adjust end after trimming.
+        end = start + len(cand)
+
+        lower = cand.lower()
+
+        # If this looks like a fuzzy domain that starts immediately after ://,
+        # treat it as part of a broken/disabled schema (e.g. _http://example.com, hppt://example.com).
+        if not lower.startswith(("http://", "https://", "ftp://", "mailto:", "//", "www.")) and "@" not in cand:
+            if start >= 3 and text[start - 3 : start] == "://":
+                continue
+            if start > 0 and text[start - 1] in "/:@":
+                continue
+
+        # Validate fuzzy IP option.
+        if (
+            cand
+            and cand[0].isdigit()
+            and "." in cand
+            and not lower.startswith(("http://", "https://", "ftp://", "//"))
+        ):
+            host, _ = _split_host_and_rest(cand)
+            if host.replace(".", "").isdigit() and _is_valid_ipv4(host):
+                if not config.fuzzy_ip:
+                    continue
+
+        # Validate // URLs: allow localhost or dotted domains, but not single-level.
+        if lower.startswith("//"):
+            # Protect against matching the // inside :// or ///.
+            if start > 0 and text[start - 1] in ":/":
+                continue
+            after = cand[2:]
+            hostport, _ = _split_host_and_rest(after)
+            if not hostport:
+                continue
+            if hostport.startswith("["):
+                continue
+            host_only = hostport
+            if "@" in host_only:
+                host_only = host_only.rsplit("@", 1)[1]
+            if ":" in host_only:
+                host_only = host_only.split(":", 1)[0]
+            if host_only.lower() != "localhost" and "." not in host_only:
+                continue
+
+            if "_" in host_only:
+                continue
+
+        # Validate fuzzy domains and emails with TLD allowlist.
+        is_scheme = lower.startswith(("http://", "https://", "ftp://", "mailto:"))
+        is_www = lower.startswith("www.")
+        is_proto_rel = lower.startswith("//")
+
+        if not is_scheme and not is_proto_rel and not is_www and "@" not in cand:
+            host, _ = _split_host_and_rest(cand)
+            if "_" in host:
+                continue
+
+            # IPv4 candidates don't use the TLD allowlist.
+            if "." in host and host.replace(".", "").isdigit() and _is_valid_ipv4(host):
+                pass
+            else:
+                parts = _split_domain_for_tld(host)
+                if parts is None:
+                    continue
+                _base, tld = parts
+                if not _is_valid_tld(tld, extra_tlds=config.extra_tlds):
+                    continue
+
+        if (
+            "@" in cand
+            and not lower.startswith(("http://", "https://", "ftp://", "//"))
+            and not lower.startswith("mailto:")
+        ):
+            # Fuzzy email requires a valid TLD.
+            local, domain = cand.rsplit("@", 1)
+            _ = local
+            host, _tail = _split_host_and_rest(domain)
+            if "_" in host:
+                continue
+            parts = _split_domain_for_tld(host)
+            if parts is None:
+                continue
+            _base, tld = parts
+            if not _is_valid_tld(tld, extra_tlds=config.extra_tlds):
+                continue
+
+        # Validate basic URL host/port if scheme-based.
+        if lower.startswith(("http://", "https://", "ftp://")):
+            after = cand.split("://", 1)[1]
+            hostport, _ = _split_host_and_rest(after)
+            if not hostport:
+                continue
+            if "@" in hostport:
+                hostport = hostport.rsplit("@", 1)[1]
+            host = hostport
+            if ":" in hostport and not hostport.startswith("["):
+                host, port = hostport.split(":", 1)
+                if port and (not port.isdigit() or int(port) > 65535):
+                    continue
+            if not host or host.startswith(("-", ".")) or host.endswith(("-", ".")) or ".." in host:
+                continue
+            if "_" in host:
+                continue
+            if "." in host and host.replace(".", "").isdigit() and not _is_valid_ipv4(host):
+                continue
+
+        href, kind = _href_for(cand)
+        href = _punycode_href(href)
+
+        out.append(LinkMatch(start=start, end=end, text=cand, href=href, kind=kind))
+
+    # Avoid overlapping matches by keeping first-longest.
+    if not out:
+        return out
+    out.sort(key=lambda x: (x.start, -(x.end - x.start)))
+    filtered: list[LinkMatch] = []
+    last_end = -1
+    for lm in out:
+        if lm.start < last_end:
+            continue
+        filtered.append(lm)
+        last_end = lm.end
+    return filtered