htomd 0.1.0__py3-none-any.whl

htomd/__init__.py

@@ -0,0 +1,37 @@
+"""Focused Markdown and metadata from decoded HTML, with no network access."""
+
+from ._metadata import read_metadata, refine_metadata
+from ._models import Diagnostics, Document, Metadata
+from ._parser import parse
+from ._render import render
+from ._selection import select
+from ._urls import document_base
+
+__all__ = ["Diagnostics", "Document", "Metadata", "convert", "extract"]
+
+
+def extract(html: str, *, url: str | None = None) -> Document:
+    """Extract relevant Markdown and explicit metadata from an HTML string.
+
+    ``url`` provides source context and resolves references; no fetching occurs.
+    Malformed HTML receives best-effort recovery. Result dataclasses are immutable.
+    """
+    if not isinstance(html, str):
+        raise TypeError("html must be a decoded str")
+    if url is not None and not isinstance(url, str):
+        raise TypeError("url must be a str or None")
+    root, parsing_notes = parse(html)
+    base = document_base(root, url)
+    metadata = read_metadata(root, url, base)
+    selected, diagnostics = select(root)
+    metadata = refine_metadata(metadata, selected)
+    markdown = render(selected, base)
+    diagnostics = Diagnostics(
+        diagnostics.strategy if markdown else "none", parsing_notes + diagnostics.notes
+    )
+    return Document(markdown, metadata, diagnostics)
+
+
+def convert(html: str, *, url: str | None = None) -> str:
+    """Return the Markdown produced by :func:`extract`."""
+    return extract(html, url=url).markdown

htomd/__main__.py

@@ -0,0 +1,6 @@
+"""Support ``python -m htomd``."""
+
+from ._cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

htomd/_cli.py

@@ -0,0 +1,92 @@
+"""Command-line conversion of HTML supplied on stdin."""
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import asdict
+from importlib.metadata import version
+
+from . import convert, extract
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="htomd",
+        description=(
+            "Extract Markdown and metadata from UTF-8 HTML on stdin. "
+            "Pipe input from cat or curl; output goes to stdout. No file arguments or fetching."
+        ),
+        epilog=(
+            "Examples:\n"
+            "  cat page.html | htomd convert > page.md\n"
+            "  cat page.html | htomd extract > page.json\n"
+            "  curl -fsSL https://example.com | htomd convert --url https://example.com\n"
+            "\nUse 'htomd help COMMAND' for command-specific help."
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    version_text = f"htomd {version('htomd')}"
+    parser.add_argument("--version", action="version", version=version_text)
+    commands = parser.add_subparsers(dest="command")
+    command_parsers: dict[str, argparse.ArgumentParser] = {}
+    for name, description in (
+        ("convert", "Write Markdown to stdout."),
+        ("extract", "Write Markdown, metadata, and diagnostics as JSON to stdout."),
+    ):
+        command = commands.add_parser(
+            name,
+            help=description,
+            description=f"Read UTF-8 HTML from stdin to EOF. {description}",
+            epilog=f"Example: cat page.html | htomd {name}",
+        )
+        command.add_argument(
+            "--url", help="Source URL for resolving references; no fetching occurs."
+        )
+        command_parsers[name] = command
+    command_parsers["version"] = commands.add_parser(
+        "version",
+        help="Show the installed package version.",
+        description="Show the installed version.",
+    )
+    help_parser = commands.add_parser(
+        "help",
+        help="Show general or command-specific help.",
+        description="Show help for a command.",
+    )
+    command_parsers["help"] = help_parser
+    help_parser.add_argument(
+        "topic", nargs="?", choices=command_parsers, help="Command to describe."
+    )
+    args = parser.parse_args(argv)
+
+    if args.command is None or args.command == "help":
+        topic = args.topic if args.command == "help" else None
+        (command_parsers[topic] if topic else parser).print_help()
+        return 0
+    if args.command == "version":
+        print(version_text)
+        return 0
+
+    try:
+        html = sys.stdin.buffer.read().decode("utf-8-sig")
+        if args.command == "convert":
+            output = convert(html, url=args.url)
+        else:
+            output = json.dumps(asdict(extract(html, url=args.url)), ensure_ascii=False, indent=2)
+            output += "\n"
+    except (OSError, UnicodeError) as error:
+        print(f"htomd: {error}", file=sys.stderr)
+        return 1
+
+    try:
+        sys.stdout.buffer.write(output.encode("utf-8"))
+        sys.stdout.buffer.flush()
+    except OSError as error:
+        # Prevent a second output failure when Python flushes stdout during shutdown.
+        with open(os.devnull, "wb") as sink:
+            os.dup2(sink.fileno(), sys.stdout.fileno())
+        if not isinstance(error, BrokenPipeError):
+            print(f"htomd: {error}", file=sys.stderr)
+        return 1
+    return 0

htomd/_metadata.py

@@ -0,0 +1,137 @@
+"""Read explicit metadata before cleanup, then refine it from selected content."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from dataclasses import replace
+
+from ._models import Metadata
+from ._tree import Node, text_content, walk
+from ._urls import safe_url
+
+ARTICLE_TYPES = frozenset(
+    {
+        "Article",
+        "NewsArticle",
+        "BlogPosting",
+        "TechArticle",
+        "ScholarlyArticle",
+        "MedicalScholarlyArticle",
+        "Report",
+        "AnalysisNewsArticle",
+        "OpinionNewsArticle",
+        "ReviewNewsArticle",
+        "BackgroundNewsArticle",
+        "APIReference",
+        "LiveBlogPosting",
+    }
+)
+
+
+def string(value: object) -> str | None:
+    return value.strip() or None if isinstance(value, str) else None
+
+
+def json_articles(value: object) -> Iterator[dict[str, object]]:
+    stack = [value]
+    while stack:
+        item = stack.pop()
+        if isinstance(item, list):
+            stack.extend(reversed(item))
+        elif isinstance(item, dict):
+            kind = item.get("@type", [])
+            kinds = [kind] if isinstance(kind, str) else kind
+            if isinstance(kinds, list) and any(
+                isinstance(entry, str) and entry.rsplit("/", 1)[-1] in ARTICLE_TYPES
+                for entry in kinds
+            ):
+                yield item
+            graph = item.get("@graph")
+            if isinstance(graph, (dict, list)):
+                stack.append(graph)
+
+
+def author_name(value: object) -> str | None:
+    if isinstance(value, str):
+        return string(value)
+    if isinstance(value, dict):
+        return string(value.get("name"))
+    if isinstance(value, list):
+        names = [name for entry in value if (name := author_name(entry))]
+        return ", ".join(names) or None
+    return None
+
+
+def read_jsonld(root: Node) -> dict[str, object]:
+    for node in walk(root):
+        if node.tag != "script" or node.attrs.get("type", "").lower() != "application/ld+json":
+            continue
+        try:
+            value = json.loads(text_content(node, normalize=False))
+        except (ValueError, RecursionError):
+            continue
+        article = next(json_articles(value), None)
+        if article is not None:
+            return article
+    return {}
+
+
+def read_metadata(root: Node, url: str | None, base: str | None) -> Metadata:
+    fields: dict[str, str] = {}
+    title = language = canonical = None
+    for node in walk(root):
+        if node.tag == "meta":
+            key = node.attrs.get("property", node.attrs.get("name", "")).lower()
+            content = string(node.attrs.get("content"))
+            if content:
+                fields.setdefault(key, content)
+        elif node.tag == "title" and title is None:
+            title = string(text_content(node))
+        elif node.tag == "html":
+            language = string(node.attrs.get("lang") or node.attrs.get("xml:lang"))
+        elif node.tag == "link" and "canonical" in node.attrs.get("rel", "").lower().split():
+            canonical = canonical or safe_url(node.attrs.get("href", ""), base) or None
+    article = read_jsonld(root)
+    return Metadata(
+        title=fields.get("og:title") or title or string(article.get("headline")),
+        author=fields.get("author") or author_name(article.get("author")),
+        description=fields.get("description")
+        or fields.get("og:description")
+        or string(article.get("description")),
+        language=language or string(article.get("inLanguage")) or fields.get("og:locale"),
+        published_time=fields.get("article:published_time")
+        or fields.get("date")
+        or string(article.get("datePublished")),
+        url=url,
+        canonical_url=canonical,
+    )
+
+
+def refine_metadata(metadata: Metadata, selected: list[Node]) -> Metadata:
+    heading = author = published = None
+    for root in selected:
+        for node in walk(root):
+            if node.tag == "h1" and heading is None:
+                heading = string(text_content(node))
+            if author is None and local_author(node):
+                author = string(text_content(node))
+            if (
+                published is None
+                and node.tag == "time"
+                and node.attrs.get("itemprop") != "dateModified"
+            ):
+                published = string(node.attrs.get("datetime")) or string(text_content(node))
+    return replace(
+        metadata,
+        title=heading or metadata.title,
+        author=metadata.author or author,
+        published_time=metadata.published_time or published,
+    )
+
+
+def local_author(node: Node) -> bool:
+    if node.attrs.get("itemprop") == "author" or "author" in node.attrs.get("rel", "").split():
+        return True
+    tokens = node.attrs.get("class", "").lower().split()
+    return any(token in {"byline", "author", "p-author"} for token in tokens)

htomd/_models.py

@@ -0,0 +1,28 @@
+"""Immutable public results; the HTML tree is deliberately private."""
+
+from dataclasses import dataclass
+from typing import Literal
+
+
+@dataclass(frozen=True, slots=True)
+class Metadata:
+    title: str | None = None
+    author: str | None = None
+    description: str | None = None
+    language: str | None = None
+    published_time: str | None = None
+    url: str | None = None
+    canonical_url: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class Diagnostics:
+    strategy: Literal["semantic", "scored", "fallback", "none"]
+    notes: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class Document:
+    markdown: str
+    metadata: Metadata
+    diagnostics: Diagnostics

htomd/_parser.py

@@ -0,0 +1,133 @@
+"""Small HTMLParser-based recovery, without pretending to implement HTML5."""
+
+from html.parser import HTMLParser
+
+from ._tree import Node
+
+VOID = frozenset(
+    [
+        "area",
+        "base",
+        "br",
+        "col",
+        "embed",
+        "hr",
+        "img",
+        "input",
+        "link",
+        "meta",
+        "param",
+        "source",
+        "track",
+        "wbr",
+    ]
+)
+P_BREAKERS = frozenset(
+    [
+        "address",
+        "article",
+        "aside",
+        "blockquote",
+        "div",
+        "dl",
+        "fieldset",
+        "footer",
+        "form",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "header",
+        "hr",
+        "main",
+        "nav",
+        "ol",
+        "p",
+        "pre",
+        "section",
+        "table",
+        "ul",
+    ]
+)
+# A matching optional end tag must be found before its enclosing scope boundary.
+IMPLIED: dict[str, tuple[frozenset[str], frozenset[str]]] = {
+    "li": (frozenset({"li"}), frozenset({"ul", "ol"})),
+    "dt": (frozenset({"dt", "dd"}), frozenset({"dl"})),
+    "dd": (frozenset({"dt", "dd"}), frozenset({"dl"})),
+    "tr": (frozenset({"tr"}), frozenset({"table", "tbody", "thead", "tfoot"})),
+    "td": (frozenset({"td", "th"}), frozenset({"tr", "table"})),
+    "th": (frozenset({"td", "th"}), frozenset({"tr", "table"})),
+    "thead": (frozenset({"thead", "tbody", "tfoot"}), frozenset({"table"})),
+    "tbody": (frozenset({"thead", "tbody", "tfoot"}), frozenset({"table"})),
+    "tfoot": (frozenset({"thead", "tbody", "tfoot"}), frozenset({"table"})),
+    "option": (frozenset({"option"}), frozenset({"select", "datalist"})),
+}
+END_SCOPES = {"li": {"ul", "ol"}, "td": {"tr", "table"}, "th": {"tr", "table"}, "tr": {"table"}}
+
+
+class TreeParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.root = Node("#document")
+        self.stack = [self.root]
+        self.recoveries = 0
+
+    def close_in_scope(self, targets: frozenset[str], boundaries: frozenset[str]) -> None:
+        for index in range(len(self.stack) - 1, 0, -1):
+            tag = self.stack[index].tag
+            if tag in targets:
+                del self.stack[index:]
+                self.recoveries += 1
+                return
+            if tag in boundaries:
+                return
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag in P_BREAKERS:
+            self.close_in_scope(frozenset({"p"}), frozenset({"table", "td", "th", "li"}))
+        if tag in IMPLIED:
+            self.close_in_scope(*IMPLIED[tag])
+        if tag == "a":
+            self.close_in_scope(frozenset({"a"}), frozenset({"p", "div", "li"}))
+        parent = self.stack[-1]
+        node = Node(tag, {name: value or "" for name, value in attrs}, parent=parent)
+        parent.children.append(node)
+        if tag not in VOID:
+            self.stack.append(node)
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self.handle_starttag(tag, attrs)
+        if tag not in VOID:
+            self.handle_endtag(tag)
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in VOID:
+            return
+        for index in range(len(self.stack) - 1, 0, -1):
+            current = self.stack[index].tag
+            if current == tag:
+                del self.stack[index:]
+                return
+            if current in END_SCOPES.get(tag, set()):
+                break
+        self.recoveries += 1
+
+    def handle_data(self, data: str) -> None:
+        self.stack[-1].children.append(data)
+
+    def unknown_decl(self, data: str) -> None:
+        self.recoveries += 1
+
+
+def parse(html: str) -> tuple[Node, tuple[str, ...]]:
+    parser = TreeParser()
+    try:
+        parser.feed(html)
+        parser.close()
+    except (AssertionError, ValueError):
+        # HTMLParser rejects malformed marked sections. Preserve the parsed prefix.
+        parser.recoveries += 1
+    notes = ("Recovered malformed or optionally closed HTML.",) if parser.recoveries else ()
+    return parser.root, notes

htomd/_render.py

@@ -0,0 +1,270 @@
+"""Stack-safe Markdown serialization, independent of selection heuristics."""
+
+from __future__ import annotations
+
+import re
+
+from ._selection import HEADINGS
+from ._tree import SPACE, Node, elements, postorder, text_content
+from ._urls import destination, safe_url
+
+EMPHASIS = {"em": "*", "i": "*", "strong": "**", "b": "**", "s": "~~", "del": "~~", "strike": "~~"}
+BLOCKS = frozenset(
+    {
+        "p",
+        "div",
+        "article",
+        "main",
+        "section",
+        "header",
+        "footer",
+        "figure",
+        "figcaption",
+        "address",
+        "details",
+        "summary",
+        "dl",
+    }
+)
+MARKUP = re.compile(r"([\\`*_~\[\]])")
+BLOCK_START = re.compile(r"(^|\n)(\s*)(#{1,6}(?=\s)|[-+](?=\s)|\d+[.)](?=\s)|[=~-]{3,}(?=\s|$))")
+
+
+def escape(value: str) -> str:
+    value = value.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+    value = MARKUP.sub(r"\\\1", value)
+    return BLOCK_START.sub(escape_block_start, value)
+
+
+def escape_block_start(match: re.Match[str]) -> str:
+    marker = match[3]
+    if marker[0].isdigit():
+        marker = marker[:-1] + "\\" + marker[-1]
+    else:
+        marker = "\\" + marker
+    return match[1] + match[2] + marker
+
+
+def longest_run(value: str, character: str) -> int:
+    return max(
+        (len(match[0]) for match in re.finditer(re.escape(character) + "+", value)), default=0
+    )
+
+
+def inline_code(node: Node) -> str:
+    value = (
+        text_content(node, normalize=False)
+        .replace("\r\n", " ")
+        .replace("\r", " ")
+        .replace("\n", " ")
+    )
+    if not value:
+        return ""
+    delimiter = "`" * (longest_run(value, "`") + 1)
+    padding = (
+        " "
+        if value.startswith("`")
+        or value.endswith("`")
+        or (value.startswith(" ") and value.endswith(" ") and value.strip())
+        else ""
+    )
+    return delimiter + padding + value + padding + delimiter
+
+
+def code_language(node: Node) -> str:
+    candidates = [node, *elements(node)]
+    if node.parent is not None:
+        candidates.append(node.parent)
+    for candidate in candidates:
+        for token in candidate.attrs.get("class", "").split():
+            if token.startswith(("language-", "lang-", "highlight-")):
+                value = token.split("-", 1)[1]
+                if re.fullmatch(r"[\w.+#-]+", value):
+                    return value
+        value = candidate.attrs.get("data-language", "")
+        if re.fullmatch(r"[\w.+#-]+", value):
+            return value
+    return ""
+
+
+def fenced_code(node: Node) -> str:
+    value = text_content(node, normalize=False).replace("\r\n", "\n").replace("\r", "\n")
+    fence = "`" * max(3, longest_run(value, "`") + 1)
+    ending = "" if value.endswith("\n") else "\n"
+    return f"\n\n{fence}{code_language(node)}\n{value}{ending}{fence}\n\n"
+
+
+def join_parts(parts: list[str]) -> str:
+    """Coalesce block boundaries without touching whitespace inside code blocks."""
+    result: list[str] = []
+    for part in parts:
+        if not part:
+            continue
+        if result and result[-1].endswith("\n") and part.startswith("\n"):
+            result[-1] = result[-1].rstrip("\n")
+            part = "\n\n" + part.lstrip("\n")
+        result.append(part)
+    return "".join(result)
+
+
+def wrap_inline(body: str, marker: str) -> str:
+    content = body.strip()
+    if not content:
+        return body
+    leading = " " if body[:1].isspace() else ""
+    trailing = " " if body[-1:].isspace() else ""
+    return leading + marker + content + marker + trailing
+
+
+def render_link(node: Node, body: str, base: str | None) -> str:
+    href = safe_url(node.attrs.get("href", ""), base)
+    if not href or not body.strip():
+        return body
+    return f"[{body.strip()}]({destination(href)})"
+
+
+def render_image(node: Node, base: str | None) -> str:
+    alt = escape(SPACE.sub(" ", node.attrs.get("alt", "")).strip())
+    source = safe_url(node.attrs.get("src", ""), base)
+    if not source or not node.attrs.get("src"):
+        return alt
+    return f"![{alt}]({destination(source)})"
+
+
+def render_list(node: Node, rendered: dict[Node, str]) -> str:
+    try:
+        number = int(node.attrs.get("start", "1"))
+    except ValueError:
+        number = 1
+    items = []
+    for child in elements(node):
+        if child.tag != "li":
+            continue
+        if node.tag == "ol":
+            value = child.attrs.get("value", "")
+            if re.fullmatch(r"-?\d{1,9}", value):
+                number = int(value)
+        marker = f"{number}. " if node.tag == "ol" else "- "
+        lines = rendered[child].strip().splitlines()
+        if lines:
+            continuation = [" " * len(marker) + line if line else "" for line in lines[1:]]
+            items.append("\n".join([marker + lines[0], *continuation]))
+        number += 1
+    return "\n" + "\n".join(items) + "\n" if items else ""
+
+
+def table_rows(node: Node) -> list[Node]:
+    rows = []
+    stack = list(reversed(elements(node)))
+    while stack:
+        child = stack.pop()
+        if child.tag == "tr":
+            rows.append(child)
+        elif child.tag in {"thead", "tbody", "tfoot"}:
+            stack.extend(reversed(elements(child)))
+    return rows
+
+
+def simple_table(rows: list[list[Node]]) -> bool:
+    if not rows or not rows[0] or len({len(row) for row in rows}) != 1:
+        return False
+    for row in rows:
+        for cell in row:
+            if cell.attrs.get("colspan", "1") != "1" or cell.attrs.get("rowspan", "1") != "1":
+                return False
+            if any(child.tag in {"table", "pre", "ul", "ol", "p"} for child in elements(cell)):
+                return False
+    return True
+
+
+def render_table(node: Node, rendered: dict[Node, str]) -> str:
+    rows = [
+        [cell for cell in elements(row) if cell.tag in {"th", "td"}] for row in table_rows(node)
+    ]
+    captions = [rendered[child].strip() for child in elements(node) if child.tag == "caption"]
+    if not simple_table(rows):
+        lines = []
+        for index, row in enumerate(rows, 1):
+            cell_values = [rendered[cell].strip().replace("\n", " ") for cell in row]
+            lines.append(f"{index}. " + " — ".join(cell_values))
+        return "\n\n" + "\n\n".join([*captions, "\n".join(lines)]) + "\n\n"
+    values = [
+        [rendered[cell].strip().replace("|", "\\|").replace("\n", " ") for cell in row]
+        for row in rows
+    ]
+    if not any(cell.tag == "th" for cell in rows[0]):
+        values.insert(0, [""] * len(rows[0]))
+    values.insert(1, ["---"] * len(rows[0]))
+    lines = ["| " + " | ".join(row) + " |" for row in values]
+    return "\n\n" + "\n\n".join([*captions, "\n".join(lines)]) + "\n\n"
+
+
+def serialize(node: Node, body: str, rendered: dict[Node, str], base: str | None) -> str:
+    tag = node.tag
+    if tag in HEADINGS:
+        return "\n\n" + "#" * int(tag[1]) + " " + body.strip() + "\n\n" if body.strip() else ""
+    if tag in EMPHASIS:
+        return wrap_inline(body, EMPHASIS[tag])
+    if tag in {"code", "kbd", "samp"}:
+        return inline_code(node)
+    if tag == "pre":
+        return fenced_code(node)
+    if tag == "a":
+        return render_link(node, body, base)
+    if tag == "img":
+        return render_image(node, base)
+    if tag in {"ul", "ol"}:
+        return render_list(node, rendered)
+    if tag == "table":
+        return render_table(node, rendered)
+    return serialize_block(tag, body)
+
+
+def serialize_block(tag: str, body: str) -> str:
+    if tag == "blockquote":
+        return (
+            "\n\n"
+            + "\n".join("> " + line if line else ">" for line in body.strip().splitlines())
+            + "\n\n"
+        )
+    if tag == "br":
+        return "  \n"
+    if tag == "hr":
+        return "\n\n---\n\n"
+    if tag == "dt":
+        return "\n\n" + wrap_inline(body.strip(), "**") + "\n"
+    if tag == "dd":
+        return "\n" + body.strip() + "\n\n"
+    if tag in BLOCKS:
+        return "\n\n" + body.strip() + "\n\n" if body.strip() else ""
+    return body
+
+
+def child_parts(node: Node, rendered: dict[Node, str]) -> list[str]:
+    parts = []
+    block_tags = BLOCKS | HEADINGS | {"ul", "ol", "pre", "table", "blockquote", "hr", "li"}
+    for index, child in enumerate(node.children):
+        if isinstance(child, Node):
+            parts.append(rendered[child])
+            continue
+        if child.isspace():
+            neighbors = (
+                node.children[max(0, index - 1) : index] + node.children[index + 1 : index + 2]
+            )
+            if any(isinstance(other, Node) and other.tag in block_tags for other in neighbors):
+                continue
+        parts.append(escape(SPACE.sub(" ", child)))
+    return parts
+
+
+def render(selected: list[Node], base: str | None) -> str:
+    output = []
+    for root in selected:
+        rendered: dict[Node, str] = {}
+        for node in postorder(root):
+            parts = child_parts(node, rendered)
+            body = join_parts(parts)
+            rendered[node] = serialize(node, body, rendered, base)
+        output.append(rendered[root])
+    result = join_parts(output).strip()
+    return result + "\n" if result else ""

htomd/_selection.py

@@ -0,0 +1,339 @@
+"""Select content using cached evidence, then clean only the selected region."""
+
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass
+
+from ._models import Diagnostics
+from ._tree import Node, elements, postorder, walk
+
+INERT = frozenset(
+    [
+        "head",
+        "title",
+        "meta",
+        "link",
+        "base",
+        "script",
+        "style",
+        "template",
+        "noscript",
+        "iframe",
+        "object",
+        "embed",
+        "svg",
+        "canvas",
+        "nav",
+        "button",
+        "input",
+        "select",
+        "textarea",
+        "dialog",
+    ]
+)
+NAV_ROLES = frozenset({"navigation", "banner", "contentinfo", "menu", "menubar", "dialog"})
+STRUCTURES = frozenset({"ul", "ol", "dl", "table", "blockquote", "pre"})
+CONTAINERS = frozenset({"article", "main", "section", "div", "body", "#document"})
+EVIDENCE = frozenset({"p", "pre", "li", "dt", "dd", "td", "th", "blockquote"})
+HEADINGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
+NEGATIVE = frozenset(
+    [
+        "advertisement",
+        "ads",
+        "advert",
+        "promo",
+        "promotion",
+        "related",
+        "share",
+        "sharing",
+        "social",
+        "cookie",
+        "consent",
+        "newsletter",
+        "comments",
+        "comment",
+        "sidebar",
+        "breadcrumb",
+        "breadcrumbs",
+        "pagination",
+        "toolbar",
+        "footer",
+        "banner",
+        "dropdown",
+        "catlinks",
+        "menu",
+        "pager",
+        "teaser",
+        "toc",
+        "well",
+    ]
+)
+POSITIVE = frozenset(
+    {"article", "content", "main", "post", "entry", "story", "text", "documentation"}
+)
+REFERENCES = frozenset({"footnotes", "references", "endnotes", "bibliography"})
+PUNCTUATION = re.compile(r"[,.;:!?。，；：！？،؛]")
+HIDDEN_STYLE = re.compile(
+    r"(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*(?:hidden|collapse))\s*(?:!important\s*)?(?:;|$)",
+    re.I,
+)
+# Sibling recovery needs modest support, and can never escape its parent container.
+SIBLING_SCORE_RATIO = 0.18
+MAX_FALLBACK_BLOCKS = 256
+
+
+@dataclass(slots=True)
+class Stats:
+    characters: int = 0
+    linked: int = 0
+    punctuation: int = 0
+    blocks: int = 0
+    headings: int = 0
+    code: int = 0
+    cells: int = 0
+    images: int = 0
+    controls: int = 0
+
+    @property
+    def density(self) -> float:
+        return self.linked / max(1, self.characters)
+
+
+def hints(node: Node) -> set[str]:
+    value = node.attrs.get("class", "") + " " + node.attrs.get("id", "")
+    value = re.sub(r"([a-z])([A-Z])", r"\1 \2", value)
+    return set(re.findall(r"[a-z0-9]+", value.lower()))
+
+
+def explicitly_hidden(node: Node) -> bool:
+    return (
+        "hidden" in node.attrs
+        or node.attrs.get("aria-hidden", "").lower() == "true"
+        or bool(HIDDEN_STYLE.search(node.attrs.get("style", "")))
+    )
+
+
+def visible_tree(root: Node) -> Node:
+    """Copy visible content so metadata remains available on the original tree."""
+    result = Node(root.tag, root.attrs.copy())
+    stack = [(root, result, False)]
+    while stack:
+        original, target, local = stack.pop()
+        local = local or original.tag in {"article", "main"} or original.attrs.get("role") == "main"
+        for child in original.children:
+            if isinstance(child, str):
+                target.children.append(child)
+                continue
+            if excluded(child, local):
+                continue
+            copied = Node(child.tag, child.attrs.copy(), parent=target)
+            target.children.append(copied)
+            stack.append((child, copied, local))
+    return result
+
+
+def excluded(node: Node, local: bool) -> bool:
+    if node.tag in INERT or explicitly_hidden(node):
+        return True
+    if node.tag == "a" and "headerlink" in hints(node):
+        return node.attrs.get("href", "").startswith("#")
+    role = node.attrs.get("role", "").lower()
+    if role in NAV_ROLES:
+        return True
+    return node.tag in {"header", "footer"} and not local
+
+
+def statistics(root: Node) -> dict[Node, Stats]:
+    result: dict[Node, Stats] = {}
+    for node in postorder(root):
+        stat = Stats()
+        for child in node.children:
+            if isinstance(child, str):
+                stat.characters += len(child.strip())
+                stat.punctuation += len(PUNCTUATION.findall(child))
+            else:
+                other = result[child]
+                for field in Stats.__slots__:
+                    setattr(stat, field, getattr(stat, field) + getattr(other, field))
+        stat.blocks += int(node.tag in EVIDENCE and stat.characters > 0)
+        stat.headings += int(node.tag in HEADINGS and stat.characters > 0)
+        stat.code += int(node.tag == "pre")
+        stat.cells += int(node.tag in {"td", "th"})
+        stat.images += int(node.tag == "img" and bool(node.attrs.get("src")))
+        stat.controls += int(node.tag in {"form", "button", "input", "select", "textarea"})
+        if node.tag == "a":
+            stat.linked = stat.characters
+        result[node] = stat
+    return result
+
+
+def conditional_clutter(node: Node, stat: Stats) -> bool:
+    if node.tag in {"#document", "html", "body", "main", "article"}:
+        return False
+    tokens = hints(node)
+    if tokens & REFERENCES or node.attrs.get("role") == "note":
+        return False
+    if not tokens & NEGATIVE or node.tag in EVIDENCE | HEADINGS:
+        return False
+    # Class hints alone never delete a node: require structural corroboration.
+    if stat.density > 0.35 or stat.controls:
+        return True
+    if "footer" in tokens and stat.density > 0.15:
+        return True
+    if node.tag == "aside" and not stat.code and not stat.cells:
+        return True
+    if stat.blocks == 0 and stat.characters < 180:
+        return True
+    if tokens & {"comments", "comment"}:
+        children = elements(node)
+        return sum(bool(hints(child) & {"comment", "reply"}) for child in children) >= 2
+    return False
+
+
+def clean(root: Node) -> tuple[Node, int]:
+    stats = statistics(root)
+    removed = 0
+    for node in walk(root):
+        kept: list[Node | str] = []
+        for child in node.children:
+            if isinstance(child, Node) and conditional_clutter(child, stats[child]):
+                removed += 1
+            else:
+                kept.append(child)
+        node.children = kept
+    return root, removed
+
+
+def plausible(node: Node, stat: Stats, *, semantic: bool = False) -> bool:
+    if not stat.characters:
+        return bool(semantic and stat.images)
+    if stat.code or stat.cells:
+        return True
+    if node.tag == "p" and stat.characters > stat.linked:
+        return True
+    if stat.density >= 0.8:
+        return bool(semantic and stat.headings and stat.blocks)
+    return node.tag in CONTAINERS | EVIDENCE | HEADINGS | STRUCTURES or semantic
+
+
+def scores(root: Node, stats: dict[Node, Stats], *, relaxed: bool) -> dict[Node, float]:
+    result: dict[Node, float] = {}
+    for node in walk(root):
+        stat = stats[node]
+        if node.tag not in CONTAINERS or not plausible(node, stat):
+            continue
+        tokens = hints(node)
+        score = math.sqrt(stat.characters) + min(stat.blocks, 30) * 2
+        score += min(stat.punctuation, 40) * 0.25 + min(stat.code + stat.cells, 12) * 2
+        score += 8 * bool(tokens & POSITIVE)
+        if not relaxed:
+            score -= 18 * bool(tokens & NEGATIVE)
+        result[node] = score * (1 - stat.density) ** 2
+    # Leaf evidence flows to nearby ancestors, without repeatedly counting nested blocks.
+    for node in walk(root):
+        if node.tag not in EVIDENCE or any(child.tag in EVIDENCE for child in elements(node)):
+            continue
+        stat = stats[node]
+        support = (1 + min(stat.characters / 100, 3)) * (1 - stat.density)
+        ancestor = node.parent
+        for weight in (1.0, 0.5, 0.25):
+            if ancestor is None:
+                break
+            if ancestor in result:
+                result[ancestor] += support * weight
+            ancestor = ancestor.parent
+    return result
+
+
+def semantic_candidate(
+    root: Node, stats: dict[Node, Stats], ranked: dict[Node, float]
+) -> Node | None:
+    candidates = []
+    landmarks = []
+    for node in walk(root):
+        landmark = node.tag == "main" or node.attrs.get("role") == "main"
+        if not (landmark or node.tag == "article"):
+            continue
+        if not plausible(node, stats[node], semantic=True):
+            continue
+        if conditional_clutter(node, stats[node]):
+            continue
+        if "teaser" in hints(node) and stats[node].density > 0.1:
+            continue
+        candidates.append(node)
+        if landmark:
+            landmarks.append(node)
+    candidates = landmarks or candidates
+    if not candidates:
+        return None
+    return max(candidates, key=lambda node: ranked.get(node, 0))
+
+
+def recover_siblings(
+    winner: Node, stats: dict[Node, Stats], ranked: dict[Node, float]
+) -> list[Node]:
+    parent = winner.parent
+    if parent is None:
+        return [winner]
+    threshold = max(5, ranked.get(winner, 0) * SIBLING_SCORE_RATIO)
+    selected = []
+    for sibling in elements(parent):
+        stat = stats[sibling]
+        if conditional_clutter(sibling, stat):
+            continue
+        supported = ranked.get(sibling, 0) >= threshold and stat.density < 0.5
+        heading = sibling.tag in HEADINGS and stat.density < 0.5
+        paragraph = sibling.tag == "p" and stat.density < 0.25 and bool(stat.characters)
+        if sibling is winner or supported or heading or paragraph:
+            selected.append(sibling)
+    return selected
+
+
+def fallback(root: Node, stats: dict[Node, Stats]) -> list[Node]:
+    selected: list[Node] = []
+    stack = [root]
+    while stack and len(selected) < MAX_FALLBACK_BLOCKS:
+        node = stack.pop()
+        stat = stats[node]
+        if conditional_clutter(node, stat):
+            continue
+        if (
+            node.tag in EVIDENCE | HEADINGS | STRUCTURES
+            and plausible(node, stat)
+            or not elements(node)
+            and stat.characters
+            and stat.density < 0.5
+        ):
+            selected.append(node)
+        else:
+            stack.extend(reversed(elements(node)))
+    return selected
+
+
+def select(root: Node) -> tuple[list[Node], Diagnostics]:
+    visible, removed = clean(visible_tree(root))
+    stats = statistics(visible)
+    ranked = scores(visible, stats, relaxed=False)
+    notes = [f"Removed {removed} conditionally identified clutter blocks."] if removed else []
+    winner = semantic_candidate(visible, stats, ranked)
+    if winner is not None:
+        return [winner], Diagnostics("semantic", tuple(notes))
+    candidates = {
+        node: score for node, score in ranked.items() if node.tag not in {"body", "#document"}
+    }
+    if not candidates or max(candidates.values()) < 5:
+        notes.append("Retried selection once with relaxed class penalties.")
+        ranked = scores(visible, stats, relaxed=True)
+        candidates = {
+            node: score for node, score in ranked.items() if node.tag not in {"body", "#document"}
+        }
+    if candidates and max(candidates.values()) >= 5:
+        winner = max(candidates, key=lambda node: candidates[node])
+        return recover_siblings(winner, stats, ranked), Diagnostics("scored", tuple(notes))
+    recovered = fallback(visible, stats)
+    notes.append(
+        "Recovered plausible blocks." if recovered else "No relevant visible content found."
+    )
+    return recovered, Diagnostics("fallback" if recovered else "none", tuple(notes))

htomd/_tree.py

@@ -0,0 +1,53 @@
+"""Ordered, private HTML tree and stack-safe traversals."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+
+SPACE = re.compile(r"\s+")
+
+
+@dataclass(eq=False, slots=True)
+class Node:
+    tag: str
+    attrs: dict[str, str] = field(default_factory=dict)
+    children: list[Node | str] = field(default_factory=list)
+    parent: Node | None = field(default=None, repr=False)
+
+
+def walk(root: Node) -> Iterator[Node]:
+    stack = [root]
+    while stack:
+        node = stack.pop()
+        yield node
+        stack.extend(child for child in reversed(node.children) if isinstance(child, Node))
+
+
+def postorder(root: Node) -> Iterator[Node]:
+    stack = [(root, False)]
+    while stack:
+        node, visited = stack.pop()
+        if visited:
+            yield node
+            continue
+        stack.append((node, True))
+        stack.extend((child, False) for child in reversed(node.children) if isinstance(child, Node))
+
+
+def text_content(root: Node, *, normalize: bool = True) -> str:
+    pieces: list[str] = []
+    stack: list[Node | str] = [root]
+    while stack:
+        item = stack.pop()
+        if isinstance(item, str):
+            pieces.append(item)
+        else:
+            stack.extend(reversed(item.children))
+    value = "".join(pieces)
+    return SPACE.sub(" ", value).strip() if normalize else value
+
+
+def elements(node: Node) -> list[Node]:
+    return [child for child in node.children if isinstance(child, Node)]

htomd/_urls.py

@@ -0,0 +1,43 @@
+"""Resolve references without fetching; reject active or unknown schemes."""
+
+import re
+from urllib.parse import quote, urljoin, urlsplit
+
+from ._tree import Node, walk
+
+SAFE_SCHEMES = frozenset({"", "http", "https", "mailto", "tel", "ftp"})
+CONTROL = re.compile(r"[\x00-\x20\x7f]+")
+
+
+def safe_url(value: str, base: str | None = None) -> str | None:
+    value = value.strip()
+    # Browsers ignore ASCII control characters in schemes. Check the normalized
+    # spelling so an entity-encoded newline cannot disguise javascript:.
+    checked = CONTROL.sub("", value)
+    try:
+        if urlsplit(checked).scheme.lower() not in SAFE_SCHEMES:
+            return None
+        resolved = urljoin(base, value) if base else value
+        if urlsplit(CONTROL.sub("", resolved)).scheme.lower() not in SAFE_SCHEMES:
+            return None
+    except ValueError:
+        return None
+    return resolved
+
+
+def document_base(root: Node, url: str | None) -> str | None:
+    source = safe_url(url) if url else None
+    for node in walk(root):
+        if node.tag != "base" or "href" not in node.attrs:
+            continue
+        candidate = safe_url(node.attrs["href"], source)
+        if candidate:
+            parts = urlsplit(candidate)
+            if parts.scheme in {"http", "https"} and parts.netloc:
+                return candidate
+    return source
+
+
+def destination(value: str) -> str:
+    # Parentheses and whitespace must not terminate a Markdown destination.
+    return quote(value, safe="/:?#@!$&'*+,;=%[]~_-.")

htomd-0.1.0.dist-info/METADATA

@@ -0,0 +1,109 @@
+Metadata-Version: 2.4
+Name: htomd
+Version: 0.1.0
+Summary: Focused Markdown and metadata from messy HTML, in pure Python
+License-Expression: MIT
+Project-URL: Repository, https://github.com/jamiedavenport/htomd
+Project-URL: Issues, https://github.com/jamiedavenport/htomd/issues
+Project-URL: Changelog, https://github.com/jamiedavenport/htomd/blob/main/CHANGELOG.md
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# htomd
+
+Extract Markdown and metadata from HTML. Pure Python 3.12+, with no runtime
+dependencies or network access.
+
+## Installation
+
+```sh
+python -m pip install htomd
+```
+
+For the command line, install in an isolated tool environment:
+
+```sh
+uv tool install htomd
+```
+
+## Python API
+
+```python
+import htomd
+
+html = "<article><h1>Hello</h1><p>Readable text.</p></article>"
+markdown = htomd.convert(html)
+document = htomd.extract(html, url="https://example.com/article")
+print(document.markdown, document.metadata.title, document.diagnostics)
+```
+
+Pass decoded HTML strings. `url` resolves relative references. `extract()` returns
+immutable results; missing metadata is `None`. Empty content yields empty Markdown;
+invalid argument types raise `TypeError`.
+
+Best suited to articles and documentation. Extraction can miss content or retain
+clutter. JavaScript, browser layout, math, and SVG are unsupported. Simple tables
+use GFM; complex tables become row/cell text.
+
+## Command line
+
+Installing the package also installs the `htomd` command. To install a locally
+built wheel in a virtual environment:
+
+```sh
+python -m venv .venv
+source .venv/bin/activate
+python -m pip install dist/htomd-0.1.0-py3-none-any.whl
+```
+
+In Windows PowerShell, activate with `.venv\Scripts\Activate.ps1` instead.
+See [Contributing](https://github.com/jamiedavenport/htomd/blob/main/CONTRIBUTING.md#releases)
+for the build commands.
+
+```sh
+curl -s https://example.com/article | htomd convert --url https://example.com/article
+cat page.html | htomd convert > page.md
+cat page.html | htomd extract > page.json
+```
+
+Both commands read stdin to EOF and accept no file arguments. Input must be UTF-8
+(an optional UTF-8 BOM is accepted); output is UTF-8. `--url` supplies source
+context for relative references and metadata; it never fetches the URL.
+
+`convert` writes Markdown unchanged. `extract` writes indented JSON containing
+`markdown`, `metadata`, and `diagnostics`, matching the Python result structure.
+Missing metadata is `null`; diagnostic notes are an array. Empty or malformed
+HTML receives the same best-effort handling as the library.
+
+Run `htomd`, `htomd help`, or `htomd --help` for usage and pipeline examples.
+Use `htomd help convert` or `htomd convert --help` for command-specific help
+(likewise for `extract`). `htomd version` or `htomd --version` prints the installed
+package version. Help and version commands exit successfully without reading stdin.
+
+`python -m htomd` supports the same commands. Successful conversion exits with code 0, I/O and
+decoding failures with code 1, and usage errors with code 2. Errors go to stderr.
+
+## Development
+
+```sh
+mise trust
+mise install
+mise run setup
+mise exec -- uv run --locked pytest
+```
+
+`mise run check` runs all checks. See
+[CONTRIBUTING.md](https://github.com/jamiedavenport/htomd/blob/main/CONTRIBUTING.md)
+for hooks and releases.
+
+MIT license, copyright 2026 JXD Ltd.
+[Fixtures](https://github.com/jamiedavenport/htomd/blob/main/tests/fixtures/real/README.md)
+have separate licenses.

htomd-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+htomd/__init__.py,sha256=5jdmI8vbSl1GzcK7AINC75sUI_AGJ_gloJv96DRnu8s,1478
+htomd/__main__.py,sha256=8NSrSYJ5OvYnJlgKGVngL5bpg8mmlguEyu6-YGRikuE,116
+htomd/_cli.py,sha256=WlTTrmFGCEVjYrfF8FiryqhmzeejcNOCMom27vcAM00,3411
+htomd/_metadata.py,sha256=v8MSFdIsrU2QHcPQwSFQ8ynUKNK7UsltJhd3tOlfyUs,4868
+htomd/_models.py,sha256=wRcOs3Wldeh_yxJX5zqSQb28ORe6ZXg6ylr5tdofoH0,692
+htomd/_parser.py,sha256=GSroMryc-lELfoULeUFbtMdjzPhI1cVWLXkV9MGyWY4,4119
+htomd/_render.py,sha256=Q7OS7M2L9tU9BtPgBpnnaE0yxleat-by_GJ2U4BGluw,9103
+htomd/_selection.py,sha256=xS2k5fHGN2hYWYUH5B9y0ZDFcgHKpCRqj_bNlknhuzI,11634
+htomd/_tree.py,sha256=AnhOyo3YGE-CgU-WLcDigoOYyAwkwv9_N5pFXjXmZSQ,1504
+htomd/_urls.py,sha256=gpaEms-Svr1mr3hzD_9zeWTzCmpGZbITM9ZDwrybG7o,1515
+htomd/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+htomd-0.1.0.dist-info/licenses/LICENSE,sha256=VHLXTDfL3E6-kxgxwOIVcf7-8NvDOP3x-Kk6kanwWF4,1064
+htomd-0.1.0.dist-info/METADATA,sha256=eVXT1zy6U7OuPVA7m3EP2mSujykAPc54DUYu2BI_NHQ,3782
+htomd-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+htomd-0.1.0.dist-info/entry_points.txt,sha256=v29vVSHQxcHtaJcFMD_yFeWIrXmNoxNdiXoJgcnpqJM,42
+htomd-0.1.0.dist-info/top_level.txt,sha256=EzaqcXzCQSBq-SYUbgIvrK1-c5wRXFNDrhmGcyFctNY,6
+htomd-0.1.0.dist-info/RECORD,,

htomd-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

htomd-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+htomd = htomd._cli:main

htomd-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 JXD Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

htomd-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+htomd