html-to-markdown 1.0.0__py3-none-any.whl

html_to_markdown/__init__.py

@@ -0,0 +1,3 @@
+from html_to_markdown.processing import convert_to_markdown
+
+__all__ = ["convert_to_markdown"]

html_to_markdown/__main__.py

@@ -0,0 +1,131 @@
+import argparse
+import sys
+
+from html_to_markdown import convert_to_markdown
+from html_to_markdown.constants import ASTERISK, ATX, ATX_CLOSED, BACKSLASH, SPACES, UNDERLINED, UNDERSCORE
+
+
+def cli(argv: list[str]) -> None:
+    """Command-line interface for html_to_markdown."""
+    parser = argparse.ArgumentParser(
+        prog="html_to_markdown",
+        description="Converts html to markdown.",
+    )
+
+    parser.add_argument(
+        "html",
+        nargs="?",
+        type=argparse.FileType("r"),
+        default=sys.stdin,
+        help="The html file to convert. Defaults to STDIN if not " "provided.",
+    )
+    parser.add_argument(
+        "-s",
+        "--strip",
+        nargs="*",
+        help="A list of tags to strip. This option can't be used with " "the --convert option.",
+    )
+    parser.add_argument(
+        "-c",
+        "--convert",
+        nargs="*",
+        help="A list of tags to convert. This option can't be used with " "the --strip option.",
+    )
+    parser.add_argument(
+        "-a",
+        "--autolinks",
+        action="store_true",
+        help="A boolean indicating whether the 'automatic link' style "
+        "should be used when a 'a' tag's contents match its href.",
+    )
+    parser.add_argument(
+        "--default-title",
+        action="store_false",
+        help="A boolean to enable setting the title of a link to its " "href, if no title is given.",
+    )
+    parser.add_argument(
+        "--heading-style",
+        default=UNDERLINED,
+        choices=(ATX, ATX_CLOSED, UNDERLINED),
+        help="Defines how headings should be converted.",
+    )
+    parser.add_argument(
+        "-b",
+        "--bullets",
+        default="*+-",
+        help="A string of bullet styles to use; the bullet will " "alternate based on nesting level.",
+    )
+    (
+        parser.add_argument(
+            "--strong-em-symbol",
+            default=ASTERISK,
+            choices=(ASTERISK, UNDERSCORE),
+            help="Use * or _ to convert strong and italics text",
+        ),
+    )
+    parser.add_argument("--sub-symbol", default="", help="Define the chars that surround '<sub>'.")
+    parser.add_argument("--sup-symbol", default="", help="Define the chars that surround '<sup>'.")
+    parser.add_argument(
+        "--newline-style",
+        default=SPACES,
+        choices=(SPACES, BACKSLASH),
+        help="Defines the style of <br> conversions: two spaces "
+        "or backslash at the and of the line thet should break.",
+    )
+    parser.add_argument(
+        "--code-language", default="", help="Defines the language that should be assumed for all " "'<pre>' sections."
+    )
+    parser.add_argument(
+        "--no-escape-asterisks",
+        dest="escape_asterisks",
+        action="store_false",
+        help="Do not escape '*' to '\\*' in text.",
+    )
+    parser.add_argument(
+        "--no-escape-underscores",
+        dest="escape_underscores",
+        action="store_false",
+        help="Do not escape '_' to '\\_' in text.",
+    )
+    parser.add_argument(
+        "-i",
+        "--keep-inline-images-in",
+        nargs="*",
+        help="Images are converted to their alt-text when the images are "
+        "located inside headlines or table cells. If some inline images "
+        "should be converted to markdown images instead, this option can "
+        "be set to a list of parent tags that should be allowed to "
+        "contain inline images.",
+    )
+    parser.add_argument(
+        "-w", "--wrap", action="store_true", help="Wrap all text paragraphs at --wrap-width characters."
+    )
+    parser.add_argument("--wrap-width", type=int, default=80)
+
+    args = parser.parse_args(argv)
+
+    result = convert_to_markdown(
+        args.html.read(),
+        strip=args.strip,
+        convert=args.convert,
+        autolinks=args.autolinks,
+        default_title=args.default_title,
+        heading_style=args.heading_style,
+        bullets=args.bullets,
+        strong_em_symbol=args.strong_em_symbol,
+        sub_symbol=args.sub_symbol,
+        sup_symbol=args.sup_symbol,
+        newline_style=args.newline_style,
+        code_language=args.code_language,
+        escape_asterisks=args.escape_asterisks,
+        escape_underscores=args.escape_underscores,
+        keep_inline_images_in=args.keep_inline_images_in,
+        wrap=args.wrap,
+        wrap_width=args.wrap_width,
+    )
+
+    print(result)  # noqa: T201
+
+
+if __name__ == "__main__":
+    cli(sys.argv[1:])

html_to_markdown/constants.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import re
+from re import Pattern
+from typing import Final, Literal
+
+convert_heading_re: Final[Pattern[str]] = re.compile(r"convert_h(\d+)")
+line_beginning_re: Final[Pattern[str]] = re.compile(r"^", re.MULTILINE)
+whitespace_re: Final[Pattern[str]] = re.compile(r"[\t ]+")
+html_heading_re: Final[Pattern[str]] = re.compile(r"h[1-6]")
+
+ASTERISK: Final[Literal["*"]] = "*"
+ATX: Final[Literal["atx"]] = "atx"
+ATX_CLOSED: Final[Literal["atx_closed"]] = "atx_closed"
+BACKSLASH: Final[Literal["backslash"]] = "backslash"
+UNDERLINED: Final[Literal["underlined"]] = "underlined"
+SPACES: Final[Literal["spaces"]] = "spaces"
+UNDERSCORE: Final[Literal["_"]] = "_"

html_to_markdown/converters.py

@@ -0,0 +1,380 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from functools import partial
+from inspect import getfullargspec
+from textwrap import fill
+from typing import Any, Callable, Literal, TypeVar, cast
+
+from bs4.element import Tag
+
+from html_to_markdown.constants import (
+    ATX_CLOSED,
+    BACKSLASH,
+    UNDERLINED,
+    line_beginning_re,
+)
+from html_to_markdown.utils import chomp, indent, underline
+
+SupportedElements = Literal[
+    "a",
+    "b",
+    "blockquote",
+    "br",
+    "code",
+    "del",
+    "em",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "hr",
+    "i",
+    "img",
+    "list",
+    "ul",
+    "ol",
+    "li",
+    "p",
+    "pre",
+    "script",
+    "style",
+    "s",
+    "strong",
+    "samp",
+    "sub",
+    "sup",
+    "table",
+    "caption",
+    "figcaption",
+    "td",
+    "th",
+    "tr",
+    "kbd",
+]
+
+ConvertsMap = Mapping[SupportedElements, Callable[[str, Tag], str]]
+
+T = TypeVar("T")
+
+
+def _create_inline_converter(markup_prefix: str) -> Callable[[Tag, str], str]:
+    """This abstracts all simple inline tags like b, em, del, ...
+    Returns a function that wraps the chomped text in a pair of the string
+    that is returned by markup_fn, with '/' inserted in the string used after
+    the text if it looks like an HTML tag. markup_fn is necessary to allow for
+    references to self.strong_em_symbol etc.
+    """
+
+    def implementation(*, tag: Tag, text: str) -> str:
+        if tag.find_parent(["pre", "code", "kbd", "samp"]):
+            return text
+
+        if not text.strip():
+            return ""
+
+        markup_suffix = markup_prefix
+        if markup_prefix.startswith("<") and markup_prefix.endswith(">"):
+            markup_suffix = "</" + markup_prefix[1:]
+
+        prefix, suffix, text = chomp(text)
+
+        return f"{prefix}{markup_prefix}{text}{markup_suffix}{suffix}"
+
+    return cast(Callable[[Tag, str], str], implementation)
+
+
+def _get_colspan(tag: Tag) -> int:
+    colspan = 1
+
+    if "colspan" in tag.attrs and isinstance(tag["colspan"], str) and tag["colspan"].isdigit():
+        colspan = int(tag["colspan"])
+
+    return colspan
+
+
+def _convert_a(*, tag: Tag, text: str, autolinks: bool, default_title: bool) -> str:
+    prefix, suffix, text = chomp(text)
+    if not text:
+        return ""
+
+    href = tag.get("href")
+    title = tag.get("title")
+
+    if autolinks and text.replace(r"\_", "_") == href and not title and not default_title:
+        return f"<{href}>"
+
+    if default_title and not title:
+        title = href
+
+    title_part = ' "{}"'.format(title.replace('"', r"\"")) if isinstance(title, str) else ""
+    return f"{prefix}[{text}]({href}{title_part}){suffix}" if href else text
+
+
+def _convert_blockquote(*, text: str, convert_as_inline: bool) -> str:
+    if convert_as_inline:
+        return text
+    return f"\n{line_beginning_re.sub('> ', text.strip())}\n\n" if text else ""
+
+
+def _convert_br(*, convert_as_inline: bool, newline_style: str) -> str:
+    if convert_as_inline:
+        return ""
+    return "\\\n" if newline_style.lower() == BACKSLASH else "  \n"
+
+
+def _convert_hn(
+    *,
+    n: int,
+    heading_style: Literal["atx", "atx_closed", "underlined"],
+    text: str,
+    convert_as_inline: bool,
+) -> str:
+    if convert_as_inline:
+        return text
+
+    text = text.strip()
+    if heading_style == UNDERLINED and n <= 2:
+        return underline(text=text, pad_char="=" if n == 1 else "-")
+
+    hashes = "#" * n
+    if heading_style == ATX_CLOSED:
+        return f"{hashes} {text} {hashes}\n\n"
+
+    return f"{hashes} {text}\n\n"
+
+
+def _convert_img(*, tag: Tag, convert_as_inline: bool, keep_inline_images_in: Iterable[str] | None) -> str:
+    alt = tag.attrs.get("alt", None) or ""
+    src = tag.attrs.get("src", None) or ""
+    title = tag.attrs.get("title", None) or ""
+    title_part = ' "{}"'.format(title.replace('"', r"\"")) if title else ""
+    parent_name = tag.parent.name if tag.parent else ""
+    if convert_as_inline and parent_name not in (keep_inline_images_in or []):
+        return alt
+
+    return f"![{alt}]({src}{title_part})"
+
+
+def _convert_list(*, tag: Tag, text: str) -> str:
+    nested = False
+
+    before_paragraph = False
+    if tag.next_sibling and getattr(tag.next_sibling, "name", None) not in {"ul", "ol"}:
+        before_paragraph = True
+
+    while tag:
+        if tag.name == "li":
+            nested = True
+            break
+
+        if not tag.parent:
+            break
+
+        tag = tag.parent
+
+    if nested:
+        return "\n" + indent(text=text, level=1).rstrip()
+
+    return text + ("\n" if before_paragraph else "")
+
+
+def _convert_li(*, tag: Tag, text: str, bullets: str) -> str:
+    parent = tag.parent
+    if parent is not None and parent.name == "ol":
+        start = (
+            int(cast(str, parent["start"]))
+            if isinstance(parent.get("start"), str) and str(parent.get("start")).isnumeric()
+            else 1
+        )
+        bullet = "%s." % (start + parent.index(tag))
+    else:
+        depth = -1
+        while tag:
+            if tag.name == "ul":
+                depth += 1
+            if not tag.parent:
+                break
+
+            tag = tag.parent
+
+        bullet = bullets[depth % len(bullets)]
+    return "{} {}\n".format(bullet, (text or "").strip())
+
+
+def _convert_p(*, wrap: bool, text: str, convert_as_inline: bool, wrap_width: int) -> str:
+    if convert_as_inline:
+        return text
+
+    if wrap:
+        text = fill(
+            text,
+            width=wrap_width,
+            break_long_words=False,
+            break_on_hyphens=False,
+        )
+
+    return f"{text}\n\n" if text else ""
+
+
+def _convert_pre(
+    *,
+    tag: Tag,
+    text: str,
+    code_language: str,
+    code_language_callback: Callable[[Tag], str] | None,
+) -> str:
+    if not text:
+        return ""
+
+    if code_language_callback:
+        code_language = code_language_callback(tag) or code_language
+
+    return f"\n```{code_language}\n{text}\n```\n"
+
+
+def _convert_td(*, tag: Tag, text: str) -> str:
+    colspan = _get_colspan(tag)
+    return " " + text.strip().replace("\n", " ") + " |" * colspan
+
+
+def _convert_th(*, tag: Tag, text: str) -> str:
+    colspan = _get_colspan(tag)
+    return " " + text.strip().replace("\n", " ") + " |" * colspan
+
+
+def _convert_tr(*, tag: Tag, text: str) -> str:
+    cells = tag.find_all(["td", "th"])
+    parent_name = tag.parent.name if tag.parent else ""
+    tag_grand_parent = tag.parent.parent if tag.parent else None
+    is_headrow = (
+        all(cell.name == "th" for cell in cells)
+        or (not tag.previous_sibling and parent_name != "tbody")
+        or (
+            not tag.previous_sibling
+            and parent_name == "tbody"
+            and (not tag_grand_parent or len(tag_grand_parent.find_all(["thead"])) < 1)
+        )
+    )
+    overline = ""
+    underline = ""
+    if is_headrow and not tag.previous_sibling:
+        # first row and is headline: print headline underline
+        full_colspan = 0
+        for cell in cells:
+            if "colspan" in cell.attrs and cell["colspan"].isdigit():
+                full_colspan += int(cell["colspan"])
+            else:
+                full_colspan += 1
+        underline += "| " + " | ".join(["---"] * full_colspan) + " |" + "\n"
+    elif not tag.previous_sibling and (
+        parent_name == "table" or (parent_name == "tbody" and not cast(Tag, tag.parent).previous_sibling)
+    ):
+        # first row, not headline, and:
+        # - the parent is table or
+        # - the parent is tbody at the beginning of a table.
+        # print empty headline above this row
+        overline += "| " + " | ".join([""] * len(cells)) + " |" + "\n"
+        overline += "| " + " | ".join(["---"] * len(cells)) + " |" + "\n"
+    return overline + "|" + text + "\n" + underline
+
+
+def create_converters_map(
+    autolinks: bool,
+    bullets: str,
+    code_language: str,
+    code_language_callback: Callable[[Tag], str] | None,
+    default_title: bool,
+    heading_style: Literal["atx", "atx_closed", "underlined"],
+    keep_inline_images_in: Iterable[str] | None,
+    newline_style: str,
+    strong_em_symbol: str,
+    sub_symbol: str,
+    sup_symbol: str,
+    wrap: bool,
+    wrap_width: int,
+) -> ConvertsMap:
+    """Create a mapping of HTML elements to their corresponding conversion functions.
+
+    Args:
+        autolinks: Whether to convert URLs into links.
+        bullets: The bullet characters to use for unordered lists.
+        code_language: The default code language to use.
+        code_language_callback: A callback to get the code language.
+        default_title: Whether to use the URL as the title for links.
+        heading_style: The style of headings.
+        keep_inline_images_in: The tags to keep inline images in.
+        newline_style: The style of newlines.
+        strong_em_symbol: The symbol to use for strong and emphasis text.
+        sub_symbol: The symbol to use for subscript text.
+        sup_symbol: The symbol to use for superscript text.
+        wrap: Whether to wrap text.
+        wrap_width: The width to wrap text at.
+
+    Returns:
+        A mapping of HTML elements to their corresponding conversion functions
+    """
+
+    def _wrapper(func: Callable[..., T]) -> Callable[[str, Tag], T]:
+        spec = getfullargspec(func)
+
+        def _inner(*, text: str, tag: Tag, convert_as_inline: bool) -> T:
+            if spec.kwonlyargs:
+                kwargs: dict[str, Any] = {}
+                if "tag" in spec.kwonlyargs:
+                    kwargs["tag"] = tag
+                if "text" in spec.kwonlyargs:
+                    kwargs["text"] = text
+                if "convert_as_inline" in spec.kwonlyargs:
+                    kwargs["convert_as_inline"] = convert_as_inline
+                return func(**kwargs)
+            return func(text)
+
+        return cast(Callable[[str, Tag], T], _inner)
+
+    return {
+        "a": _wrapper(partial(_convert_a, autolinks=autolinks, default_title=default_title)),
+        "b": _wrapper(partial(_create_inline_converter(2 * strong_em_symbol))),
+        "blockquote": _wrapper(partial(_convert_blockquote)),
+        "br": _wrapper(partial(_convert_br, newline_style=newline_style)),
+        "code": _wrapper(_create_inline_converter("`")),
+        "del": _wrapper(_create_inline_converter("~~")),
+        "em": _wrapper(_create_inline_converter(strong_em_symbol)),
+        "h1": _wrapper(partial(_convert_hn, n=1, heading_style=heading_style)),
+        "h2": _wrapper(partial(_convert_hn, n=2, heading_style=heading_style)),
+        "h3": _wrapper(partial(_convert_hn, n=3, heading_style=heading_style)),
+        "h4": _wrapper(partial(_convert_hn, n=4, heading_style=heading_style)),
+        "h5": _wrapper(partial(_convert_hn, n=5, heading_style=heading_style)),
+        "h6": _wrapper(partial(_convert_hn, n=6, heading_style=heading_style)),
+        "hr": _wrapper(lambda _: "\n\n---\n\n"),
+        "i": _wrapper(partial(_create_inline_converter(strong_em_symbol))),
+        "img": _wrapper(partial(_convert_img, keep_inline_images_in=keep_inline_images_in)),
+        "list": _wrapper(_convert_list),
+        "ul": _wrapper(_convert_list),
+        "ol": _wrapper(_convert_list),
+        "li": _wrapper(partial(_convert_li, bullets=bullets)),
+        "p": _wrapper(partial(_convert_p, wrap=wrap, wrap_width=wrap_width)),
+        "pre": _wrapper(
+            partial(
+                _convert_pre,
+                code_language=code_language,
+                code_language_callback=code_language_callback,
+            )
+        ),
+        "script": _wrapper(lambda _: ""),
+        "style": _wrapper(lambda _: ""),
+        "s": _wrapper(_create_inline_converter("~~")),
+        "strong": _wrapper(_create_inline_converter(strong_em_symbol * 2)),
+        "samp": _wrapper(_create_inline_converter("`")),
+        "sub": _wrapper(_create_inline_converter(sub_symbol)),
+        "sup": _wrapper(_create_inline_converter(sup_symbol)),
+        "table": _wrapper(lambda text: f"\n\n{text}\n"),
+        "caption": _wrapper(lambda text: f"{text}\n"),
+        "figcaption": _wrapper(lambda text: f"\n\n{text}\n\n"),
+        "td": _wrapper(_convert_td),
+        "th": _wrapper(_convert_th),
+        "tr": _wrapper(_convert_tr),
+        "kbd": _wrapper(_create_inline_converter("`")),
+    }

html_to_markdown/processing.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable, Literal, cast
+
+from bs4 import BeautifulSoup, Comment, Doctype, NavigableString, Tag
+
+from html_to_markdown.constants import (
+    ASTERISK,
+    SPACES,
+    UNDERLINED,
+    html_heading_re,
+    whitespace_re,
+)
+from html_to_markdown.converters import ConvertsMap, create_converters_map
+from html_to_markdown.utils import escape
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from bs4 import PageElement
+
+SupportedTag = Literal[
+    "a",
+    "b",
+    "blockquote",
+    "br",
+    "code",
+    "del",
+    "em",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "hr",
+    "i",
+    "img",
+    "list",
+    "ul",
+    "ol",
+    "li",
+    "p",
+    "pre",
+    "script",
+    "style",
+    "s",
+    "strong",
+    "samp",
+    "sub",
+    "sup",
+    "table",
+    "caption",
+    "figcaption",
+    "td",
+    "th",
+    "tr",
+    "kbd",
+]
+
+
+def _is_nested_tag(el: PageElement) -> bool:
+    return isinstance(el, Tag) and el.name in {
+        "ol",
+        "ul",
+        "li",
+        "table",
+        "thead",
+        "tbody",
+        "tfoot",
+        "tr",
+        "td",
+        "th",
+    }
+
+
+def _process_tag(
+    tag: Tag,
+    *,
+    autolinks: bool,
+    bullets: str,
+    code_language: str,
+    code_language_callback: Callable[[Any], str] | None,
+    convert: Iterable[str] | None,
+    convert_as_inline: bool = False,
+    converters_map: ConvertsMap | None = None,
+    default_title: bool,
+    escape_asterisks: bool,
+    escape_misc: bool,
+    escape_underscores: bool,
+    heading_style: Literal["atx", "atx_closed", "underlined"],
+    keep_inline_images_in: Iterable[str] | None,
+    newline_style: str,
+    strip: Iterable[str] | None,
+    strong_em_symbol: str,
+    sub_symbol: str,
+    sup_symbol: str,
+    wrap: bool,
+    wrap_width: int,
+) -> str:
+    if converters_map is None:
+        converters_map = create_converters_map(
+            autolinks=autolinks,
+            bullets=bullets,
+            code_language=code_language,
+            code_language_callback=code_language_callback,
+            default_title=default_title,
+            heading_style=heading_style,
+            keep_inline_images_in=keep_inline_images_in,
+            newline_style=newline_style,
+            strong_em_symbol=strong_em_symbol,
+            sub_symbol=sub_symbol,
+            sup_symbol=sup_symbol,
+            wrap=wrap,
+            wrap_width=wrap_width,
+        )
+
+    text = ""
+    is_heading = html_heading_re.match(tag.name) is not None
+    is_cell = tag.name in {"td", "th"}
+    convert_children_as_inline = convert_as_inline or is_heading or is_cell
+
+    if _is_nested_tag(tag):
+        for el in tag.children:
+            can_extract = (
+                not el.previous_sibling
+                or not el.next_sibling
+                or _is_nested_tag(el.previous_sibling)
+                or _is_nested_tag(el.next_sibling)
+            )
+            if can_extract and isinstance(el, NavigableString) and not el.strip():
+                el.extract()
+
+    for el in filter(lambda value: not isinstance(value, (Comment, Doctype)), tag.children):
+        if isinstance(el, NavigableString):
+            text += _process_text(
+                el=el,
+                escape_misc=escape_misc,
+                escape_asterisks=escape_asterisks,
+                escape_underscores=escape_underscores,
+            )
+        elif isinstance(el, Tag):
+            text += _process_tag(
+                tag=el,
+                convert_as_inline=convert_children_as_inline,
+                strip=strip,
+                convert=convert,
+                escape_misc=escape_misc,
+                escape_asterisks=escape_asterisks,
+                escape_underscores=escape_underscores,
+                converters_map=converters_map,
+                autolinks=autolinks,
+                bullets=bullets,
+                code_language=code_language,
+                code_language_callback=code_language_callback,
+                default_title=default_title,
+                heading_style=heading_style,
+                keep_inline_images_in=keep_inline_images_in,
+                newline_style=newline_style,
+                strong_em_symbol=strong_em_symbol,
+                sub_symbol=sub_symbol,
+                sup_symbol=sup_symbol,
+                wrap=wrap,
+                wrap_width=wrap_width,
+            )
+
+    tag_name: SupportedTag | None = cast(SupportedTag, tag.name.lower()) if tag.name.lower() in converters_map else None
+
+    if tag_name and _should_convert_tag(tag_name=tag.name, strip=strip, convert=convert):
+        return converters_map[tag_name](  # type: ignore[call-arg]
+            tag=tag, text=text, convert_as_inline=convert_as_inline
+        )
+
+    return text
+
+
+def _process_text(
+    *,
+    el: NavigableString,
+    escape_misc: bool,
+    escape_asterisks: bool,
+    escape_underscores: bool,
+) -> str:
+    text = str(el) or ""
+
+    # normalize whitespace if we're not inside a preformatted element
+    if not el.find_parent("pre"):
+        text = whitespace_re.sub(" ", text)
+
+    # escape special characters if we're not inside a preformatted or code element
+    if not el.find_parent(["pre", "code", "kbd", "samp"]):
+        text = escape(
+            text=text,
+            escape_misc=escape_misc,
+            escape_asterisks=escape_asterisks,
+            escape_underscores=escape_underscores,
+        )
+
+    # remove trailing whitespaces if any of the following condition is true:
+    # - current text node is the last node in li
+    # - current text node is followed by an embedded list
+    if (
+        el.parent
+        and el.parent.name == "li"
+        and (not el.next_sibling or getattr(el.next_sibling, "name", None) in {"ul", "ol"})
+    ):
+        text = text.rstrip()
+
+    return text
+
+
+def _should_convert_tag(*, tag_name: str, strip: Iterable[str] | None, convert: Iterable[str] | None) -> bool:
+    if strip is not None:
+        return tag_name not in strip
+    if convert is not None:
+        return tag_name in convert
+    return True
+
+
+def convert_to_markdown(
+    html: str,
+    *,
+    soup: BeautifulSoup | None = None,
+    autolinks: bool = True,
+    bullets: str = "*+-",
+    code_language: str = "",
+    code_language_callback: Callable[[Any], str] | None = None,
+    convert: Iterable[str] | None = None,
+    default_title: bool = False,
+    escape_asterisks: bool = True,
+    escape_misc: bool = True,
+    escape_underscores: bool = True,
+    heading_style: Literal["underlined", "atx", "atx_closed"] = UNDERLINED,
+    keep_inline_images_in: Iterable[str] | None = None,
+    newline_style: Literal["spaces", "backslash"] = SPACES,
+    strip: Iterable[str] | None = None,
+    strong_em_symbol: Literal["*", "_"] = ASTERISK,
+    sub_symbol: str = "",
+    sup_symbol: str = "",
+    wrap: bool = False,
+    wrap_width: int = 80,
+    convert_as_inline: bool = False,
+) -> str:
+    """Convert HTML to Markdown.
+
+    Args:
+        html: The HTML to convert.
+        soup: The BeautifulSoup object to convert.
+        autolinks: Whether to convert links to Markdown.
+        bullets: The bullet characters to use for unordered lists.
+        code_language: The default code language to use.
+        code_language_callback: A callback function to determine the code language.
+        convert: The HTML elements to convert.
+        default_title: Whether to use the default title.
+        escape_asterisks: Whether to escape asterisks.
+        escape_misc: Whether to escape miscellaneous characters.
+        escape_underscores: Whether to escape underscores.
+        heading_style: The style to use for headings.
+        keep_inline_images_in: The tags to keep inline images in.
+        newline_style: The style to use for newlines.
+        strip: The HTML elements to strip.
+        strong_em_symbol: The symbol to use for strong and emphasis.
+        sub_symbol: The symbol to use for subscript.
+        sup_symbol: The symbol to use for superscript.
+        wrap: Whether to wrap text.
+        wrap_width: The width to wrap text at.
+        convert_as_inline: Whether to convert elements as inline.
+
+    Returns:
+        The Markdown.
+    """
+    if soup is None:
+        from bs4 import BeautifulSoup
+
+        soup = BeautifulSoup(html, "html.parser")
+
+    return _process_tag(
+        autolinks=autolinks,
+        bullets=bullets,
+        code_language=code_language,
+        code_language_callback=code_language_callback,
+        convert=convert,
+        convert_as_inline=convert_as_inline,
+        default_title=default_title,
+        escape_asterisks=escape_asterisks,
+        escape_misc=escape_misc,
+        escape_underscores=escape_underscores,
+        heading_style=heading_style,
+        keep_inline_images_in=keep_inline_images_in,
+        newline_style=newline_style,
+        strip=strip,
+        strong_em_symbol=strong_em_symbol,
+        sub_symbol=sub_symbol,
+        sup_symbol=sup_symbol,
+        tag=soup,
+        wrap=wrap,
+        wrap_width=wrap_width,
+    )

html_to_markdown/utils.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import re
+
+from html_to_markdown.constants import line_beginning_re
+
+
+def chomp(text: str) -> tuple[str, str, str]:
+    """If the text in an inline tag like b, a, or em contains a leading or trailing
+    space, strip the string and return a space as suffix of prefix, if needed.
+
+    Args:
+        text: The text to chomp.
+
+    Returns:
+        A tuple containing the prefix, suffix, and the stripped text.
+    """
+    prefix = " " if text and text[0] == " " else ""
+    suffix = " " if text and text[-1] == " " else ""
+    text = text.strip()
+    return prefix, suffix, text
+
+
+def escape(*, text: str, escape_misc: bool, escape_asterisks: bool, escape_underscores: bool) -> str:
+    """Escape special characters in text.
+
+    Args:
+        text: The text to escape.
+        escape_misc: Whether to escape miscellaneous characters.
+        escape_asterisks: Whether to escape asterisks.
+        escape_underscores: Whether to escape underscores.
+
+    Returns:
+        The escaped text.
+    """
+    if not text:
+        return ""
+    if escape_misc:
+        text = re.sub(r"([\\&<`[>~#=+|-])", r"\\\1", text)
+        text = re.sub(r"([0-9])([.)])", r"\1\\\2", text)
+    if escape_asterisks:
+        text = text.replace("*", r"\*")
+    if escape_underscores:
+        text = text.replace("_", r"\_")
+    return text
+
+
+def indent(*, text: str, level: int) -> str:
+    """Indent text by a given level.
+
+    Args:
+        text: The text to indent.
+        level: The level of indentation.
+
+    Returns:
+        The indented text.
+    """
+    return line_beginning_re.sub("\t" * level, text) if text else ""
+
+
+def underline(*, text: str, pad_char: str) -> str:
+    """Underline text with a given character.
+
+    Args:
+        text: The text to underline.
+        pad_char: The character to use for underlining.
+
+    Returns:
+        The underlined text.
+    """
+    text = (text or "").rstrip()
+    return f"{text}\n{pad_char * len(text)}\n\n" if text else ""

html_to_markdown-1.0.0.dist-info/METADATA

@@ -0,0 +1,194 @@
+Metadata-Version: 2.3
+Name: html-to-markdown
+Version: 1.0.0
+Summary: Convert HTML to markdown
+Author-email: Na'aman Hirschfeld <nhirschfeld@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: beautifulsoup,converter,html,markdown,text-processing
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Text Processing :: Markup
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Utilities
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: beautifulsoup4>=4.12.3
+Description-Content-Type: text/markdown
+
+# html_to_markdown
+
+This library is a refactored and modernized fork of [markdownify](https://pypi.org/project/markdownify/), supporting
+Python 3.9 and offering strong typing.
+
+### Differences from the Markdownify
+
+- The refactored codebase uses a strict functional approach - no classes are involved.
+- There is full typing with strict MyPy adherence in place.
+- The `convert_to_markdown` allows passing a pre-configured instance of `Beautifulsoup`.
+- This library releases follows standard semver. Its version v1.0.0 was branched from markdownify's v0.13.1, at which
+  point versioning is no longer aligned.
+
+## Installation
+
+```shell
+pip install html_to_markdown
+```
+
+## Usage
+
+Convert some HTML to Markdown:
+
+```python
+from html_to_markdown import convert_to_markdown
+
+convert_to_markdown('<b>Yay</b> <a href="http://github.com">GitHub</a>')  # > '**Yay** [GitHub](http://github.com)'
+```
+
+Specify tags to exclude:
+
+```python
+from html_to_markdown import convert_to_markdown
+
+convert_to_markdown('<b>Yay</b> <a href="http://github.com">GitHub</a>', strip=['a'])  # > '**Yay** GitHub'
+```
+
+\...or specify the tags you want to include:
+
+```python
+from html_to_markdown import convert_to_markdown
+
+convert_to_markdown('<b>Yay</b> <a href="http://github.com">GitHub</a>', convert=['b'])  # > '**Yay** GitHub'
+```
+
+# Options
+
+html_to_markdown supports the following options:
+
+strip
+
+:   A list of tags to strip. This option can\'t be used with the
+`convert` option.
+
+convert
+
+:   A list of tags to convert. This option can\'t be used with the
+`strip` option.
+
+autolinks
+
+:   A boolean indicating whether the \"automatic link\" style should be
+used when a `a` tag\'s contents match its href. Defaults to `True`.
+
+default_title
+
+:   A boolean to enable setting the title of a link to its href, if no
+title is given. Defaults to `False`.
+
+heading_style
+
+:   Defines how headings should be converted. Accepted values are `ATX`,
+`ATX_CLOSED`, `SETEXT`, and `UNDERLINED` (which is an alias for
+`SETEXT`). Defaults to `UNDERLINED`.
+
+bullets
+
+:   An iterable (string, list, or tuple) of bullet styles to be used. If
+the iterable only contains one item, it will be used regardless of
+how deeply lists are nested. Otherwise, the bullet will alternate
+based on nesting level. Defaults to `'*+-'`.
+
+strong_em_symbol
+
+:   In markdown, both `*` and `_` are used to encode **strong** or
+*emphasized* texts. Either of these symbols can be chosen by the
+options `ASTERISK` (default) or `UNDERSCORE` respectively.
+
+sub_symbol, sup_symbol
+
+:   Define the chars that surround `<sub>` and `<sup>` text. Defaults to
+an empty string, because this is non-standard behavior. Could be
+something like `~` and `^` to result in `~sub~` and `^sup^`. If the
+value starts with `<` and ends with `>`, it is treated as an HTML
+tag and a `/` is inserted after the `<` in the string used after the
+text; this allows specifying `<sub>` to use raw HTML in the output
+for subscripts, for example.
+
+newline_style
+
+:   Defines the style of marking linebreaks (`<br>`) in markdown. The
+default value `SPACES` of this option will adopt the usual two
+spaces and a newline, while `BACKSLASH` will convert a linebreak to
+`\\n` (a backslash and a newline). While the latter convention is
+non-standard, it is commonly preferred and supported by a lot of
+interpreters.
+
+code_language
+
+:   Defines the language that should be assumed for all `<pre>`
+sections. Useful, if all code on a page is in the same programming
+language and should be annotated with ``[python]{.title-ref}[ or
+similar. Defaults to ]{.title-ref}[\'\']{.title-ref}\` (empty
+string) and can be any string.
+
+code_language_callback
+
+:   When the HTML code contains `pre` tags that in some way provide the
+code language, for example as class, this callback can be used to
+extract the language from the tag and prefix it to the converted
+`pre` tag. The callback gets one single argument, an BeautifylSoup
+object, and returns a string containing the code language, or
+`None`. An example to use the class name as code language could be:
+
+        def callback(el):
+            return el['class'][0] if el.has_attr('class') else None
+
+    Defaults to `None`.
+
+escape_asterisks
+
+:   If set to `False`, do not escape `*` to `\*` in text. Defaults to
+`True`.
+
+escape_underscores
+
+:   If set to `False`, do not escape `_` to `\_` in text. Defaults to
+`True`.
+
+escape_misc
+
+:   If set to `False`, do not escape miscellaneous punctuation
+characters that sometimes have Markdown significance in text.
+Defaults to `True`.
+
+keep_inline_images_in
+
+:   Images are converted to their alt-text when the images are located
+inside headlines or table cells. If some inline images should be
+converted to markdown images instead, this option can be set to a
+list of parent tags that should be allowed to contain inline images,
+for example `['td']`. Defaults to an empty list.
+
+wrap, wrap_width
+
+:   If `wrap` is set to `True`, all text paragraphs are wrapped at
+`wrap_width` characters. Defaults to `False` and `80`. Use with
+`newline_style=BACKSLASH` to keep line breaks in paragraphs.
+
+Options may be specified as kwargs to the `html_to_markdown` function, or as
+a nested `Options` class in `MarkdownConverter` subclasses.
+
+# CLI
+
+Use `html_to_markdown example.html > example.md` or pipe input from stdin
+(`cat example.html | html_to_markdown > example.md`). Call `html_to_markdown -h`
+to see all available options. They are the same as listed above and take
+the same arguments.

html_to_markdown-1.0.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+html_to_markdown/__init__.py,sha256=_WXeqic-7b6hvidTXkPQwAfLa4YOEAEP-mOUXjx_25k,95
+html_to_markdown/__main__.py,sha256=Wll22XKFmiNSIpdbGzC75b5_Unc3HYOTA6oXA414Tl8,4412
+html_to_markdown/constants.py,sha256=vUjffZ0vFq56jbXF5bBNzomfJwgsp0TWqdUzhkp6bks,687
+html_to_markdown/converters.py,sha256=q1wpzsYl-FRR9qbB983gAkem_-7mgYZ7hOgziofjIDM,12238
+html_to_markdown/processing.py,sha256=9l3zq_kdyvU0TnTk5g4uuYI6Jbu1gY7NQ11u3IBKyFU,9029
+html_to_markdown/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+html_to_markdown/utils.py,sha256=HJUDej5HSpXRtYv-OkCyD0hwnPnVfQCwY6rBRlIOt9s,1989
+html_to_markdown-1.0.0.dist-info/METADATA,sha256=0MObULuhTHiyvVcytDBN_liafpyFjgp5brgoWQYEglA,6478
+html_to_markdown-1.0.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+html_to_markdown-1.0.0.dist-info/entry_points.txt,sha256=jhMqXDYvIyzQDLKjCn4xCyzCCbAMl94tzQx_HiG5Qi0,67
+html_to_markdown-1.0.0.dist-info/licenses/LICENSE,sha256=06BS7zd6oPCrbzAqrThGFboRlbssgBsqDJGqKyZW2Og,1117
+html_to_markdown-1.0.0.dist-info/RECORD,,

html_to_markdown-1.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.25.0
+Root-Is-Purelib: true
+Tag: py3-none-any

html_to_markdown-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+html_to_markdown = html_to_markdown.__main__:cli

html_to_markdown-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright 2012-2018 Matthew Tretter
+Copyright 2024 Na'aman Hirschfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.