domdown 0.1.1__py3-none-any.whl

domdown/__init__.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from .api import html_to_markdown
+from ._core import DomdownOptions, HtmlMetadata, HtmlToMarkdownResult
+from ._pipeline import HtmlToMarkdownPipeline
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    "DomdownOptions",
+    "HtmlMetadata",
+    "HtmlToMarkdownResult",
+    "HtmlToMarkdownPipeline",
+    "html_to_markdown",
+]

domdown/_constants/__init__.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from .selectors import (
+    BOILERPLATE_PHRASES,
+    CONTENT_SELECTORS,
+    CONTENT_SELECTORS_EXACT,
+    CONTENT_SELECTORS_FALLBACK,
+    DEFAULT_REMOVE_SELECTORS,
+    JS_SHELL_PHRASES,
+    HEADER_MARKERS,
+    NOISE_MARKERS,
+    RELATED_PHRASES,
+    REFINABLE_CHILD_TAGS,
+    ROOT_SELECTORS,
+    SHARE_SELECTORS,
+    SKIP_TAGS,
+)
+
+__all__ = [
+    "BOILERPLATE_PHRASES",
+    "CONTENT_SELECTORS",
+    "CONTENT_SELECTORS_EXACT",
+    "CONTENT_SELECTORS_FALLBACK",
+    "DEFAULT_REMOVE_SELECTORS",
+    "JS_SHELL_PHRASES",
+    "HEADER_MARKERS",
+    "NOISE_MARKERS",
+    "RELATED_PHRASES",
+    "REFINABLE_CHILD_TAGS",
+    "ROOT_SELECTORS",
+    "SHARE_SELECTORS",
+    "SKIP_TAGS",
+]

domdown/_constants/selectors.py

@@ -0,0 +1,305 @@
+from __future__ import annotations
+
+# Tags that should be dropped entirely during cleanup and rendering.
+SKIP_TAGS = {
+    "aside",
+    "button",
+    "form",
+    "iframe",
+    "input",
+    "link",
+    "meta",
+    "noscript",
+    "script",
+    "select",
+    "style",
+    "svg",
+    "textarea",
+}
+
+# Selectors that are typically noise in article-like pages.
+SHARE_SELECTORS = (
+    ".share-widget",
+    ".heateor_sss_sharing_container",
+    ".heateor_sss_more",
+    ".more-link",
+    "[class*='share']",
+    "[id*='share']",
+    "[class*='social-sharing']",
+    "[id*='social-sharing']",
+    "[class*='social-links']",
+    "[id*='social-links']",
+    "[class*='socials']",
+    "[id*='socials']",
+)
+
+# Direct children that are still broad enough to be considered content shells
+# during subtree refinement.
+REFINABLE_CHILD_TAGS = (
+    "article",
+    "aside",
+    "blockquote",
+    "div",
+    "figure",
+    "main",
+    "ol",
+    "section",
+    "table",
+    "ul",
+)
+
+DEFAULT_REMOVE_SELECTORS = (
+    ".ad-fixed__wrapper",
+    ".post-hero__ad",
+    ".post-hero__content",
+    ".post-hero__image",
+    ".ad-leaderboard",
+    ".ad__disclaimer",
+    ".ad__title",
+    ".ad__upsell",
+    ".float-share",
+    ".mobile-share",
+    "a[href='#main']",
+    "a[href='#content']",
+    ".post-head",
+    ".post-header",
+    ".post-badge",
+    ".post-access-cta",
+    ".story-tools",
+    ".text-settings",
+    ".text-settings-menu",
+    ".text-settings-dropdown-story",
+    ".text-settings-dropdown-nav",
+    ".comments-wrapper",
+    ".comments-picks-list",
+    ".staff-picks-title",
+    ".wp-forum-connect-comments",
+    ".xf_thread_iframe_wrapper",
+    ".search-container",
+    "#search-result-wrapper",
+    ".searchresults",
+    ".noresults",
+    ".comment-pick",
+    ".single-most-read",
+    ".component-most-read",
+    ".news-form",
+    ".article-info",
+    ".article_util",
+    ".container_postbtn",
+    ".another_category",
+    ".company-box",
+    ".box_comment",
+    ".box_related_article",
+    ".sidebar",
+    "#sidebar",
+    ".et_pb_title_meta_container",
+    ".et_pb_text_0",
+    ".tag-list",
+    ".dsm_open_icon",
+    ".dsm_close_icon",
+    ".dsm-faq-item-open_icon",
+    ".dsm-faq-item-close_icon",
+    ".sharebelow",
+    ".schema_org",
+    ".tags",
+    ".article-categories",
+    "[class*='article-categories']",
+    ".story-title",
+    ".postmeta",
+    "[class*='follow']",
+    "[class*='sponsored']",
+    "[rel*='sponsored']",
+    "[class*='note-b']",
+    "[class*='dog_two']",
+) + SHARE_SELECTORS
+
+# Selectors that usually point at the content subtree inside a larger page shell.
+# Exact selectors are preferred first because substring selectors can match chrome.
+CONTENT_SELECTORS_EXACT = (
+    ".articlebody",
+    ".post-body",
+    ".entry-content",
+    ".content",
+    ".article_cont",
+    ".tt_article_useless_p_margin",
+    ".page--body",
+    "#content",
+    ".content-body",
+    ".post-content",
+    ".post__body",
+    ".post__content",
+    ".s-blog-post__body",
+    ".BodyText__content",
+    ".hb-content__text",
+    ".markdown-body",
+    ".story-body",
+    ".story-shell",
+)
+
+# Broad selectors are only used as a fallback when exact selectors do not find
+# a confident content subtree.
+CONTENT_SELECTORS_FALLBACK = (
+    "[class*='body']",
+    "[id*='body']",
+    "[class*='entry']",
+    "[id*='entry']",
+    "[class*='content']",
+    "[id*='content']",
+)
+
+CONTENT_SELECTORS = CONTENT_SELECTORS_EXACT + CONTENT_SELECTORS_FALLBACK
+
+# Root selectors used to choose the most relevant content shell before cleanup.
+ROOT_SELECTORS = (
+    "article",
+    "main",
+    "main.page--body",
+    "[role='article']",
+    "[role='main']",
+    ".post-body",
+    ".articlebody",
+    ".entry-content",
+    ".content",
+    ".article_cont",
+    ".tt_article_useless_p_margin",
+    ".page--body",
+    "#content",
+    ".content-body",
+    ".post-content",
+    ".post__body",
+    ".post__content",
+    ".s-blog-post__body",
+    ".BodyText__content",
+    ".hb-content__text",
+    ".markdown-body",
+    ".story-body",
+    ".story-shell",
+    "[class*='body']",
+    "[id*='body']",
+    "[class*='entry']",
+    "[id*='entry']",
+    "[class*='content']",
+    "[id*='content']",
+    "body",
+)
+
+# Class or id markers that usually indicate page chrome, ads, or engagement widgets.
+NOISE_MARKERS = (
+    "ai-links",
+    "announcement",
+    "banner",
+    "brand",
+    "author",
+    "bio",
+    "cookie",
+    "share",
+    "social",
+    "breadcrumb",
+    "lead",
+    "deck",
+    "standfirst",
+    "teaser",
+    "excerpt",
+    "related",
+    "recommend",
+    "feedback",
+    "newsletter",
+    "subscribe",
+    "promo",
+    "toolbar",
+    "pagination",
+    "pager",
+    "toc",
+    "table-of-contents",
+    "debug",
+    "cta",
+    "nav",
+    "footer",
+    "tags",
+    "postmeta",
+    "story-title",
+    "post-head",
+    "navigation",
+    "cookie",
+    "consent",
+    "popup",
+    "modal",
+    "advert",
+    "sponsor",
+    "logo",
+    "masthead",
+    "notice",
+    "overlay",
+    "follow",
+    "sponsored",
+    "note-b",
+    "dog_two",
+    # Additional markers from defuddle
+    "read-more",
+    "article-card",
+    "post-meta",
+    "article-actions",
+    "post-actions",
+    "byline",
+    "author-block",
+    "meta-info",
+    "sidebar",
+    "aside",
+    "related-posts",
+    "related-content",
+    "promo-box",
+    "sponsor-box",
+    "ad-box",
+    "promo-block",
+)
+
+# Structural chrome markers for top-of-article wrappers that repeat metadata.
+HEADER_MARKERS = (
+    "byline",
+    "author",
+    "date",
+    "time",
+    "heading--sub",
+    "title-sub",
+    "teaser",
+    "kicker",
+    "standfirst",
+    "deck",
+)
+
+# Phrases commonly used to label related-link blocks.
+RELATED_PHRASES = (
+    "related categories",
+    "related topics",
+    "related articles",
+    "related posts",
+    "recommended",
+    "latest news",
+    "you may also like",
+    "more from",
+    "most read",
+)
+
+# Common boilerplate phrases that appear in documentation and feedback shells.
+BOILERPLATE_PHRASES = (
+    "thanks for letting us know this page needs work",
+    "we're sorry we let you down",
+    "help improve",
+    "learn how to contribute",
+    "view this page on github",
+    "report a problem with this content",
+    "if you've got a moment",
+    "how we can make the documentation better",
+    "this post is for paid members only",
+    "no results found",
+    "developing story",
+    "stay tuned for updates",
+)
+
+# Placeholder phrases used by portal-style shells that require JavaScript
+# before the real body is rendered.
+JS_SHELL_PHRASES = (
+    "this app needs javascript to run",
+    "please enable javascript in your browser and try again",
+    "javascript is required",
+)

domdown/_core/__init__.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from .context import PipelineContext
+from .metadata import HtmlMetadata
+from .options import DomdownOptions
+from .result import HtmlToMarkdownResult
+
+__all__ = [
+    "DomdownOptions",
+    "HtmlMetadata",
+    "HtmlToMarkdownResult",
+    "PipelineContext",
+]

domdown/_core/context.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .metadata import HtmlMetadata
+from .options import DomdownOptions
+
+
+@dataclass(slots=True)
+class PipelineContext:
+    """Mutable state passed between pipeline stages."""
+
+    html: str
+    options: DomdownOptions
+    document: Any | None = None
+    cleaned_html: str | None = None
+    markdown: str = ""
+    metadata: HtmlMetadata | None = None
+    frontmatter: str | None = None
+    rendered_document: str | None = None
+    warnings: list[str] = field(default_factory=list)

domdown/_core/metadata.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class HtmlMetadata:
+    """Normalized article metadata extracted from the source HTML."""
+
+    title: str | None = None
+    site_name: str | None = None
+    source: str | None = None
+    author: tuple[str, ...] = ()
+    published: str | None = None
+    created: str | None = None
+    description: str | None = None
+    tags: tuple[str, ...] = ()
+    language: str | None = None
+    canonical_url: str | None = None
+    image: str | None = None

domdown/_core/options.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class DomdownOptions:
+    """Configuration for HTML parsing, cleanup, and output shaping."""
+
+    base_url: str | None = None
+    created: str | None = None
+    extract_metadata: bool = True
+    emit_frontmatter: bool = True
+    prefer_article_body: bool = True
+    author_priority: str = "visible"
+    frontmatter_tags: tuple[str, ...] = ()
+    preserve_images: bool = True
+    preserve_tables: bool = True
+    preserve_code_blocks: bool = True
+    strip_hidden: bool = True
+    remove_selectors: tuple[str, ...] = ()
+    keep_selectors: tuple[str, ...] = ()
+    unwrap_selectors: tuple[str, ...] = ()

domdown/_core/result.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .metadata import HtmlMetadata
+
+
+@dataclass(frozen=True, slots=True)
+class HtmlToMarkdownResult:
+    """Final output produced by the pipeline."""
+
+    markdown: str
+    cleaned_html: str | None = None
+    metadata: HtmlMetadata | None = None
+    frontmatter: str | None = None
+    document: str | None = None
+    warnings: tuple[str, ...] = ()

domdown/_document/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from .clean import clean_root
+from .parse import parse_html
+from .select import choose_root
+
+__all__ = ["choose_root", "clean_root", "parse_html"]