html-to-markdown 2.6.3__cp310-abi3-macosx_11_0_arm64.whl → 2.14.0__cp310-abi3-macosx_11_0_arm64.whl

html_to_markdown/__init__.py

@@ -15,12 +15,17 @@ V1 API (backward compatibility):
     markdown = convert_to_markdown(html, heading_style="atx")
 """
 
+import contextlib
+
 from html_to_markdown.api import (
     InlineImage,
     InlineImageConfig,
     InlineImageWarning,
+    OptionsHandle,
     convert,
+    convert_with_handle,
     convert_with_inline_images,
+    create_options_handle,
 )
 from html_to_markdown.exceptions import (
     ConflictingOptionsError,
@@ -32,6 +37,10 @@ from html_to_markdown.exceptions import (
 from html_to_markdown.options import ConversionOptions, PreprocessingOptions
 from html_to_markdown.v1_compat import convert_to_markdown, markdownify
 
+# Optional metadata support
+with contextlib.suppress(ImportError):
+    from html_to_markdown.api import MetadataConfig, convert_with_metadata
+
 __all__ = [
     "ConflictingOptionsError",
     "ConversionOptions",
@@ -41,12 +50,17 @@ __all__ = [
     "InlineImageConfig",
     "InlineImageWarning",
     "InvalidParserError",
+    "MetadataConfig",
     "MissingDependencyError",
+    "OptionsHandle",
     "PreprocessingOptions",
     "convert",
     "convert_to_markdown",
+    "convert_with_handle",
     "convert_with_inline_images",
+    "convert_with_metadata",
+    "create_options_handle",
     "markdownify",
 ]
 
-__version__ = "2.5.7"
+__version__ = "2.14.0"

html_to_markdown/_html_to_markdown.pyi

@@ -0,0 +1,196 @@
+from typing import Literal, TypedDict
+
+class PreprocessingOptions:
+    enabled: bool
+    preset: Literal["minimal", "standard", "aggressive"]
+    remove_navigation: bool
+    remove_forms: bool
+
+    def __init__(
+        self,
+        *,
+        enabled: bool = False,
+        preset: Literal["minimal", "standard", "aggressive"] = "standard",
+        remove_navigation: bool = True,
+        remove_forms: bool = True,
+    ) -> None: ...
+
+class ConversionOptions:
+    heading_style: Literal["underlined", "atx", "atx_closed"]
+    list_indent_type: Literal["spaces", "tabs"]
+    list_indent_width: int
+    bullets: str
+    strong_em_symbol: str
+    escape_asterisks: bool
+    escape_underscores: bool
+    escape_misc: bool
+    escape_ascii: bool
+    code_language: str
+    autolinks: bool
+    default_title: bool
+    br_in_tables: bool
+    hocr_spatial_tables: bool
+    highlight_style: Literal["double-equal", "html", "bold", "none"]
+    extract_metadata: bool
+    whitespace_mode: Literal["normalized", "strict"]
+    strip_newlines: bool
+    wrap: bool
+    wrap_width: int
+    convert_as_inline: bool
+    sub_symbol: str
+    sup_symbol: str
+    newline_style: Literal["spaces", "backslash"]
+    code_block_style: Literal["indented", "backticks", "tildes"]
+    keep_inline_images_in: list[str]
+    preprocessing: PreprocessingOptions
+    encoding: str
+    debug: bool
+    strip_tags: list[str]
+    preserve_tags: list[str]
+
+    def __init__(
+        self,
+        *,
+        heading_style: Literal["underlined", "atx", "atx_closed"] = "underlined",
+        list_indent_type: Literal["spaces", "tabs"] = "spaces",
+        list_indent_width: int = 4,
+        bullets: str = "*+-",
+        strong_em_symbol: str = "*",
+        escape_asterisks: bool = False,
+        escape_underscores: bool = False,
+        escape_misc: bool = False,
+        escape_ascii: bool = False,
+        code_language: str = "",
+        autolinks: bool = True,
+        default_title: bool = False,
+        br_in_tables: bool = False,
+        hocr_spatial_tables: bool = True,
+        highlight_style: Literal["double-equal", "html", "bold", "none"] = "double-equal",
+        extract_metadata: bool = True,
+        whitespace_mode: Literal["normalized", "strict"] = "normalized",
+        strip_newlines: bool = False,
+        wrap: bool = False,
+        wrap_width: int = 80,
+        convert_as_inline: bool = False,
+        sub_symbol: str = "",
+        sup_symbol: str = "",
+        newline_style: Literal["spaces", "backslash"] = "spaces",
+        code_block_style: Literal["indented", "backticks", "tildes"] = "indented",
+        keep_inline_images_in: list[str] = [],
+        preprocessing: PreprocessingOptions | None = None,
+        encoding: str = "utf-8",
+        debug: bool = False,
+        strip_tags: list[str] = [],
+        preserve_tags: list[str] = [],
+    ) -> None: ...
+
+class InlineImageConfig:
+    max_decoded_size_bytes: int
+    filename_prefix: str | None
+    capture_svg: bool
+    infer_dimensions: bool
+
+    def __init__(
+        self,
+        max_decoded_size_bytes: int = ...,
+        filename_prefix: str | None = None,
+        capture_svg: bool = True,
+        infer_dimensions: bool = False,
+    ) -> None: ...
+
+class ConversionOptionsHandle:
+    def __init__(self, options: ConversionOptions | None = None) -> None: ...
+
+class InlineImage(TypedDict):
+    data: bytes
+    format: str
+    filename: str | None
+    description: str | None
+    dimensions: tuple[int, int] | None
+    source: Literal["img_data_uri", "svg_element"]
+    attributes: dict[str, str]
+
+class InlineImageWarning(TypedDict):
+    index: int
+    message: str
+
+class MetadataConfig:
+    extract_document: bool
+    extract_headers: bool
+    extract_links: bool
+    extract_images: bool
+    extract_structured_data: bool
+    max_structured_data_size: int
+
+    def __init__(
+        self,
+        *,
+        extract_document: bool = True,
+        extract_headers: bool = True,
+        extract_links: bool = True,
+        extract_images: bool = True,
+        extract_structured_data: bool = True,
+        max_structured_data_size: int = 1_000_000,
+    ) -> None: ...
+
+class DocumentMetadata(TypedDict):
+    title: str | None
+    description: str | None
+    keywords: list[str]
+    author: str | None
+    canonical_url: str | None
+    base_href: str | None
+    language: str | None
+    text_direction: str | None  # "ltr" | "rtl" | "auto" | None
+    open_graph: dict[str, str]
+    twitter_card: dict[str, str]
+    meta_tags: dict[str, str]
+
+class HeaderMetadata(TypedDict):
+    level: int  # 1-6
+    text: str
+    id: str | None
+    depth: int
+    html_offset: int
+
+class LinkMetadata(TypedDict):
+    href: str
+    text: str
+    title: str | None
+    link_type: str  # "anchor" | "internal" | "external" | "email" | "phone" | "other"
+    rel: list[str]
+    attributes: dict[str, str]
+
+class ImageMetadata(TypedDict):
+    src: str
+    alt: str | None
+    title: str | None
+    dimensions: tuple[int, int] | None  # (width, height)
+    image_type: str  # "data_uri" | "inline_svg" | "external" | "relative"
+    attributes: dict[str, str]
+
+class StructuredData(TypedDict):
+    data_type: str  # "json_ld" | "microdata" | "rdfa"
+    raw_json: str
+    schema_type: str | None
+
+class ExtendedMetadata(TypedDict):
+    document: DocumentMetadata
+    headers: list[HeaderMetadata]
+    links: list[LinkMetadata]
+    images: list[ImageMetadata]
+    structured_data: list[StructuredData]
+
+def convert(html: str, options: ConversionOptions | None = None) -> str: ...
+def convert_with_inline_images(
+    html: str,
+    options: ConversionOptions | None = None,
+    image_config: InlineImageConfig | None = None,
+) -> tuple[str, list[InlineImage], list[InlineImageWarning]]: ...
+def convert_with_metadata(
+    html: str,
+    options: ConversionOptions | None = None,
+    metadata_config: MetadataConfig | None = None,
+) -> tuple[str, ExtendedMetadata]: ...
+def create_options_handle(options: ConversionOptions | None = None) -> ConversionOptionsHandle: ...
+def convert_with_options_handle(html: str, handle: ConversionOptionsHandle) -> str: ...

html_to_markdown/api.py

@@ -1,20 +1,27 @@
-"""New v2 functional API for HTML to Markdown conversion.
-
-This module provides the new functional API with dataclass-based options,
-using the Rust backend for conversion.
-"""
+"""High-level Python API backed by the Rust core."""
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Literal, TypedDict, cast
+from typing import TYPE_CHECKING, Literal, TypedDict
 
-import html_to_markdown._html_to_markdown as _rust  # type: ignore[import-not-found]
+import html_to_markdown._html_to_markdown as _rust
+from html_to_markdown._html_to_markdown import (
+    ConversionOptionsHandle as OptionsHandle,
+)
+from html_to_markdown._html_to_markdown import InlineImageConfig
 from html_to_markdown.options import ConversionOptions, PreprocessingOptions
 
-if TYPE_CHECKING:
-    from html_to_markdown._html_to_markdown import InlineImageConfig
-else:
-    InlineImageConfig = _rust.InlineImageConfig  # type: ignore[misc, assignment]
+_HAS_METADATA = False
+try:
+    from html_to_markdown._html_to_markdown import ExtendedMetadata, MetadataConfig
+
+    _HAS_METADATA = True
+except ImportError:
+    MetadataConfig = None  # type: ignore[misc,assignment]
+    if TYPE_CHECKING:
+        from html_to_markdown._html_to_markdown import ExtendedMetadata  # pragma: no cover
+    else:
+        ExtendedMetadata = dict[str, object]  # type: ignore[assignment]
 
 
 class InlineImage(TypedDict):
@@ -37,7 +44,6 @@ class InlineImageWarning(TypedDict):
 
 
 def _to_rust_preprocessing(options: PreprocessingOptions) -> _rust.PreprocessingOptions:
-    """Convert high-level preprocessing options to the Rust bindings."""
     return _rust.PreprocessingOptions(
         enabled=options.enabled,
         preset=options.preset,
@@ -50,7 +56,6 @@ def _to_rust_options(
     options: ConversionOptions,
     preprocessing: PreprocessingOptions,
 ) -> _rust.ConversionOptions:
-    """Convert high-level conversion options to the Rust bindings."""
     return _rust.ConversionOptions(
         heading_style=options.heading_style,
         list_indent_type=options.list_indent_type,
@@ -91,23 +96,17 @@ def convert(
     options: ConversionOptions | None = None,
     preprocessing: PreprocessingOptions | None = None,
 ) -> str:
-    """Convert HTML to Markdown using the Rust backend.
+    """Convert HTML to Markdown using the Rust backend."""
+    if options is None and preprocessing is None:
+        return _rust.convert(html, None)
 
-    Args:
-        html: HTML string to convert.
-        options: Conversion configuration options (defaults to ConversionOptions()).
-        preprocessing: HTML preprocessing options (defaults to PreprocessingOptions()).
-
-    Returns:
-        Converted Markdown string.
-    """
     if options is None:
         options = ConversionOptions()
     if preprocessing is None:
         preprocessing = PreprocessingOptions()
 
     rust_options = _to_rust_options(options, preprocessing)
-    return cast("str", _rust.convert(html, rust_options))
+    return _rust.convert(html, rust_options)
 
 
 def convert_with_inline_images(
@@ -116,10 +115,7 @@ def convert_with_inline_images(
     preprocessing: PreprocessingOptions | None = None,
     image_config: InlineImageConfig | None = None,
 ) -> tuple[str, list[InlineImage], list[InlineImageWarning]]:
-    """Convert HTML and extract inline images.
-
-    Returns Markdown along with extracted inline images and any warnings.
-    """
+    """Convert HTML and extract inline images."""
     if options is None:
         options = ConversionOptions()
     if preprocessing is None:
@@ -128,17 +124,73 @@ def convert_with_inline_images(
         image_config = InlineImageConfig()
 
     rust_options = _to_rust_options(options, preprocessing)
-    markdown, images, warnings = cast(
-        "tuple[str, list[InlineImage], list[InlineImageWarning]]",
-        _rust.convert_with_inline_images(html, rust_options, image_config),
-    )
+    markdown, images, warnings = _rust.convert_with_inline_images(html, rust_options, image_config)
     return markdown, list(images), list(warnings)
 
 
+def create_options_handle(
+    options: ConversionOptions | None = None,
+    preprocessing: PreprocessingOptions | None = None,
+) -> OptionsHandle:
+    """Create a reusable ConversionOptions handle backed by Rust."""
+    if options is None:
+        options = ConversionOptions()
+    if preprocessing is None:
+        preprocessing = PreprocessingOptions()
+    rust_options = _to_rust_options(options, preprocessing)
+    return _rust.create_options_handle(rust_options)
+
+
+def convert_with_handle(html: str, handle: OptionsHandle) -> str:
+    """Convert HTML using a pre-parsed ConversionOptions handle."""
+    return _rust.convert_with_options_handle(html, handle)
+
+
+if _HAS_METADATA:
+
+    def convert_with_metadata(
+        html: str,
+        options: ConversionOptions | None = None,
+        preprocessing: PreprocessingOptions | None = None,
+        metadata_config: MetadataConfig | None = None,
+    ) -> tuple[str, ExtendedMetadata]:
+        """Convert HTML and extract comprehensive metadata.
+
+        Args:
+            html: HTML string to convert
+            options: Optional conversion configuration
+            preprocessing: Optional preprocessing configuration
+            metadata_config: Optional metadata extraction configuration
+
+        Returns:
+            Tuple of (markdown, metadata_dict) where metadata_dict contains:
+            - document: Document-level metadata (title, description, lang, etc.)
+            - headers: List of header elements with hierarchy
+            - links: List of extracted hyperlinks with classification
+            - images: List of extracted images with metadata
+            - structured_data: List of JSON-LD, Microdata, or RDFa blocks
+        """
+        if options is None:
+            options = ConversionOptions()
+        if preprocessing is None:
+            preprocessing = PreprocessingOptions()
+        if metadata_config is None:
+            metadata_config = MetadataConfig()
+
+        rust_options = _to_rust_options(options, preprocessing)
+        markdown, metadata = _rust.convert_with_metadata(html, rust_options, metadata_config)
+        return markdown, metadata
+
+
 __all__ = [
     "InlineImage",
     "InlineImageConfig",
     "InlineImageWarning",
+    "MetadataConfig",
+    "OptionsHandle",
     "convert",
+    "convert_with_handle",
     "convert_with_inline_images",
+    "convert_with_metadata",
+    "create_options_handle",
 ]

html_to_markdown/v1_compat.py

@@ -11,6 +11,15 @@ import warnings
 from html_to_markdown import ConversionOptions, PreprocessingOptions
 from html_to_markdown import convert as convert_v2
 
+DEPRECATION_MESSAGE = (
+    "The v1 compatibility layer is deprecated and will be removed in v3.0. "
+    "Use html_to_markdown.convert() with ConversionOptions instead."
+)
+
+
+def _warn_deprecated(api_name: str, *, stacklevel: int = 2) -> None:
+    warnings.warn(f"{api_name} is deprecated. {DEPRECATION_MESSAGE}", DeprecationWarning, stacklevel=stacklevel)
+
 
 def convert_to_markdown(
     html: str,
@@ -104,12 +113,7 @@ def convert_to_markdown(
         Use :func:`html_to_markdown.convert` with :class:`ConversionOptions` instead.
         The v1 API is provided for backward compatibility only.
     """
-    warnings.warn(
-        "convert_to_markdown() is deprecated and will be removed in v3.0. "
-        "Use html_to_markdown.convert() with ConversionOptions instead.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
+    _warn_deprecated("convert_to_markdown()", stacklevel=2)
 
     if code_language_callback is not None:
         raise NotImplementedError(
@@ -181,12 +185,7 @@ def markdownify(*args: object, **kwargs: object) -> str:
     .. deprecated:: 2.0
         Use html_to_markdown.convert() instead.
     """
-    warnings.warn(
-        "markdownify() is deprecated and will be removed in v3.0. "
-        "Use html_to_markdown.convert() with ConversionOptions instead.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
+    _warn_deprecated("markdownify()", stacklevel=2)
     return convert_to_markdown(*args, **kwargs)  # type: ignore[arg-type]