htmlquill 0.1.0__py3-none-any.whl

htmlquill/__init__.py

@@ -0,0 +1,11 @@
+"""htmlquill — HTML to Markdown converter."""
+
+from __future__ import annotations
+
+try:
+    from htmlquill._version import __version__, version
+except ImportError:
+    __version__ = version = "0.0.0+unknown"
+from htmlquill.core import html_to_markdown, url_to_markdown
+
+__all__ = ["__version__", "version", "html_to_markdown", "url_to_markdown"]

htmlquill/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

htmlquill/analyse.py

@@ -0,0 +1,75 @@
+"""Markdown analysis helpers."""
+
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import asdict, dataclass
+
+
+@dataclass(frozen=True)
+class MarkdownStats:
+    lines: int
+    nonblank_lines: int
+    chars: int
+    words: int
+    headings: int
+    headings_by_level: dict[str, int]
+    code_blocks: int
+    inline_code_spans: int
+    images: int
+    links: int
+    tables: int
+    blockquotes: int
+    list_items: int
+    frontmatter: bool
+    estimated_reading_minutes: int
+
+    def to_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+_HEADING_RE = re.compile(r"^(#{1,6})\s+\S", re.MULTILINE)
+_FENCED_CODE_RE = re.compile(r"(^|\n)```.*?\n.*?(\n```|$)", re.DOTALL)
+_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]+\)")
+_LINK_RE = re.compile(r"(?<!!)\[[^\]]+\]\([^)]+\)")
+_INLINE_CODE_RE = re.compile(r"(?<!`)`[^`\n]+`(?!`)")
+_LIST_ITEM_RE = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s+\S", re.MULTILINE)
+_BLOCKQUOTE_RE = re.compile(r"^\s*>\s?", re.MULTILINE)
+_TABLE_SEPARATOR_RE = re.compile(r"^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$")
+
+
+def _strip_fenced_code(markdown: str) -> str:
+    return _FENCED_CODE_RE.sub("\n", markdown)
+
+
+def count_markdown_stats(markdown: str) -> MarkdownStats:
+    lines = markdown.splitlines()
+    nonblank = [line for line in lines if line.strip()]
+
+    headings_by_level = {f"h{i}": 0 for i in range(1, 7)}
+    for match in _HEADING_RE.finditer(markdown):
+        headings_by_level[f"h{len(match.group(1))}"] += 1
+
+    without_blocks = _strip_fenced_code(markdown)
+    words = len(re.findall(r"\b[\w'-]+\b", without_blocks))
+
+    tables = sum(1 for i in range(1, len(lines)) if _TABLE_SEPARATOR_RE.match(lines[i]))
+
+    return MarkdownStats(
+        lines=len(lines),
+        nonblank_lines=len(nonblank),
+        chars=len(markdown),
+        words=words,
+        headings=sum(headings_by_level.values()),
+        headings_by_level=headings_by_level,
+        code_blocks=sum(1 for _ in _FENCED_CODE_RE.finditer(markdown)),
+        inline_code_spans=len(_INLINE_CODE_RE.findall(without_blocks)),
+        images=len(_IMAGE_RE.findall(markdown)),
+        links=len(_LINK_RE.findall(markdown)),
+        tables=tables,
+        blockquotes=len(_BLOCKQUOTE_RE.findall(markdown)),
+        list_items=len(_LIST_ITEM_RE.findall(markdown)),
+        frontmatter=markdown.startswith("---\n"),
+        estimated_reading_minutes=max(1, math.ceil(words / 220)) if words else 0,
+    )

htmlquill/auth.py

@@ -0,0 +1,268 @@
+"""Authentication/session state helpers for htmlquill."""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+import warnings
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from htmlquill.paths import default_auth_path, env_flag
+
+
+@dataclass(frozen=True)
+class CookieConfig:
+    name: str
+    value: str
+    domain: str | None = None
+    path: str | None = None
+    secure: bool = False
+    http_only: bool = False
+
+
+@dataclass(frozen=True)
+class AuthProfile:
+    name: str
+    kind: str
+    cookies: tuple[CookieConfig, ...] = ()
+    playwright_storage_state: Path | None = None
+    chromium_user_data_dir: Path | None = None
+
+
+@dataclass(frozen=True)
+class AuthStore:
+    version: int = 1
+    profiles: dict[str, AuthProfile] = field(default_factory=dict)
+    source_path: Path | None = None
+
+
+@dataclass(frozen=True)
+class ResolvedAuth:
+    profile_name: str | None = None
+    cookies: list[dict[str, object]] | None = None
+    playwright_storage_state: str | None = None
+    chromium_user_data_dir: str | None = None
+
+
+# Use shared helpers from htmlquill.paths
+_env_flag = env_flag
+default_auth_path = default_auth_path
+
+
+def auth_enabled_for_run(no_auth: bool) -> bool:
+    """Return whether auth loading is enabled for this invocation."""
+
+    if no_auth:
+        return False
+    return not _env_flag("HTMLQUILL_NO_AUTH")
+
+
+def resolve_auth_path(
+    *,
+    explicit_auth_path: str | Path | None,
+    config_auth_path: str | None,
+    config_dir: Path | None,
+) -> Path:
+    """Resolve auth path from explicit path, env, config, or default."""
+
+    if explicit_auth_path is not None:
+        return Path(explicit_auth_path).expanduser()
+
+    env_path = os.environ.get("HTMLQUILL_AUTH")
+    if env_path:
+        return Path(env_path).expanduser()
+
+    if config_auth_path:
+        configured = Path(config_auth_path).expanduser()
+        if configured.is_absolute() or config_dir is None:
+            return configured
+        return (config_dir / configured).resolve()
+
+    return default_auth_path(config_dir)
+
+
+def _warn_or_fail_on_permissions(path: Path, *, strict_permissions: bool) -> None:
+    if os.name == "nt":
+        return
+    mode = stat.S_IMODE(path.stat().st_mode)
+    if mode & 0o077:
+        msg = (
+            f"auth file {path} is group/world accessible (mode {oct(mode)}); "
+            "recommended mode is 0o600"
+        )
+        if strict_permissions:
+            raise PermissionError(msg)
+        warnings.warn(msg, stacklevel=2)
+
+
+def _parse_cookie(index: int, raw: Any) -> CookieConfig:
+    if not isinstance(raw, dict):
+        raise ValueError(f"cookies[{index}] must be an object")
+
+    name = raw.get("name")
+    value = raw.get("value")
+    if not isinstance(name, str) or not isinstance(value, str):
+        raise ValueError(f"cookies[{index}] requires string 'name' and 'value'")
+
+    domain = raw.get("domain")
+    path = raw.get("path")
+    secure = bool(raw.get("secure", False))
+    http_only = bool(raw.get("httpOnly", raw.get("http_only", False)))
+
+    if domain is not None and not isinstance(domain, str):
+        raise ValueError(f"cookies[{index}].domain must be a string")
+    if path is not None and not isinstance(path, str):
+        raise ValueError(f"cookies[{index}].path must be a string")
+
+    return CookieConfig(
+        name=name,
+        value=value,
+        domain=domain,
+        path=path,
+        secure=secure,
+        http_only=http_only,
+    )
+
+
+def _expand_profile_path(raw_path: Any, *, base_dir: Path) -> Path | None:
+    if raw_path in (None, ""):
+        return None
+    if not isinstance(raw_path, str):
+        raise ValueError("profile path values must be strings")
+    parsed = Path(raw_path).expanduser()
+    if parsed.is_absolute():
+        return parsed
+    return (base_dir / parsed).resolve()
+
+
+def _parse_profile(name: str, raw: Any, *, base_dir: Path) -> AuthProfile:
+    if not isinstance(raw, dict):
+        raise ValueError(f"profiles.{name} must be an object")
+
+    kind = raw.get("kind", "cookies")
+    if not isinstance(kind, str):
+        raise ValueError(f"profiles.{name}.kind must be a string")
+
+    cookies_data = raw.get("cookies", [])
+    if not isinstance(cookies_data, list):
+        raise ValueError(f"profiles.{name}.cookies must be an array")
+    cookies = tuple(_parse_cookie(i, c) for i, c in enumerate(cookies_data))
+
+    playwright_storage_state = _expand_profile_path(
+        raw.get("playwright_storage_state"), base_dir=base_dir
+    )
+    chromium_user_data_dir = _expand_profile_path(
+        raw.get("chromium_user_data_dir"), base_dir=base_dir
+    )
+
+    return AuthProfile(
+        name=name,
+        kind=kind,
+        cookies=cookies,
+        playwright_storage_state=playwright_storage_state,
+        chromium_user_data_dir=chromium_user_data_dir,
+    )
+
+
+def load_auth(path: Path, *, strict_permissions: bool = True) -> AuthStore:
+    """Load auth JSON from *path*."""
+
+    expanded = path.expanduser()
+    _warn_or_fail_on_permissions(expanded, strict_permissions=strict_permissions)
+
+    try:
+        payload = json.loads(expanded.read_text(encoding="utf-8"))
+    except OSError as exc:
+        raise ValueError(f"failed to read auth file {expanded}: {exc}") from exc
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"failed to parse auth file {expanded}: {exc}") from exc
+
+    if not isinstance(payload, dict):
+        raise ValueError("auth JSON root must be an object")
+
+    version = payload.get("version", 1)
+    if not isinstance(version, int):
+        raise ValueError("auth version must be an integer")
+
+    profiles_raw = payload.get("profiles", {})
+    if not isinstance(profiles_raw, dict):
+        raise ValueError("auth profiles must be an object")
+
+    base_dir = expanded.parent
+    profiles = {
+        profile_name: _parse_profile(profile_name, raw, base_dir=base_dir)
+        for profile_name, raw in profiles_raw.items()
+    }
+
+    return AuthStore(version=version, profiles=profiles, source_path=expanded)
+
+
+def resolve_auth_profile(auth_store: AuthStore, name: str | None) -> AuthProfile | None:
+    """Resolve one auth profile by *name*."""
+
+    if name is None:
+        return None
+    if name not in auth_store.profiles:
+        available = ", ".join(sorted(auth_store.profiles)) or "(none)"
+        raise ValueError(f"auth profile {name!r} not found; available: {available}")
+    return auth_store.profiles[name]
+
+
+def resolve_auth(
+    auth_store: AuthStore | None,
+    *,
+    profile_name: str | None,
+) -> ResolvedAuth:
+    """Resolve concrete fetch auth values for an optional auth profile."""
+
+    if auth_store is None:
+        return ResolvedAuth(profile_name=None)
+
+    profile = resolve_auth_profile(auth_store, profile_name)
+    if profile is None:
+        return ResolvedAuth(profile_name=None)
+
+    cookie_payload: list[dict[str, object]] | None = None
+    if profile.cookies:
+        cookie_payload = [
+            {
+                "name": cookie.name,
+                "value": cookie.value,
+                "domain": cookie.domain,
+                "path": cookie.path,
+                "secure": cookie.secure,
+                "httpOnly": cookie.http_only,
+            }
+            for cookie in profile.cookies
+        ]
+
+    return ResolvedAuth(
+        profile_name=profile.name,
+        cookies=cookie_payload,
+        playwright_storage_state=(
+            str(profile.playwright_storage_state)
+            if profile.playwright_storage_state is not None
+            else None
+        ),
+        chromium_user_data_dir=(
+            str(profile.chromium_user_data_dir)
+            if profile.chromium_user_data_dir is not None
+            else None
+        ),
+    )
+
+
+def redacted_auth_dict(resolved: ResolvedAuth) -> dict[str, object]:
+    """Return a redacted dict suitable for ``--print-config`` output."""
+
+    cookies_count = len(resolved.cookies or [])
+    return {
+        "profile": resolved.profile_name,
+        "cookies": "<redacted>" if cookies_count else None,
+        "cookies_count": cookies_count,
+        "playwright_storage_state": resolved.playwright_storage_state,
+        "chromium_user_data_dir": resolved.chromium_user_data_dir,
+    }

htmlquill/challenge.py

@@ -0,0 +1,49 @@
+"""Challenge/interstitial page detection helpers."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+DEFAULT_CHALLENGE_MARKERS: tuple[str, ...] = (
+    "Performing security verification",
+    "Please wait for verification",
+    "security service to protect against malicious bots",
+    "verifies you are not a bot",
+    "Checking if the site connection is secure",
+    "Just a moment",
+    "js_challenge",
+    "You've been blocked by network security",
+    "blocked by network security",
+    "If you think you've been blocked by mistake, file a ticket",
+    "Please try to login with your Reddit account",
+)
+
+
+class ChallengePageError(RuntimeError):
+    """Raised when fetched HTML appears to be a challenge page."""
+
+
+def is_challenge_page(
+    html: str,
+    url: str | None = None,
+    markers: Sequence[str] = DEFAULT_CHALLENGE_MARKERS,
+) -> bool:
+    """Return ``True`` if *html* contains known challenge page markers."""
+
+    del url  # reserved for future URL-specific marker behavior
+    lower = html.lower()
+    return any(marker.lower() in lower for marker in markers)
+
+
+def assert_not_challenge_page(
+    html: str,
+    *,
+    url: str | None = None,
+    markers: Sequence[str] = DEFAULT_CHALLENGE_MARKERS,
+) -> None:
+    """Raise :class:`ChallengePageError` when *html* looks like an interstitial."""
+
+    if is_challenge_page(html, url=url, markers=markers):
+        raise ChallengePageError(
+            "received a security verification page instead of article HTML"
+        )

htmlquill/clean.py

@@ -0,0 +1,136 @@
+"""HTML parsing and cleanup for htmlquill."""
+
+from __future__ import annotations
+
+from bs4 import BeautifulSoup, Tag
+
+DROP_SELECTORS = [
+    "script",
+    "style",
+    "noscript",
+    "template",
+    "nav",
+    "footer",
+    "[hidden]",
+    '[aria-hidden="true"]',
+]
+
+NOISE_TEXTS = {
+    "Press enter or click to view image in full size",
+}
+
+ACTION_HREF_PARTS = (
+    "/m/signin",
+    "operation=register",
+)
+
+
+def _drop_action_controls(root: BeautifulSoup | Tag) -> None:
+    """Remove action control elements from the DOM."""
+    # Drop anchors that point to action/sign-in flows
+    for a in list(root.find_all("a")):
+        href = str(a.get("href", ""))
+        if any(part in href for part in ACTION_HREF_PARTS):
+            a.decompose()
+            continue
+        # Drop empty anchors without images that have role=button
+        if a.get("role") == "button":
+            has_img = a.find("img") is not None
+            if not has_img and not a.get_text(strip=True):
+                a.decompose()
+
+
+def _drop_noise_text_nodes(root: BeautifulSoup | Tag) -> None:
+    """Remove known noise text from the DOM."""
+    for text_node in list(root.find_all(string=True)):
+        if str(text_node).strip() in NOISE_TEXTS:
+            # Remove the parent element if it only contains noise
+            parent = text_node.parent
+            if parent is not None and isinstance(parent, Tag):
+                parent_text = parent.get_text(strip=True)
+                if parent_text in NOISE_TEXTS:
+                    parent.decompose()
+                else:
+                    text_node.extract()
+            else:
+                text_node.extract()
+
+
+def _drop_non_figure_svgs(root: BeautifulSoup | Tag) -> None:
+    """Drop SVG outside figure context; keep figure-embedded scientific SVG."""
+    for svg in list(root.find_all("svg")):
+        if svg.find_parent("figure") is None:
+            svg.decompose()
+
+
+def _content_score(node: Tag) -> int:
+    """Score a candidate content root by content density."""
+    score = 0
+    score += len(node.find_all("p")) * 3
+    score += len(node.find_all(["h1", "h2", "h3", "h4", "h5", "h6"])) * 2
+    text_len = len(node.get_text(strip=True))
+    score += text_len // 100
+    # Penalize nodes with many action/sign-in links
+    action_links = 0
+    for a in node.find_all("a"):
+        href = str(a.get("href", ""))
+        if "/m/signin" in href or "operation=register" in href:
+            action_links += 1
+    score -= action_links * 5
+    return score
+
+
+def parse_and_clean(html: str) -> BeautifulSoup | Tag:
+    """Parse HTML and remove unwanted elements.
+
+    Strips scripts, styles, navigation, footers, hidden elements, and other
+    non-content nodes. Selects the best content container using a
+    content-scoring heuristic that prefers ``<article>`` over ``<main>``,
+    then ``<body>``, or the whole document.
+
+    Parameters
+    ----------
+    html
+        Raw HTML string to parse.
+
+    Returns
+    -------
+    BeautifulSoup | Tag
+        The cleaned root element ready for rendering.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+
+    for selector in DROP_SELECTORS:
+        for node in soup.select(selector):
+            node.decompose()
+
+    _drop_non_figure_svgs(soup)
+
+    for node in list(soup.find_all(style=True)):
+        attrs = node.attrs
+        if attrs is None:
+            continue
+        style = str(attrs.get("style", "")).replace(" ", "").lower()
+        if "display:none" in style or "visibility:hidden" in style:
+            node.decompose()
+
+    # Drop action controls and noise text
+    _drop_action_controls(soup)
+    _drop_noise_text_nodes(soup)
+
+    # Gather content candidates and score them
+    candidates: list[Tag] = []
+    for tag in soup.find_all("article"):
+        candidates.append(tag)
+    for tag in soup.find_all("main"):
+        candidates.append(tag)
+    # Also consider [role=main]
+    for tag in soup.select("[role=main]"):
+        if tag not in candidates:
+            candidates.append(tag)
+
+    if candidates:
+        best = max(candidates, key=_content_score)
+        return best
+
+    return soup.body or soup