markdown-this 0.1.0__tar.gz

markdown_this-0.1.0/PKG-INFO

@@ -0,0 +1,58 @@
+Metadata-Version: 2.3
+Name: markdown-this
+Version: 0.1.0
+Summary: Extract web pages and supported special URLs as Markdown
+Keywords: html,markdown,readability,github,arxiv
+Author: Martín Gaitán
+Author-email: Martín Gaitán <gaitan@gmail.com>
+License: MIT
+Requires-Dist: beautifulsoup4>=4.14.3
+Requires-Dist: markdownify>=1.2.2
+Requires-Dist: readability-lxml>=0.8.4.1
+Requires-Dist: requests>=2.32.5
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/mgaitan/lobstersgram/tree/main/packages/markdown-this
+Project-URL: Repository, https://github.com/mgaitan/lobstersgram
+Project-URL: Issues, https://github.com/mgaitan/lobstersgram/issues
+Description-Content-Type: text/markdown
+
+# markdown-this
+
+[![PyPI](https://img.shields.io/pypi/v/markdown-this)](https://pypi.org/project/markdown-this/)
+
+Extract the readable content of a URL and convert it to Markdown. The package
+also handles GitHub repositories and Markdown files through the GitHub API,
+and arXiv abstract pages through their HTML representation.
+
+## Installation
+
+```bash
+uv add markdown-this
+```
+
+## Usage
+
+```python
+from markdown_this import extract_main_content
+
+title, markdown, fallback_text, intro = extract_main_content(
+    "https://example.com/article"
+)
+```
+
+`extract_main_content` returns the title, extracted Markdown, plain-text
+fallback, and a short introduction suitable for a notification or preview.
+
+The lower-level fetchers and normalization helpers are available from the
+package modules when an application needs more control over the pipeline.
+
+## Development
+
+```bash
+uv run pytest packages/markdown-this/tests
+uv run ruff check packages/markdown-this
+```
+
+## License
+
+MIT

markdown_this-0.1.0/README.md

@@ -0,0 +1,40 @@
+# markdown-this
+
+[![PyPI](https://img.shields.io/pypi/v/markdown-this)](https://pypi.org/project/markdown-this/)
+
+Extract the readable content of a URL and convert it to Markdown. The package
+also handles GitHub repositories and Markdown files through the GitHub API,
+and arXiv abstract pages through their HTML representation.
+
+## Installation
+
+```bash
+uv add markdown-this
+```
+
+## Usage
+
+```python
+from markdown_this import extract_main_content
+
+title, markdown, fallback_text, intro = extract_main_content(
+    "https://example.com/article"
+)
+```
+
+`extract_main_content` returns the title, extracted Markdown, plain-text
+fallback, and a short introduction suitable for a notification or preview.
+
+The lower-level fetchers and normalization helpers are available from the
+package modules when an application needs more control over the pipeline.
+
+## Development
+
+```bash
+uv run pytest packages/markdown-this/tests
+uv run ruff check packages/markdown-this
+```
+
+## License
+
+MIT

markdown_this-0.1.0/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "markdown-this"
+version = "0.1.0"
+description = "Extract web pages and supported special URLs as Markdown"
+readme = "README.md"
+requires-python = ">=3.12"
+keywords = [
+    "html",
+    "markdown",
+    "readability",
+    "github",
+    "arxiv",
+]
+dependencies = [
+    "beautifulsoup4>=4.14.3",
+    "markdownify>=1.2.2",
+    "readability-lxml>=0.8.4.1",
+    "requests>=2.32.5",
+]
+
+[[project.authors]]
+name = "Martín Gaitán"
+email = "gaitan@gmail.com"
+
+[project.license]
+text = "MIT"
+
+[project.urls]
+Homepage = "https://github.com/mgaitan/lobstersgram/tree/main/packages/markdown-this"
+Repository = "https://github.com/mgaitan/lobstersgram"
+Issues = "https://github.com/mgaitan/lobstersgram/issues"
+
+[build-system]
+requires = ["uv_build>=0.7,<0.12"]
+build-backend = "uv_build"

markdown_this-0.1.0/pyproject.toml.orig

@@ -0,0 +1,24 @@
+[project]
+name = "markdown-this"
+version = "0.1.0"
+description = "Extract web pages and supported special URLs as Markdown"
+readme = "README.md"
+authors = [{ name = "Martín Gaitán", email = "gaitan@gmail.com" }]
+license = { text = "MIT" }
+requires-python = ">=3.12"
+keywords = ["html", "markdown", "readability", "github", "arxiv"]
+dependencies = [
+    "beautifulsoup4>=4.14.3",
+    "markdownify>=1.2.2",
+    "readability-lxml>=0.8.4.1",
+    "requests>=2.32.5",
+]
+
+[project.urls]
+Homepage = "https://github.com/mgaitan/lobstersgram/tree/main/packages/markdown-this"
+Repository = "https://github.com/mgaitan/lobstersgram"
+Issues = "https://github.com/mgaitan/lobstersgram/issues"
+
+[build-system]
+requires = ["uv_build>=0.7,<0.12"]
+build-backend = "uv_build"

markdown_this-0.1.0/src/markdown_this/__init__.py

@@ -0,0 +1,45 @@
+"""Convert web pages and supported special URLs to Markdown."""
+
+from markdown_this.extractor import ContentDownloadError, extract_main_content
+from markdown_this.fetchers import (
+    _github_repo_match,
+    fetch_arxiv_abstract,
+    fetch_github_blob_markdown,
+    fetch_github_readme,
+    fetch_html,
+    fetch_url,
+)
+from markdown_this.html import make_images_absolute, preprocess_figures
+from markdown_this.markdown import (
+    _extract_leading_heading,
+    _is_html_badge_block,
+    _make_markdown_images_absolute,
+    _make_markdown_links_absolute,
+    _normalize_markdown_links,
+    _strip_badge_paragraphs,
+    extract_intro,
+    markdown_to_text,
+    strip_leading_title_heading,
+)
+
+__all__ = [
+    "ContentDownloadError",
+    "_extract_leading_heading",
+    "_github_repo_match",
+    "_is_html_badge_block",
+    "_make_markdown_images_absolute",
+    "_make_markdown_links_absolute",
+    "_normalize_markdown_links",
+    "_strip_badge_paragraphs",
+    "extract_intro",
+    "extract_main_content",
+    "fetch_arxiv_abstract",
+    "fetch_github_blob_markdown",
+    "fetch_github_readme",
+    "fetch_html",
+    "fetch_url",
+    "make_images_absolute",
+    "markdown_to_text",
+    "preprocess_figures",
+    "strip_leading_title_heading",
+]

markdown_this-0.1.0/src/markdown_this/extractor.py

@@ -0,0 +1,80 @@
+"""The main URL-to-Markdown extraction pipeline."""
+
+from __future__ import annotations
+
+from logging import getLogger
+
+from bs4 import BeautifulSoup
+from markdownify import markdownify as html_to_md
+from readability import Document
+
+from markdown_this.fetchers import (
+    DEFAULT_REQUEST_TIMEOUT,
+    fetch_arxiv_abstract,
+    fetch_github_blob_markdown,
+    fetch_github_readme,
+    fetch_html,
+)
+from markdown_this.html import make_images_absolute, preprocess_figures
+from markdown_this.markdown import (
+    _make_markdown_links_absolute,
+    _normalize_markdown_links,
+    extract_intro,
+    markdown_to_text,
+)
+
+logger = getLogger(__name__)
+
+
+class ContentDownloadError(RuntimeError):
+    """Raised when a URL cannot provide HTML content."""
+
+    def __init__(self) -> None:
+        super().__init__("Failed to download content")
+
+
+def extract_main_content(
+    url: str,
+    request_timeout: int = DEFAULT_REQUEST_TIMEOUT,
+    min_content_length: int = 200,
+    intro_min_length: int = 40,
+) -> tuple[str, str, str, str]:
+    """Return ``(title, markdown, fallback_text, intro)`` for *url*."""
+    if github_blob_result := fetch_github_blob_markdown(url, request_timeout):
+        title, markdown = github_blob_result
+        fallback_text = markdown_to_text(markdown)
+        return title, markdown, fallback_text, extract_intro(markdown, fallback_text, intro_min_length)
+
+    if github_result := fetch_github_readme(url, request_timeout):
+        title, markdown = github_result
+        fallback_text = markdown_to_text(markdown)
+        return title, markdown, fallback_text, extract_intro(markdown, fallback_text, intro_min_length)
+
+    if arxiv_result := fetch_arxiv_abstract(url, request_timeout):
+        title, markdown = arxiv_result
+        fallback_text = markdown_to_text(markdown)
+        return title, markdown, fallback_text, extract_intro(markdown, fallback_text, intro_min_length)
+
+    downloaded = fetch_html(url, request_timeout)
+    if not downloaded:
+        raise ContentDownloadError
+
+    content_html = ""
+    title = url
+    try:
+        document = Document(downloaded)
+        content_html = document.summary() or ""
+        title = document.title() or url
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("readability failed error=%s", exc)
+
+    if not content_html or len(content_html.strip()) < min_content_length:
+        content_html = downloaded
+
+    content_html = preprocess_figures(make_images_absolute(content_html, url))
+    extracted_markdown = html_to_md(content_html)
+    extracted_markdown = _normalize_markdown_links(extracted_markdown)
+    extracted_markdown = _make_markdown_links_absolute(extracted_markdown, url)
+    fallback_text = BeautifulSoup(content_html, "html.parser").get_text(separator="\n").strip()
+    intro = extract_intro(extracted_markdown, fallback_text, intro_min_length)
+    return title, extracted_markdown, fallback_text, intro

markdown_this-0.1.0/src/markdown_this/fetchers.py

@@ -0,0 +1,186 @@
+"""HTTP fetchers and special URL handlers."""
+
+from __future__ import annotations
+
+import base64
+import re
+import urllib.parse
+from logging import getLogger
+
+import requests
+from bs4 import BeautifulSoup, UnicodeDammit
+
+from markdown_this.markdown import (
+    _extract_leading_heading,
+    _make_markdown_images_absolute,
+    _strip_badge_paragraphs,
+)
+
+DEFAULT_REQUEST_TIMEOUT = 20
+GITHUB_REPO_PATH_PARTS = 2
+logger = getLogger(__name__)
+
+_GITHUB_REPO_RE = re.compile(r"^https?://github\.com/(?P<owner>[^/]+)/(?P<repo>[^/?#]+)(?:[/?#].*)?$")
+_GITHUB_BLOB_RE = re.compile(
+    r"^https?://github\.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)/blob/(?P<branch>[^/]+)/(?P<path>.+\.(?:md|markdown))$",
+    re.IGNORECASE,
+)
+_ARXIV_ABS_RE = re.compile(r"^https?://arxiv\.org/abs/(?P<arxiv_id>[^?#]+)(?:[?#].*)?$", re.IGNORECASE)
+
+
+def fetch_url(url: str, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> str:
+    """Follow redirects and return the final URL."""
+    logger.debug("fetch_url start url=%s", url)
+    response = requests.get(
+        url,
+        timeout=timeout,
+        allow_redirects=True,
+        headers={"User-Agent": "lobsters-telegraph-bot"},
+    )
+    response.raise_for_status()
+    logger.debug("fetch_url final url=%s status=%s", response.url, response.status_code)
+    return response.url
+
+
+def fetch_html(url: str, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> str | None:
+    """Fetch HTML and decode UTF-8, falling back to BeautifulSoup detection."""
+    response = requests.get(url, timeout=timeout, headers={"User-Agent": "lobsters-telegraph-bot"})
+    response.raise_for_status()
+    try:
+        return response.content.decode("utf-8")
+    except UnicodeDecodeError:
+        dammit = UnicodeDammit(response.content, is_html=True)
+        return dammit.unicode_markup or response.content.decode("latin-1")
+
+
+def _github_repo_match(url: str) -> re.Match[str] | None:
+    """Return a match when *url* is a GitHub repository root URL."""
+    match = _GITHUB_REPO_RE.match(url)
+    if match is None:
+        return None
+    if len(urllib.parse.urlparse(url).path.strip("/").split("/")) != GITHUB_REPO_PATH_PARTS:
+        return None
+    return match
+
+
+def _fetch_github_api_file(api_url: str, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> str | None:
+    """Fetch, decode, and normalize a Markdown file from the GitHub API."""
+    try:
+        response = requests.get(
+            api_url,
+            timeout=timeout,
+            headers={"User-Agent": "lobsters-telegraph-bot", "Accept": "application/vnd.github+json"},
+        )
+        response.raise_for_status()
+        data = response.json()
+        name: str = data.get("name", "")
+        if not name.lower().endswith((".md", ".markdown")):
+            logger.debug("GitHub API file is not Markdown: name=%r url=%s", name, api_url)
+            return None
+        markdown = base64.b64decode(data.get("content", "")).decode("utf-8")
+        markdown = _strip_badge_paragraphs(markdown)
+        download_url: str = data.get("download_url") or ""
+        if download_url:
+            markdown = _make_markdown_images_absolute(markdown, download_url.rsplit("/", 1)[0] + "/")
+    except (requests.RequestException, ValueError, KeyError) as exc:
+        logger.warning("GitHub API file failed url=%s error=%s", api_url, exc)
+        return None
+    return markdown
+
+
+def fetch_github_readme(url: str, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> tuple[str, str] | None:
+    """Return ``(title, markdown)`` for a GitHub repository root URL."""
+    match = _github_repo_match(url)
+    if match is None:
+        return None
+    owner, repo = match.group("owner"), match.group("repo")
+    title = f"{owner}/{repo}"
+    try:
+        response = requests.get(
+            f"https://api.github.com/repos/{owner}/{repo}",
+            timeout=timeout,
+            headers={"User-Agent": "lobsters-telegraph-bot", "Accept": "application/vnd.github+json"},
+        )
+        response.raise_for_status()
+        data = response.json()
+        full_name = data.get("full_name") or title
+        description = (data.get("description") or "").strip()
+        title = f"{full_name} – {description}" if description else full_name  # noqa: RUF001
+    except requests.RequestException as exc:
+        logger.warning("GitHub repo info failed error=%s", exc)
+
+    markdown = _fetch_github_api_file(f"https://api.github.com/repos/{owner}/{repo}/readme", timeout)
+    if markdown is None:
+        return None
+    return title, markdown
+
+
+def fetch_github_blob_markdown(url: str, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> tuple[str, str] | None:
+    """Return ``(title, markdown)`` for a GitHub Markdown blob URL."""
+    match = _GITHUB_BLOB_RE.match(url)
+    if match is None:
+        return None
+    owner, repo, branch, path = (match.group(name) for name in ("owner", "repo", "branch", "path"))
+    api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}?ref={branch}"
+    markdown = _fetch_github_api_file(api_url, timeout)
+    if markdown is None:
+        return None
+    heading, markdown = _extract_leading_heading(markdown)
+    return heading or f"{owner}/{repo}/{path}", markdown
+
+
+def _parse_arxiv_html(html: str, arxiv_id: str) -> tuple[str, list[str]]:
+    """Parse an arXiv abstract page into a title and Markdown fragments."""
+    soup = BeautifulSoup(html, "html.parser")
+    title_tag = soup.find("h1", class_="title")
+    if title_tag:
+        descriptor = title_tag.find("span", class_="descriptor")
+        if descriptor:
+            descriptor.extract()
+        title = title_tag.get_text(separator=" ", strip=True)
+    else:
+        title = arxiv_id
+
+    parts: list[str] = []
+    authors_tag = soup.find("div", class_="authors")
+    if authors_tag:
+        descriptor = authors_tag.find("span", class_="descriptor")
+        if descriptor:
+            descriptor.extract()
+        author_links = authors_tag.find_all("a")
+        authors_text = ", ".join(author.get_text(strip=True) for author in author_links)
+        if not authors_text:
+            authors_text = authors_tag.get_text(separator=" ", strip=True)
+        if authors_text:
+            parts.append(f"**Authors:** {authors_text}")
+
+    abstract_tag = soup.find("blockquote", class_="abstract")
+    if abstract_tag:
+        descriptor = abstract_tag.find("span", class_="descriptor")
+        if descriptor:
+            descriptor.extract()
+        abstract_text = abstract_tag.get_text(separator=" ", strip=True)
+        if abstract_text:
+            parts.append(f"> {abstract_text}")
+    return title, parts
+
+
+def fetch_arxiv_abstract(url: str, timeout: int = DEFAULT_REQUEST_TIMEOUT) -> tuple[str, str] | None:
+    """Return ``(title, markdown)`` for an arXiv abstract URL."""
+    match = _ARXIV_ABS_RE.match(url)
+    if match is None:
+        return None
+    arxiv_id = match.group("arxiv_id")
+    abs_url = f"https://arxiv.org/abs/{arxiv_id}"
+    try:
+        html = fetch_html(abs_url, timeout)
+    except requests.RequestException as exc:
+        logger.warning("arXiv fetch failed url=%s error=%s", abs_url, exc)
+        return None
+    if not html:
+        return None
+    title, parts = _parse_arxiv_html(html, arxiv_id)
+    if not parts:
+        logger.warning("arXiv content extraction returned no content url=%s", abs_url)
+        return None
+    return title, "\n\n".join(parts)

markdown_this-0.1.0/src/markdown_this/html.py

@@ -0,0 +1,77 @@
+"""HTML cleanup helpers used before Markdown conversion."""
+
+from __future__ import annotations
+
+import contextlib
+import urllib.parse
+
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+
+def _best_src_for_img(img: Tag) -> str:
+    """Return the best candidate URL for an ``<img>`` element."""
+    src = (img.get("src") or "").strip()
+    if src and not src.startswith("data:"):
+        return src
+
+    data_src = (img.get("data-src") or "").strip()
+    if data_src and not data_src.startswith("data:"):
+        return data_src
+
+    srcset = (img.get("srcset") or "").strip()
+    if srcset:
+        best_url, best_width = "", -1
+        for candidate in srcset.split(","):
+            parts = candidate.strip().split()
+            if not parts:
+                continue
+            url = parts[0]
+            width = 0
+            if len(parts) > 1 and parts[1].endswith("w"):
+                with contextlib.suppress(ValueError):
+                    width = int(parts[1].removesuffix("w"))
+            if width > best_width:
+                best_width, best_url = width, url
+        if best_url:
+            return best_url
+    return ""
+
+
+def make_images_absolute(content_html: str, base_url: str) -> str:
+    """Resolve image URLs and remove images that cannot be fetched over HTTP(S)."""
+    soup = BeautifulSoup(content_html, "html.parser")
+    for img in soup.find_all("img"):
+        src = _best_src_for_img(img)
+        if not src:
+            img.decompose()
+            continue
+        absolute = urllib.parse.urljoin(base_url, src)
+        if absolute.startswith(("http://", "https://")):
+            img["src"] = absolute
+        else:
+            img.decompose()
+    return str(soup)
+
+
+def preprocess_figures(content_html: str) -> str:
+    """Convert text figures to blockquotes while preserving image figures."""
+    soup = BeautifulSoup(content_html, "html.parser")
+    for figure in soup.find_all("figure"):
+        figcaption = figure.find("figcaption")
+        has_body_text = any(
+            element.find_parent("figcaption") is None and bool(element.get_text(strip=True))
+            for element in figure.find_all(["p", "div"])
+        )
+        if not has_body_text:
+            continue
+
+        if figcaption:
+            figcaption.extract()
+        blockquote = soup.new_tag("blockquote")
+        for child in list(figure.children):
+            blockquote.append(child.extract())
+        figure.replace_with(blockquote)
+        if figcaption:
+            blockquote.insert_after(figcaption)
+    return str(soup)

markdown_this-0.1.0/src/markdown_this/markdown.py

@@ -0,0 +1,147 @@
+"""Markdown cleanup and text extraction helpers."""
+
+from __future__ import annotations
+
+import re
+import urllib.parse
+from logging import getLogger
+
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+logger = getLogger(__name__)
+
+_BADGE_RE = re.compile(r"\[!\[[^\]]*\]\([^)]+\)\]\([^)]+\)")
+_HTML_P_BLOCK_RE = re.compile(r"<p(?:\s[^>]*)?>.*?</p>", re.IGNORECASE | re.DOTALL)
+_BROKEN_LINK_RE = re.compile(r"\[(\s*\n[ \t]*\n[ \t]*)([^\]]*)\]\(")
+_HARD_BREAK_BEFORE_LINK_RE = re.compile(r"(?<!\\) {2,}\n(?=\[[^\]]+\]\([^)]+\))")
+
+
+def _extract_leading_heading(markdown: str) -> tuple[str | None, str]:
+    """Extract and remove the first heading when it starts the Markdown."""
+    stripped = markdown.lstrip("\n")
+    match = re.match(r"^#{1,6}\s+(.*?)\s*$", stripped, re.MULTILINE)
+    if match:
+        return match.group(1).strip(), stripped[match.end() :].lstrip("\n")
+    return None, markdown
+
+
+def _is_html_badge_block(html: str) -> bool:  # noqa: C901
+    """Return True if *html* is a ``<p>`` element containing at least two badge images."""
+    soup = BeautifulSoup(html, "html.parser")
+    p = soup.find("p")
+    if p is None:
+        return False
+    img_count = 0
+    for child in p.children:
+        match child:
+            case str() as text:
+                if text.strip():
+                    return False
+            case Tag(name="img"):
+                img_count += 1
+            case Tag(name="a"):
+                for grandchild in child.children:
+                    match grandchild:
+                        case str() as text:
+                            if text.strip():
+                                return False
+                        case Tag(name="img"):
+                            img_count += 1
+                        case _:
+                            return False
+            case _:
+                return False
+    return img_count >= 2  # noqa: PLR2004
+
+
+def _strip_badge_paragraphs(markdown: str) -> str:
+    """Remove badge-only paragraphs from Markdown."""
+    markdown = _HTML_P_BLOCK_RE.sub(
+        lambda match: "" if _is_html_badge_block(match.group(0)) else match.group(0),
+        markdown,
+    )
+    result: list[str] = []
+    for paragraph in markdown.split("\n\n"):
+        remaining = _BADGE_RE.sub("", paragraph).strip()
+        if remaining:
+            result.append(paragraph)
+    return "\n\n".join(result)
+
+
+def _make_markdown_images_absolute(markdown: str, base_url: str) -> str:
+    """Resolve relative image URLs in Markdown against *base_url*."""
+
+    def replace(match: re.Match[str]) -> str:
+        alt, image_url = match.group(1), match.group(2)
+        if image_url.startswith(("http://", "https://", "data:")):
+            return match.group(0)
+        return f"![{alt}]({urllib.parse.urljoin(base_url, image_url)})"
+
+    return re.sub(r"!\[([^\]]*)\]\(([^)]+)\)", replace, markdown)
+
+
+def _make_markdown_links_absolute(markdown: str, base_url: str) -> str:
+    """Resolve relative Markdown link URLs against *base_url*."""
+
+    def replace(match: re.Match[str]) -> str:
+        text, href = match.group(1), match.group(2).strip()
+        if href.startswith(("http://", "https://", "mailto:", "tel:", "data:")):
+            return match.group(0)
+        absolute = urllib.parse.urljoin(base_url, href)
+        if not absolute.startswith(("http://", "https://")):
+            return match.group(0)
+        return f"[{text}]({absolute})"
+
+    return re.sub(r"(?<!!)\[([^\]]+)\]\(([^)]+)\)", replace, markdown)
+
+
+def _normalize_markdown_links(markdown: str) -> str:
+    """Fix Markdown links broken by blank lines or hard breaks."""
+
+    def fix(match: re.Match[str]) -> str:
+        text = match.group(2).replace("\n", " ").strip()
+        return f"[{text}]("
+
+    normalized = _BROKEN_LINK_RE.sub(fix, markdown)
+    return _HARD_BREAK_BEFORE_LINK_RE.sub("\n", normalized)
+
+
+def strip_leading_title_heading(markdown: str, title: str) -> str:
+    """Remove a leading Markdown heading when it duplicates *title*."""
+    stripped = markdown.lstrip("\n")
+    match = re.match(r"^#{1,6}\s+(.*)\s*$", stripped, re.MULTILINE)
+    if match and match.group(1).strip().lower() == title.strip().lower():
+        return stripped[match.end() :].lstrip("\n")
+    return markdown
+
+
+def markdown_to_text(markdown_text: str) -> str:
+    """Reduce Markdown to plain text suitable for previews."""
+    text = markdown_text
+    text = re.sub(r"!\[([^\]]*)\]\([^)]+\)", "", text)
+    text = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
+    text = re.sub(r"`([^`]+)`", r"\1", text)
+    text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)
+    text = re.sub(r"\[\]\([^)]+\)", "", text)
+    text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE)
+    text = re.sub(r"^>\s?", "", text, flags=re.MULTILINE)
+    text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE)
+    return re.sub(r"[_*]{1,3}([^_*]+)[_*]{1,3}", r"\1", text)
+
+
+def extract_intro(markdown_text: str, fallback_text: str, min_length: int = 40) -> str:
+    """Return the first substantial paragraph, falling back to plain text."""
+    text = markdown_to_text(markdown_text)
+    for chunk in text.split("\n\n"):
+        line = chunk.strip()
+        if not line:
+            continue
+        intro = line.replace("\n", " ").strip()
+        if len(intro) >= min_length:
+            return intro
+    for line in fallback_text.splitlines():
+        stripped = line.strip()
+        if stripped:
+            return stripped
+    return ""