wsws-md 0.1.2__tar.gz

wsws_md-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Joe Mount
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

wsws_md-0.1.2/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: wsws-md
+Version: 0.1.2
+Summary: Convert World Socialist Web Site articles to Markdown
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: markdownify>=0.13
+Requires-Dist: lxml>=4.9
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Requires-Dist: pytest-cov>=5.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: wsws-md[test]; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: types-beautifulsoup4; extra == "dev"
+Dynamic: license-file
+
+# WSWS Markdown
+
+[![CI](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/ci.yml/badge.svg)](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/ci.yml)
+[![Publish to PyPI](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/publish.yml/badge.svg)](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/publish.yml)
+
+A tool to convert articles from the World Socialist Web Site to Markdown format.
+
+## Install
+
+```sh
+pip install wsws-md
+```
+
+## Usage
+
+```sh
+wswsmd "https://www.wsws.org/en/articles/2026/07/03/coeq-j03.html"
+```
+
+Prints the article as Markdown (with YAML frontmatter containing title, author,
+date, description and source URL) to stdout. Use `-o FILE` / `--output FILE`
+to write to a file instead:
+
+```sh
+wswsmd "https://www.wsws.org/en/articles/2026/07/03/coeq-j03.html" -o article.md
+```
+
+## Development
+
+```sh
+pip install -e ".[test]"
+pytest
+```
+
+Pull request titles must follow [Conventional Commits](https://www.conventionalcommits.org/)
+— see [CONTRIBUTING.md](CONTRIBUTING.md). Releases are automated with
+[release-please](https://github.com/googleapis/release-please).

wsws_md-0.1.2/README.md

@@ -0,0 +1,37 @@
+# WSWS Markdown
+
+[![CI](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/ci.yml/badge.svg)](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/ci.yml)
+[![Publish to PyPI](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/publish.yml/badge.svg)](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/publish.yml)
+
+A tool to convert articles from the World Socialist Web Site to Markdown format.
+
+## Install
+
+```sh
+pip install wsws-md
+```
+
+## Usage
+
+```sh
+wswsmd "https://www.wsws.org/en/articles/2026/07/03/coeq-j03.html"
+```
+
+Prints the article as Markdown (with YAML frontmatter containing title, author,
+date, description and source URL) to stdout. Use `-o FILE` / `--output FILE`
+to write to a file instead:
+
+```sh
+wswsmd "https://www.wsws.org/en/articles/2026/07/03/coeq-j03.html" -o article.md
+```
+
+## Development
+
+```sh
+pip install -e ".[test]"
+pytest
+```
+
+Pull request titles must follow [Conventional Commits](https://www.conventionalcommits.org/)
+— see [CONTRIBUTING.md](CONTRIBUTING.md). Releases are automated with
+[release-please](https://github.com/googleapis/release-please).

wsws_md-0.1.2/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "wsws-md"
+version = "0.1.2"
+description = "Convert World Socialist Web Site articles to Markdown"
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.9"
+dependencies = [
+    "requests>=2.28",
+    "beautifulsoup4>=4.12",
+    "markdownify>=0.13",
+    "lxml>=4.9",
+]
+
+[project.scripts]
+wswsmd = "wswsmd.cli:main"
+
+[project.optional-dependencies]
+test = ["pytest>=7.0", "pytest-cov>=5.0"]
+dev = [
+    "wsws-md[test]",
+    "ruff>=0.6",
+    "mypy>=1.10",
+    "types-requests",
+    "types-beautifulsoup4",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+addopts = "--cov=wswsmd --cov-report=term-missing --cov-fail-under=90"
+
+[tool.coverage.run]
+source = ["src/wswsmd"]
+
+[tool.ruff]
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.mypy]
+python_version = "3.10"
+packages = ["wswsmd"]
+warn_unused_configs = true
+
+[[tool.mypy.overrides]]
+module = "wswsmd.parse"
+disable_error_code = ["union-attr", "index", "call-arg", "return-value"]

wsws_md-0.1.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

wsws_md-0.1.2/src/wsws_md.egg-info/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: wsws-md
+Version: 0.1.2
+Summary: Convert World Socialist Web Site articles to Markdown
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: markdownify>=0.13
+Requires-Dist: lxml>=4.9
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Requires-Dist: pytest-cov>=5.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: wsws-md[test]; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: types-beautifulsoup4; extra == "dev"
+Dynamic: license-file
+
+# WSWS Markdown
+
+[![CI](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/ci.yml/badge.svg)](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/ci.yml)
+[![Publish to PyPI](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/publish.yml/badge.svg)](https://github.com/JoeMountWSWS/wsws-md/actions/workflows/publish.yml)
+
+A tool to convert articles from the World Socialist Web Site to Markdown format.
+
+## Install
+
+```sh
+pip install wsws-md
+```
+
+## Usage
+
+```sh
+wswsmd "https://www.wsws.org/en/articles/2026/07/03/coeq-j03.html"
+```
+
+Prints the article as Markdown (with YAML frontmatter containing title, author,
+date, description and source URL) to stdout. Use `-o FILE` / `--output FILE`
+to write to a file instead:
+
+```sh
+wswsmd "https://www.wsws.org/en/articles/2026/07/03/coeq-j03.html" -o article.md
+```
+
+## Development
+
+```sh
+pip install -e ".[test]"
+pytest
+```
+
+Pull request titles must follow [Conventional Commits](https://www.conventionalcommits.org/)
+— see [CONTRIBUTING.md](CONTRIBUTING.md). Releases are automated with
+[release-please](https://github.com/googleapis/release-please).

wsws_md-0.1.2/src/wsws_md.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+LICENSE
+README.md
+pyproject.toml
+src/wsws_md.egg-info/PKG-INFO
+src/wsws_md.egg-info/SOURCES.txt
+src/wsws_md.egg-info/dependency_links.txt
+src/wsws_md.egg-info/entry_points.txt
+src/wsws_md.egg-info/requires.txt
+src/wsws_md.egg-info/top_level.txt
+src/wswsmd/__init__.py
+src/wswsmd/cli.py
+src/wswsmd/convert.py
+src/wswsmd/fetch.py
+src/wswsmd/parse.py
+tests/test_cli.py
+tests/test_convert.py
+tests/test_fetch.py
+tests/test_parse.py
+tests/test_parse_edge_cases.py

wsws_md-0.1.2/src/wsws_md.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

wsws_md-0.1.2/src/wsws_md.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+wswsmd = wswsmd.cli:main

wsws_md-0.1.2/src/wsws_md.egg-info/requires.txt

@@ -0,0 +1,15 @@
+requests>=2.28
+beautifulsoup4>=4.12
+markdownify>=0.13
+lxml>=4.9
+
+[dev]
+wsws-md[test]
+ruff>=0.6
+mypy>=1.10
+types-requests
+types-beautifulsoup4
+
+[test]
+pytest>=7.0
+pytest-cov>=5.0

wsws_md-0.1.2/src/wsws_md.egg-info/top_level.txt

@@ -0,0 +1 @@
+wswsmd

wsws_md-0.1.2/src/wswsmd/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.1"

wsws_md-0.1.2/src/wswsmd/cli.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import argparse
+import sys
+
+import requests
+
+from wswsmd.convert import to_markdown
+from wswsmd.fetch import fetch_html
+from wswsmd.parse import parse_article
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="wswsmd",
+        description="Convert a World Socialist Web Site article to Markdown.",
+    )
+    parser.add_argument("url", help="URL of the WSWS article to convert")
+    parser.add_argument(
+        "-o",
+        "--output",
+        metavar="FILE",
+        help="write Markdown to FILE instead of printing to stdout",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    try:
+        html = fetch_html(args.url)
+        article = parse_article(html, args.url)
+    except requests.RequestException as exc:
+        print(f"wswsmd: failed to fetch {args.url}: {exc}", file=sys.stderr)
+        return 1
+    except ValueError as exc:
+        print(f"wswsmd: {exc}", file=sys.stderr)
+        return 1
+
+    markdown = to_markdown(article)
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(markdown)
+    else:
+        print(markdown, end="")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

wsws_md-0.1.2/src/wswsmd/convert.py

@@ -0,0 +1,43 @@
+import re
+
+from markdownify import ATX, markdownify
+
+from wswsmd.parse import Article
+
+_BLANK_LINES = re.compile(r"\n{3,}")
+
+
+def _yaml_str(value: str) -> str:
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ")
+    return f'"{escaped}"'
+
+
+def _frontmatter(article: Article) -> str:
+    lines = [
+        "---",
+        f"title: {_yaml_str(article.title)}",
+        f"author: {_yaml_str(article.author)}",
+        f"date: {_yaml_str(article.date_iso)}",
+        f"description: {_yaml_str(article.description)}",
+        f"url: {_yaml_str(article.url)}",
+        "---",
+    ]
+    return "\n".join(lines)
+
+
+def _byline(article: Article) -> str:
+    parts = [p for p in (article.author, article.date_display) if p]
+    return f"*{' — '.join(parts)}*" if parts else ""
+
+
+def to_markdown(article: Article) -> str:
+    body = markdownify(article.body_html, heading_style=ATX, bullets="-").strip()
+    body = _BLANK_LINES.sub("\n\n", body)
+
+    sections = [_frontmatter(article), f"# {article.title}"]
+    byline = _byline(article)
+    if byline:
+        sections.append(byline)
+    sections.append(body)
+
+    return "\n\n".join(sections) + "\n"

wsws_md-0.1.2/src/wswsmd/fetch.py

@@ -0,0 +1,14 @@
+import requests
+
+TIMEOUT_SECONDS = 15
+
+
+def fetch_html(url: str) -> str:
+    """Fetch a URL and return its response body as text.
+
+    Raises requests.RequestException (or a subclass) on network errors
+    or non-2xx status codes.
+    """
+    response = requests.get(url, timeout=TIMEOUT_SECONDS)
+    response.raise_for_status()
+    return response.text

wsws_md-0.1.2/src/wswsmd/parse.py

@@ -0,0 +1,107 @@
+from dataclasses import dataclass
+
+from bs4 import BeautifulSoup, Tag
+
+TITLE_SUFFIX = " - World Socialist Web Site"
+
+
+@dataclass
+class Article:
+    title: str
+    author: str
+    date_iso: str
+    date_display: str
+    url: str
+    description: str
+    body_html: str
+
+
+def _meta_content(soup: BeautifulSoup, **attrs) -> str:
+    tag = soup.find("meta", attrs=attrs)
+    return tag["content"].strip() if tag and tag.get("content") else ""
+
+
+def _extract_title(soup: BeautifulSoup) -> str:
+    title = _meta_content(soup, property="og:title")
+    if title:
+        return title
+    if soup.title and soup.title.string:
+        text = soup.title.string.strip()
+        if text.endswith(TITLE_SUFFIX):
+            text = text[: -len(TITLE_SUFFIX)]
+        return text
+    return ""
+
+
+def _extract_author(soup: BeautifulSoup) -> str:
+    header = soup.find("header")
+    if not header:
+        return ""
+    link = header.find("a", href=lambda h: h and h.startswith("/en/authors/"))
+    return link.get_text(strip=True) if link else ""
+
+
+def _extract_date(soup: BeautifulSoup) -> tuple[str, str]:
+    header = soup.find("header")
+    time_tag = header.find("time") if header else soup.find("time")
+    if not time_tag:
+        return "", ""
+    return time_tag.get("datetime", "").strip(), time_tag.get_text(strip=True)
+
+
+def _extract_body(soup: BeautifulSoup) -> Tag:
+    section = soup.select_one('[itemprop="articleBody"]')
+    if section is None:
+        raise ValueError(
+            "Could not find article body (no [itemprop=articleBody] element)"
+        )
+
+    # The real prose lives in a nested div.content; everything above it
+    # (share-icon <ul>, layout <style>/<div> wrappers) is chrome, and
+    # everything below/around it ("Read more", comments, related topics)
+    # is a sibling, not part of the article.
+    content = section.select_one("div.content") or section.find("div", recursive=False)
+    if content is None:
+        raise ValueError("Could not find article body content")
+
+    # Widgets (audio player, newsletter signup, promo banners) are the
+    # only direct-child <div>s here; real paragraphs are always <p>.
+    for widget in content.find_all("div", recursive=False):
+        widget.decompose()
+    for aside in content.find_all("aside"):
+        aside.decompose()
+    # Photos are wrapped as <figure><img/><figcaption>...</figcaption></figure>;
+    # dropping the whole <figure> removes the image and its caption together.
+    for figure in content.find_all("figure"):
+        figure.decompose()
+    for tag in content.find_all(["script", "style", "svg"]):
+        tag.decompose()
+    for img in content.find_all("img"):
+        classes = img.get("class") or []
+        if "dn-m" in classes:
+            img.decompose()
+
+    return content
+
+
+def parse_article(html: str, url: str) -> Article:
+    soup = BeautifulSoup(html, "lxml")
+
+    title = _extract_title(soup)
+    if not title:
+        raise ValueError(f"Could not find an article title at {url}")
+
+    author = _extract_author(soup)
+    date_iso, date_display = _extract_date(soup)
+    description = _meta_content(soup, name="description")
+    body = _extract_body(soup)
+
+    return Article(
+        title=title,
+        author=author,
+        date_iso=date_iso,
+        date_display=date_display,
+        url=url,
+        description=description,
+        body_html=body.decode_contents(),
+    )

wsws_md-0.1.2/tests/test_cli.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import requests
+
+from wswsmd import cli
+
+FIXTURE = Path(__file__).parent / "fixtures" / "coeq-j03.html"
+URL = "https://www.wsws.org/en/articles/2026/07/03/coeq-j03.html"
+
+
+def test_build_parser_requires_url():
+    parser = cli.build_parser()
+    with pytest.raises(SystemExit):
+        parser.parse_args([])
+
+
+def test_build_parser_accepts_output_flag():
+    parser = cli.build_parser()
+    args = parser.parse_args([URL, "-o", "out.md"])
+    assert args.url == URL
+    assert args.output == "out.md"
+
+
+def test_main_prints_markdown_to_stdout(capsys):
+    html = FIXTURE.read_text()
+    with patch("wswsmd.cli.fetch_html", return_value=html):
+        exit_code = cli.main([URL])
+
+    captured = capsys.readouterr()
+    assert exit_code == 0
+    assert "The Colorado primary" in captured.out
+
+
+def test_main_writes_markdown_to_file(tmp_path):
+    html = FIXTURE.read_text()
+    output_file = tmp_path / "article.md"
+    with patch("wswsmd.cli.fetch_html", return_value=html):
+        exit_code = cli.main([URL, "-o", str(output_file)])
+
+    assert exit_code == 0
+    assert "The Colorado primary" in output_file.read_text()
+
+
+def test_main_reports_fetch_errors(capsys):
+    with patch("wswsmd.cli.fetch_html", side_effect=requests.ConnectionError("boom")):
+        exit_code = cli.main([URL])
+
+    captured = capsys.readouterr()
+    assert exit_code == 1
+    assert "failed to fetch" in captured.err
+
+
+def test_main_reports_parse_errors(capsys):
+    with patch("wswsmd.cli.fetch_html", return_value="<html><body></body></html>"):
+        exit_code = cli.main([URL])
+
+    captured = capsys.readouterr()
+    assert exit_code == 1
+    assert "wswsmd:" in captured.err

wsws_md-0.1.2/tests/test_convert.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+
+import pytest
+
+from wswsmd.convert import to_markdown
+from wswsmd.parse import parse_article
+
+FIXTURE = Path(__file__).parent / "fixtures" / "coeq-j03.html"
+URL = "https://www.wsws.org/en/articles/2026/07/03/coeq-j03.html"
+
+
+@pytest.fixture
+def markdown():
+    html = FIXTURE.read_text()
+    article = parse_article(html, URL)
+    return to_markdown(article)
+
+
+def test_frontmatter(markdown):
+    assert markdown.startswith("---\n")
+    front, _, rest = markdown[4:].partition("\n---\n")
+    assert (
+        'title: "The Colorado primary and the growing support for socialism"' in front
+    )
+    assert 'author: "Patrick Martin"' in front
+    assert f'url: "{URL}"' in front
+
+    assert "# The Colorado primary and the growing support for socialism" in rest
+    assert "*Patrick Martin — 2 July 2026*" in rest
+
+
+def test_body_is_markdown_paragraphs(markdown):
+    assert "Diana DeGette" in markdown
+    assert "<p>" not in markdown
+    assert "<div" not in markdown
+
+
+def test_no_leftover_ads_or_navigation(markdown):
+    assert "AudioNative" not in markdown
+    assert "Read more" not in markdown
+    assert "Related Topics" not in markdown
+
+
+def test_inline_emphasis_preserved(markdown):
+    assert (
+        "*Washington Post*" in markdown
+    )  # <em>Washington Post</em> converts to emphasis

wsws_md-0.1.2/tests/test_fetch.py

@@ -0,0 +1,29 @@
+from unittest.mock import Mock, patch
+
+import pytest
+import requests
+
+from wswsmd.fetch import TIMEOUT_SECONDS, fetch_html
+
+
+def test_fetch_html_returns_response_text():
+    mock_response = Mock()
+    mock_response.text = "<html>ok</html>"
+    mock_response.raise_for_status.return_value = None
+
+    with patch("wswsmd.fetch.requests.get", return_value=mock_response) as mock_get:
+        result = fetch_html("https://example.com/article")
+
+    assert result == "<html>ok</html>"
+    mock_get.assert_called_once_with(
+        "https://example.com/article", timeout=TIMEOUT_SECONDS
+    )
+
+
+def test_fetch_html_raises_on_http_error():
+    mock_response = Mock()
+    mock_response.raise_for_status.side_effect = requests.HTTPError("404")
+
+    with patch("wswsmd.fetch.requests.get", return_value=mock_response):
+        with pytest.raises(requests.HTTPError):
+            fetch_html("https://example.com/missing")

wsws_md-0.1.2/tests/test_parse.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+
+import pytest
+
+from wswsmd.parse import parse_article
+
+FIXTURE = Path(__file__).parent / "fixtures" / "coeq-j03.html"
+URL = "https://www.wsws.org/en/articles/2026/07/03/coeq-j03.html"
+
+PHOTO_FIXTURE = Path(__file__).parent / "fixtures" / "yfab-j12.html"
+PHOTO_URL = "https://www.wsws.org/en/articles/2026/07/12/yfab-j12.html"
+
+
+@pytest.fixture
+def article():
+    html = FIXTURE.read_text()
+    return parse_article(html, URL)
+
+
+@pytest.fixture
+def photo_article():
+    html = PHOTO_FIXTURE.read_text()
+    return parse_article(html, PHOTO_URL)
+
+
+def test_metadata(article):
+    assert article.title == "The Colorado primary and the growing support for socialism"
+    assert article.author == "Patrick Martin"
+    assert article.date_iso == "2026-07-03T02:01:26.000Z"
+    assert article.date_display == "2 July 2026"
+    assert article.url == URL
+    assert "Democratic Socialists of America" in article.description
+
+
+def test_body_contains_article_text(article):
+    assert "Diana DeGette" in article.body_html
+    assert "Socialist Equality Party" in article.body_html
+
+
+def test_body_excludes_ads_and_widgets(article):
+    assert "AudioNative" not in article.body_html
+    assert "elevenlabs" not in article.body_html.lower()
+    assert "jun25-declaration-of-independence" not in article.body_html
+
+
+def test_body_excludes_trailing_navigation(article):
+    assert "Read more" not in article.body_html
+    assert "Related Topics" not in article.body_html
+    assert "Contact us" not in article.body_html
+
+
+def test_body_has_no_duplicate_mobile_images(article):
+    assert "dn-m" not in article.body_html
+
+
+def test_photo_metadata(photo_article):
+    assert photo_article.title == (
+        "Durham Miners' Gala used by labour and union bureaucracy "
+        "to back Andy Burnham as new prime minister"
+    )
+    assert photo_article.author == "Our reporters"
+    assert photo_article.date_iso == "2026-07-12T17:42:21.000Z"
+    assert photo_article.url == PHOTO_URL
+
+
+def test_body_excludes_photos_and_captions(photo_article):
+    assert "<figure" not in photo_article.body_html
+    assert "<figcaption" not in photo_article.body_html
+    assert "<img" not in photo_article.body_html
+    assert (
+        "Andrea Egan speaking at the Durham Miners' Gala" not in photo_article.body_html
+    )

wsws_md-0.1.2/tests/test_parse_edge_cases.py

@@ -0,0 +1,94 @@
+import pytest
+from bs4 import BeautifulSoup
+
+from wswsmd.parse import (
+    _extract_author,
+    _extract_body,
+    _extract_date,
+    _extract_title,
+    parse_article,
+)
+
+
+def soup_of(html: str) -> BeautifulSoup:
+    return BeautifulSoup(html, "lxml")
+
+
+def test_extract_title_falls_back_to_html_title_tag():
+    soup = soup_of(
+        "<html><head><title>Some Article - World Socialist Web Site</title>"
+        "</head></html>"
+    )
+    assert _extract_title(soup) == "Some Article"
+
+
+def test_extract_title_keeps_full_text_when_no_suffix():
+    soup = soup_of("<html><head><title>Some Article</title></head></html>")
+    assert _extract_title(soup) == "Some Article"
+
+
+def test_extract_title_returns_empty_when_no_title_available():
+    soup = soup_of("<html><head></head></html>")
+    assert _extract_title(soup) == ""
+
+
+def test_extract_author_returns_empty_without_header():
+    soup = soup_of("<html><body><p>no header here</p></body></html>")
+    assert _extract_author(soup) == ""
+
+
+def test_extract_date_returns_empty_without_time_tag():
+    soup = soup_of("<html><body><header></header></body></html>")
+    assert _extract_date(soup) == ("", "")
+
+
+def test_extract_body_raises_without_article_body():
+    soup = soup_of("<html><body><div>no article body here</div></body></html>")
+    with pytest.raises(ValueError, match="articleBody"):
+        _extract_body(soup)
+
+
+def test_extract_body_raises_without_content_div():
+    soup = soup_of('<html><body><div itemprop="articleBody"></div></body></html>')
+    with pytest.raises(ValueError, match="body content"):
+        _extract_body(soup)
+
+
+def test_extract_body_strips_scripts_styles_and_svgs():
+    soup = soup_of(
+        '<html><body><div itemprop="articleBody"><div class="content">'
+        "<p>Keep me</p>"
+        "<script>evil()</script>"
+        "<style>.x{}</style>"
+        "<svg></svg>"
+        "</div></div></body></html>"
+    )
+    content = _extract_body(soup)
+    html = str(content)
+    assert "Keep me" in html
+    assert "<script" not in html
+    assert "<style" not in html
+    assert "<svg" not in html
+
+
+def test_extract_body_strips_mobile_only_images():
+    soup = soup_of(
+        '<html><body><div itemprop="articleBody"><div class="content">'
+        "<p>Text</p>"
+        '<img class="dn-m" src="mobile.jpg">'
+        '<img class="dn-d" src="desktop.jpg">'
+        "</div></div></body></html>"
+    )
+    content = _extract_body(soup)
+    html = str(content)
+    assert "mobile.jpg" not in html
+    assert "desktop.jpg" in html
+
+
+def test_parse_article_raises_without_title():
+    html = (
+        '<html><head></head><body><div itemprop="articleBody">'
+        '<div class="content"><p>Body text</p></div></div></body></html>'
+    )
+    with pytest.raises(ValueError, match="title"):
+        parse_article(html, "https://example.com/x")