docs2epub 0.1.0__tar.gz

docs2epub-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install deps
+        run: uv sync --extra dev
+
+      - name: Run tests
+        run: uv run pytest

docs2epub-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build
+        run: uv build
+
+      - name: Publish to PyPI
+        env:
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+        run: uv publish

docs2epub-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.dist/
+dist/
+.venv/
+.pytest_cache/
+.DS_Store

docs2epub-0.1.0/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.4
+Name: docs2epub
+Version: 0.1.0
+Summary: Turn documentation sites into an EPUB (Kindle-friendly).
+Author: Breno Brito
+License: MIT
+Requires-Python: >=3.12
+Requires-Dist: beautifulsoup4>=4.14.3
+Requires-Dist: ebooklib>=0.20
+Requires-Dist: lxml>=6.0.2
+Requires-Dist: requests>=2.32.5
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# docs2epub
+
+Turn documentation sites into an EPUB (Kindle-friendly).
+
+Initial focus: Docusaurus sites that expose a **Next** button (docs navigation).
+
+## Install (dev)
+
+This project uses Python 3.12+.
+
+```bash
+uv sync
+uv run docs2epub --help
+```
+
+## Usage
+
+### Docusaurus “Next” crawl
+
+```bash
+# Default output is EPUB2 (Kindle-friendly) via pandoc
+uv run docs2epub \
+  --start-url "https://www.techinterviewhandbook.org/software-engineering-interview-guide/" \
+  --out "dist/tech-interview-handbook.epub" \
+  --title "Tech Interview Handbook" \
+  --author "Yangshun Tay"
+
+# Optional: build EPUB3 (ebooklib)
+uv run docs2epub \
+  --format epub3 \
+  --start-url "https://www.techinterviewhandbook.org/software-engineering-interview-guide/" \
+  --out "dist/tech-interview-handbook.epub" \
+  --title "Tech Interview Handbook" \
+  --author "Yangshun Tay"
+```
+
+## Roadmap
+
+- Add additional discovery strategies: `sitemap.xml`, sidebar parsing, and explicit link lists.
+- Optional: send-to-kindle (email), once Gmail auth is set up.

docs2epub-0.1.0/README.md

@@ -0,0 +1,40 @@
+# docs2epub
+
+Turn documentation sites into an EPUB (Kindle-friendly).
+
+Initial focus: Docusaurus sites that expose a **Next** button (docs navigation).
+
+## Install (dev)
+
+This project uses Python 3.12+.
+
+```bash
+uv sync
+uv run docs2epub --help
+```
+
+## Usage
+
+### Docusaurus “Next” crawl
+
+```bash
+# Default output is EPUB2 (Kindle-friendly) via pandoc
+uv run docs2epub \
+  --start-url "https://www.techinterviewhandbook.org/software-engineering-interview-guide/" \
+  --out "dist/tech-interview-handbook.epub" \
+  --title "Tech Interview Handbook" \
+  --author "Yangshun Tay"
+
+# Optional: build EPUB3 (ebooklib)
+uv run docs2epub \
+  --format epub3 \
+  --start-url "https://www.techinterviewhandbook.org/software-engineering-interview-guide/" \
+  --out "dist/tech-interview-handbook.epub" \
+  --title "Tech Interview Handbook" \
+  --author "Yangshun Tay"
+```
+
+## Roadmap
+
+- Add additional discovery strategies: `sitemap.xml`, sidebar parsing, and explicit link lists.
+- Optional: send-to-kindle (email), once Gmail auth is set up.

docs2epub-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "docs2epub"
+version = "0.1.0"
+description = "Turn documentation sites into an EPUB (Kindle-friendly)."
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "MIT" }
+authors = [
+  { name = "Breno Brito" },
+]
+dependencies = [
+  "beautifulsoup4>=4.14.3",
+  "ebooklib>=0.20",
+  "lxml>=6.0.2",
+  "requests>=2.32.5",
+]
+
+[project.scripts]
+docs2epub = "docs2epub.cli:main"
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0.0",
+  "pytest-cov>=4.1.0",
+]
+
+[build-system]
+requires = ["hatchling>=1.26.0"]
+build-backend = "hatchling.build"
+
+[tool.uv]
+package = true
+
+[tool.pytest.ini_options]
+addopts = "-q"
+testpaths = ["tests"]

docs2epub-0.1.0/src/docs2epub/__init__.py

@@ -0,0 +1,5 @@
+__all__ = [
+  "__version__",
+]
+
+__version__ = "0.1.0"

docs2epub-0.1.0/src/docs2epub/cli.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from .docusaurus_next import DocusaurusNextOptions, iter_docusaurus_next
+from .epub import EpubMetadata, build_epub
+from .pandoc_epub2 import build_epub2_with_pandoc
+
+
+def _build_parser() -> argparse.ArgumentParser:
+  p = argparse.ArgumentParser(
+    prog="docs2epub",
+    description="Turn documentation sites into an EPUB (Kindle-friendly).",
+  )
+
+  p.add_argument(
+    "--start-url",
+    required=True,
+    help="Starting URL for scraping (initially: Docusaurus docs page).",
+  )
+  p.add_argument(
+    "--base-url",
+    default=None,
+    help="Base URL used to resolve relative links (defaults to start-url).",
+  )
+  p.add_argument("--max-pages", type=int, default=None)
+  p.add_argument("--sleep-s", type=float, default=0.5)
+
+  p.add_argument("--title", required=True)
+  p.add_argument("--author", required=True)
+  p.add_argument("--language", default="en")
+  p.add_argument("--identifier", default=None)
+  p.add_argument("--publisher", default=None)
+
+  p.add_argument(
+    "--format",
+    default="epub2",
+    choices=["epub2", "epub3"],
+    help="Output format. Default: epub2 (Kindle-friendly).",
+  )
+
+  p.add_argument(
+    "--out",
+    required=True,
+    help="Output EPUB file path.",
+  )
+
+  return p
+
+
+def main(argv: list[str] | None = None) -> int:
+  args = _build_parser().parse_args(argv)
+
+  options = DocusaurusNextOptions(
+    start_url=args.start_url,
+    base_url=args.base_url,
+    max_pages=args.max_pages,
+    sleep_s=args.sleep_s,
+  )
+
+  chapters = iter_docusaurus_next(options)
+  if not chapters:
+    raise SystemExit("No chapters scraped (did not find article content).")
+
+  out_path: Path
+
+  if args.format == "epub2":
+    out_path = build_epub2_with_pandoc(
+      chapters=chapters,
+      out_file=Path(args.out),
+      title=args.title,
+      author=args.author,
+      language=args.language,
+      publisher=args.publisher,
+      identifier=args.identifier,
+    )
+  else:
+    meta = EpubMetadata(
+      title=args.title,
+      author=args.author,
+      language=args.language,
+      identifier=args.identifier,
+      publisher=args.publisher,
+    )
+
+    out_path = build_epub(
+      chapters=chapters,
+      out_file=Path(args.out),
+      meta=meta,
+    )
+
+  print(f"Scraped {len(chapters)} pages")
+  print(f"EPUB written to: {out_path.resolve()}")
+  return 0

docs2epub-0.1.0/src/docs2epub/docusaurus_next.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from urllib.parse import urljoin
+
+import requests
+from bs4 import BeautifulSoup, Tag
+
+from .model import Chapter
+
+
+DEFAULT_USER_AGENT = "docs2epub/0.1 (+https://github.com/brenorb/docs2epub)"
+
+
+@dataclass(frozen=True)
+class DocusaurusNextOptions:
+  start_url: str
+  base_url: str | None = None
+  max_pages: int | None = None
+  sleep_s: float = 0.5
+  user_agent: str = DEFAULT_USER_AGENT
+
+
+def _slugify_filename(text: str) -> str:
+  value = text.strip().lower()
+  value = re.sub(r"[^\w\s-]", "", value)
+  value = re.sub(r"[\s_-]+", "-", value)
+  value = value.strip("-")
+  return value or "chapter"
+
+
+def _extract_article(soup: BeautifulSoup) -> Tag:
+  article = soup.find("article")
+  if article:
+    return article
+  main = soup.find("main")
+  if main:
+    article = main.find("article")
+    if article:
+      return article
+  raise RuntimeError("Could not find <article> in page HTML")
+
+
+def _remove_unwanted(article: Tag) -> None:
+  for selector in [
+    'nav[aria-label="Breadcrumbs"]',
+    'nav[aria-label="Docs pages"]',
+    "div.theme-doc-footer",
+    "div.theme-doc-footer-edit-meta-row",
+    "div.theme-doc-version-badge",
+    "script",
+    "style",
+    "noscript",
+    "iframe",
+    "svg",
+    "button",
+  ]:
+    for el in list(article.select(selector)):
+      el.decompose()
+
+
+def _absolutize_urls(container: Tag, base_url: str) -> None:
+  for el in container.find_all(True):
+    if el.has_attr("href"):
+      href = str(el.get("href") or "")
+      if href.startswith("/"):
+        el["href"] = urljoin(base_url, href)
+    if el.has_attr("src"):
+      src = str(el.get("src") or "")
+      if src.startswith("/"):
+        el["src"] = urljoin(base_url, src)
+
+
+def _extract_next_url(soup: BeautifulSoup, base_url: str) -> str | None:
+  nav = soup.select_one('nav[aria-label="Docs pages"]')
+  if not nav:
+    return None
+
+  for a in nav.find_all("a", href=True):
+    text = " ".join(a.get_text(" ", strip=True).split())
+    if text.lower().startswith("next"):
+      return urljoin(base_url, a["href"])
+
+  return None
+
+
+def iter_docusaurus_next(options: DocusaurusNextOptions) -> list[Chapter]:
+  session = requests.Session()
+  session.headers.update({"User-Agent": options.user_agent})
+
+  url = options.start_url
+  base_url = options.base_url or options.start_url
+
+  visited: set[str] = set()
+  chapters: list[Chapter] = []
+
+  idx = 1
+  while True:
+    if options.max_pages is not None and idx > options.max_pages:
+      break
+
+    if url in visited:
+      break
+    visited.add(url)
+
+    resp = session.get(url, timeout=30)
+    resp.raise_for_status()
+
+    soup = BeautifulSoup(resp.text, "lxml")
+    article = _extract_article(soup)
+
+    title_el = article.find(["h1", "h2"])
+    title = (
+      " ".join(title_el.get_text(" ", strip=True).split()) if title_el else f"Chapter {idx}"
+    )
+
+    _remove_unwanted(article)
+    _absolutize_urls(article, base_url=base_url)
+
+    for a in list(article.select('a.hash-link[href^="#"]')):
+      a.decompose()
+
+    html = article.decode_contents()
+
+    chapters.append(Chapter(index=idx, title=title, url=url, html=html))
+
+    next_url = _extract_next_url(soup, base_url=base_url)
+    if not next_url:
+      break
+
+    url = next_url
+    idx += 1
+
+    if options.sleep_s > 0:
+      import time
+
+      time.sleep(options.sleep_s)
+
+  return chapters

docs2epub-0.1.0/src/docs2epub/epub.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable
+
+from bs4 import BeautifulSoup
+from ebooklib import epub
+
+from .model import Chapter
+
+
+EPUB_CSS = """
+body {
+  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+  line-height: 1.55;
+}
+pre, code {
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+}
+pre {
+  white-space: pre-wrap;
+  word-wrap: break-word;
+  padding: 0.75rem;
+  border: 1px solid #ddd;
+  border-radius: 0.5rem;
+  background: #f7f7f7;
+}
+.chapter-sep {
+  margin-top: 1.75rem;
+  border-top: 1px solid #ddd;
+}
+"""
+
+
+@dataclass(frozen=True)
+class EpubMetadata:
+  title: str
+  author: str
+  language: str = "en"
+  identifier: str | None = None
+  publisher: str | None = None
+  created_at: datetime | None = None
+
+
+def _extract_body_inner_html(html: str) -> str:
+  soup = BeautifulSoup(html, "lxml")
+  body = soup.find("body")
+  if not body:
+    return html
+  return body.decode_contents()
+
+
+def _strip_first_h1(html_fragment: str) -> str:
+  soup = BeautifulSoup(html_fragment, "lxml")
+  first_h1 = soup.find("h1")
+  if first_h1:
+    first_h1.decompose()
+  body = soup.find("body")
+  if body:
+    return body.decode_contents()
+  return str(soup)
+
+
+def build_epub(
+  *,
+  chapters: Iterable[Chapter],
+  out_file: str | Path,
+  meta: EpubMetadata,
+) -> Path:
+  out_path = Path(out_file)
+  out_path.parent.mkdir(parents=True, exist_ok=True)
+
+  book = epub.EpubBook()
+
+  identifier = meta.identifier or f"urn:uuid:{uuid.uuid4()}"
+  book.set_identifier(identifier)
+  book.set_title(meta.title)
+  book.set_language(meta.language)
+  book.add_author(meta.author)
+
+  if meta.publisher:
+    book.add_metadata("DC", "publisher", meta.publisher)
+
+  created_at = meta.created_at or datetime.now(timezone.utc)
+  book.add_metadata("DC", "date", created_at.isoformat())
+
+  style_item = epub.EpubItem(
+    uid="style_nav",
+    file_name="style/style.css",
+    media_type="text/css",
+    content=EPUB_CSS.encode("utf-8"),
+  )
+  book.add_item(style_item)
+
+  chapter_items: list[epub.EpubHtml] = []
+  toc_items: list[epub.Link] = []
+
+  for ch in chapters:
+    body_inner = _extract_body_inner_html(ch.html)
+    body_inner = _strip_first_h1(body_inner)
+
+    content = f"""<h1>{ch.title}</h1>
+<div class=\"chapter-sep\"></div>
+{body_inner}
+"""
+
+    item = epub.EpubHtml(
+      title=ch.title,
+      file_name=f"chap_{ch.index:03d}.xhtml",
+      lang=meta.language,
+    )
+    item.content = content
+    item.add_item(style_item)
+
+    book.add_item(item)
+    chapter_items.append(item)
+    toc_items.append(epub.Link(item.file_name, ch.title, f"chap_{ch.index:03d}"))
+
+  book.toc = tuple(toc_items)
+  book.spine = ["nav", *chapter_items]
+
+  book.add_item(epub.EpubNcx())
+  book.add_item(epub.EpubNav())
+
+  epub.write_epub(str(out_path), book, {})
+  return out_path

docs2epub-0.1.0/src/docs2epub/kindle_html.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import re
+
+from bs4 import BeautifulSoup
+
+
+def clean_html_for_kindle_epub2(html_fragment: str) -> str:
+  """Best-effort HTML cleanup for Kindle-friendly EPUB2.
+
+  This is intentionally conservative: it strips known-problematic attributes
+  and tags that commonly cause Send-to-Kindle conversion issues.
+  """
+
+  soup = BeautifulSoup(html_fragment, "lxml")
+
+  # EPUB2: <u> tag isn't consistently supported; convert to a span.
+  for u in list(soup.find_all("u")):
+    span = soup.new_tag("span")
+    span["style"] = "text-decoration: underline;"
+    span.string = u.get_text() if u.string is None else u.string
+    if u.string is None:
+      # Keep children by moving them into the span.
+      for child in list(u.contents):
+        span.append(child)
+    u.replace_with(span)
+
+  # Remove tabindex attributes (not allowed in EPUB2 XHTML).
+  for el in soup.find_all(attrs={"tabindex": True}):
+    try:
+      del el["tabindex"]
+    except KeyError:
+      pass
+
+  # Remove start attribute from ordered lists (not allowed in EPUB2 XHTML).
+  for ol in soup.find_all("ol"):
+    if ol.has_attr("start"):
+      del ol["start"]
+
+  # Strip duplicate ids in a simple way: if an id repeats, rename it.
+  seen_ids: set[str] = set()
+  for el in soup.find_all(attrs={"id": True}):
+    raw = str(el.get("id") or "").strip()
+    if not raw:
+      continue
+    if raw not in seen_ids:
+      seen_ids.add(raw)
+      continue
+    suffix = 2
+    new_id = f"{raw}-{suffix}"
+    while new_id in seen_ids:
+      suffix += 1
+      new_id = f"{raw}-{suffix}"
+    el["id"] = new_id
+    seen_ids.add(new_id)
+
+  # Remove empty fragment links that point to missing ids (best-effort).
+  # If href="#something" but no element has id="something", drop href.
+  all_ids = {str(el.get("id")) for el in soup.find_all(attrs={"id": True})}
+  for a in soup.find_all("a", href=True):
+    href = str(a.get("href") or "")
+    if href.startswith("#") and len(href) > 1:
+      frag = href[1:]
+      if frag not in all_ids:
+        del a["href"]
+
+  # Normalize weird whitespace artifacts.
+  text = str(soup)
+  text = re.sub(r"\s+", " ", text)
+  return text.strip()

docs2epub-0.1.0/src/docs2epub/model.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Chapter:
+  index: int
+  title: str
+  url: str
+  html: str