novelfetch 2.0.9__py3-none-any.whl

core/__init__.py

@@ -0,0 +1,5 @@
+"""Core shared framework used by both the TUI and GUI frontends.
+
+Hosts logic that is independent of any particular UI: progress tracking,
+translation, EPUB export, and source dispatch helpers.
+"""

core/downloader.py

@@ -0,0 +1,168 @@
+"""Download orchestrator — shared by both GUI and TUI.
+
+This module replaces the duplicated download logic that previously lived in
+:mod:`gui.screens.utils` (``_download_novel``) and :mod:`tui.download`
+(``DownloadProgressScreen._download_all``).  It owns the concurrent
+semaphore, chapter‑by‑chapter orchestration, progress reporting, and the
+final metadata/cover write‑back.
+
+The public API is a single async function:
+
+    await download(source, qualified_slug, chapters, title, ...)
+
+which returns ``(saved, failed)`` — exactly matching the signatures that
+callers currently expect.
+"""
+
+import asyncio
+import os
+
+from core.library import (
+    read_meta,
+    write_chapter,
+    write_meta,
+)
+
+DOWNLOAD_CONCURRENCY: int = 4
+
+
+async def download(
+    source,
+    qualified_slug,
+    chapters,
+    title,
+    *,
+    translate=False,
+    lang="",
+    total=None,
+    progress_cb=None,
+    base_dir=None,
+) -> tuple[int, int]:
+    """Download every chapter of a novel and persist the results.
+
+    Parameters
+    ----------
+    source : Source
+        The scraper source providing ``read_chapter`` (and ``cover_url`` if
+        a cover is needed).
+    qualified_slug : str
+        The source-qualified slug, e.g. ``royalroad:some-title``.
+    chapters : list[dict]
+        Chapter entries as returned by :func:`core.utils._get_chapters`.
+    title : str
+        Novel title (written into meta.json).
+    translate : bool
+        When True, each chapter is fetched via :func:`source.read_chapter`,
+        then translated with :func:`core.translation._translate_text`.
+    lang : str
+        Translation language code (e.g. ``"ar"``).
+    total : int, optional
+        Total chapter count for meta.json.  Defaults to ``len(chapters)``.
+    progress_cb : callable, optional
+        ``progress_cb(done, saved)`` is called after each chapter completes.
+    base_dir : str, optional
+        Root of the novels directory.  Defaults to the process working
+        directory (matching historic ``CWD``-relative behaviour).
+
+    Returns
+    -------
+    (saved, failed) : tuple[int, int]
+        *saved* counts chapters that were successfully fetched/saved;
+        *failed* counts chapters that could not be fetched.
+    """
+    novels_dir = os.path.join(base_dir or os.getcwd(), "novels")
+    if total is None:
+        total = len(chapters)
+    saved = 0
+    failed = 0
+    done = 0
+
+    sem = asyncio.Semaphore(DOWNLOAD_CONCURRENCY)
+
+    async def _process(ch):
+        nonlocal saved, failed, done
+        safe_title = ch["title"].replace("/", "-").replace(" ", "_")
+        # Determine the file path for this chapter
+        if translate:
+            path = os.path.join(novels_dir, qualified_slug, f"{safe_title}_{lang}.txt")
+        else:
+            path = os.path.join(novels_dir, qualified_slug, f"{safe_title}.txt")
+
+        # Skip if the chapter file already exists (preserves "already
+        # downloaded" behaviour from the former GUI _download_novel).
+        if os.path.exists(path):
+            done += 1
+            if progress_cb is not None:
+                progress_cb(done, saved)
+            return
+
+        async with sem:
+            try:
+                if translate:
+                    lines = await source.read_chapter(ch["url"])
+                    if not lines:
+                        failed += 1
+                    else:
+                        text = "\n\n".join(lines)
+                        # Translate using the shared core.translation helper.
+                        # Import inside the function to avoid triggering the
+                        # module-level deep_translator dependency.
+                        try:
+                            from core.translation import _translate_text as _tt
+
+                            translated = _tt(text, target=lang)
+                        except ImportError:
+                            translated = text
+                        if not translated:
+                            failed += 1
+                        else:
+                            write_chapter(
+                                qualified_slug,
+                                ch["title"],
+                                translated,
+                                lang=lang,
+                                base_dir=base_dir,
+                            )
+                            saved += 1
+                else:
+                    lines = await source.read_chapter(ch["url"])
+                    if not lines:
+                        failed += 1
+                    else:
+                        text = "\n\n".join(lines)
+                        write_chapter(
+                            qualified_slug,
+                            ch["title"],
+                            text,
+                            lang=None,
+                            base_dir=base_dir,
+                        )
+                        saved += 1
+            except Exception:
+                failed += 1
+        done += 1
+        if progress_cb is not None:
+            progress_cb(done, saved)
+
+    await asyncio.gather(*(_process(ch) for ch in chapters))
+
+    # ---- completion: write meta.json + cover ----
+    # Build meta
+    try:
+        cover = await source.cover_url(
+            qualified_slug.split(":", 1)[-1]
+            if ":" in qualified_slug
+            else qualified_slug
+        )
+    except Exception:
+        cover = ""
+    meta = {"title": title, "cover": cover}
+    if translate and lang:
+        meta["lang"] = lang
+    meta["chapters"] = total
+    # Merge with existing meta (preserves 'tracked' from prior tracking)
+    existing_meta = read_meta(qualified_slug, base_dir)
+    existing_meta.update(meta)
+    write_meta(qualified_slug, existing_meta, base_dir)
+
+    return saved, failed

core/epub.py

@@ -0,0 +1,143 @@
+import asyncio
+import html as _html
+import os
+import xml.etree.ElementTree as ET
+import zipfile
+
+from ebooklib import epub
+
+from core.http_client import get_client
+from core.library import _chapter_sort_key
+
+
+def _validate_epub(path):
+    """Validate EPUB structure. Returns (ok, error_message)."""
+    if not os.path.isfile(path) or os.path.getsize(path) == 0:
+        return False, "EPUB file is empty or missing"
+
+    try:
+        zf = zipfile.ZipFile(path)
+    except zipfile.BadZipFile:
+        return False, "EPUB is not a valid ZIP archive"
+
+    with zf:
+        names = zf.namelist()
+
+        opf_name = next((n for n in names if n.endswith(".opf")), None)
+        if not opf_name:
+            return False, "EPUB missing OPF manifest"
+
+        try:
+            opf_xml = zf.read(opf_name)
+            root = ET.fromstring(opf_xml)
+            ns = {"opf": "http://www.idpf.org/2007/opf"}
+            spine = root.find(".//opf:spine", ns)
+            if spine is None:
+                return False, "EPUB has no spine element"
+            itemrefs = spine.findall("opf:itemref", ns)
+            if not itemrefs:
+                return False, "EPUB spine has no entries"
+        except Exception as e:
+            return False, f"Failed to parse OPF: {e}"
+
+        manifest = root.find(".//opf:manifest", ns)
+        for itemref in itemrefs:
+            idref = itemref.get("idref")
+            item = (
+                manifest.find(f"opf:item[@id='{idref}']", ns)
+                if manifest is not None
+                else None
+            )
+            if item is not None:
+                href = item.get("href")
+                if href and href in names:
+                    data = zf.read(href)
+                    if len(data) < 10:
+                        return False, f"Chapter '{href}' is empty or too small"
+
+    return True, ""
+
+
+async def _export_epub(slug, source=None, chapters=None):
+    chap_dir = os.path.join("novels", slug)
+    raw_slug = slug.split(":", 1)[-1] if ":" in slug else slug
+    title_parts = raw_slug.split("/", 1)
+    title_stem = title_parts[-1] if len(title_parts) > 1 else title_parts[0]
+    title = title_stem.replace("-", " ").title()
+
+    book = epub.EpubBook()
+    book.set_identifier(slug.replace("/", "-").replace(":", "-"))
+    book.set_title(title)
+    book.set_language("en")
+
+    # Cover
+    cover_data = None
+    author = "Unknown"
+    if source:
+        try:
+            url = await source.cover_url(raw_slug)
+            if url:
+                c = get_client()
+                try:
+                    r = await c.get(url)
+                    if r.status_code == 200:
+                        cover_data = r.content
+                finally:
+                    await c.aclose()
+        except Exception:
+            pass
+    book.add_author(author)
+    if cover_data:
+        book.set_cover("cover-image.jpg", cover_data)
+
+    # Chapters: either in-memory list or read from disk
+    if chapters is not None:
+        txt_list = list(chapters)
+    else:
+        if not os.path.isdir(chap_dir):
+            return None
+        txt_files = []
+        for root, _dirs, files in os.walk(chap_dir):
+            for f in sorted(files):
+                if f.endswith(".txt"):
+                    rel = os.path.relpath(os.path.join(root, f), chap_dir)
+                    txt_files.append(rel)
+        txt_files.sort(key=_chapter_sort_key)
+        txt_list = []
+        for fname in txt_files:
+            with open(os.path.join(chap_dir, fname), encoding="utf-8") as f:
+                content = f.read()
+            txt_list.append((fname, content))
+
+    if not txt_list:
+        return None
+
+    epub_chapters = []
+    for i, (fname, content) in enumerate(txt_list):
+        ch_title = (
+            os.path.splitext(os.path.basename(fname))[0].replace("_", " ").title()
+        )
+        paragraphs = content.split("\n\n")
+        parts = [f"<p>{_html.escape(p.strip())}</p>" for p in paragraphs if p.strip()]
+        html = f"<h1>{_html.escape(ch_title)}</h1>" + "".join(parts)
+        ch = epub.EpubHtml(
+            title=ch_title, file_name=f"chap_{i + 1:04d}.xhtml", lang="en"
+        )
+        ch.content = html
+        book.add_item(ch)
+        epub_chapters.append(ch)
+
+    book.toc = epub_chapters
+    book.spine = ["nav"] + epub_chapters
+    book.add_item(epub.EpubNcx())
+    book.add_item(epub.EpubNav())
+
+    safe = title.replace(" ", "_").replace("/", "-")
+    out = os.path.join(chap_dir, f"{safe}.epub")
+    os.makedirs(os.path.dirname(out) or ".", exist_ok=True)
+    await asyncio.to_thread(epub.write_epub, out, book, {})
+    ok, err = _validate_epub(out)
+    if not ok:
+        os.remove(out)
+        return None
+    return out

core/http_client.py

@@ -0,0 +1,262 @@
+"""Shared httpx.AsyncClient factory with Android SSL workarounds.
+
+On Android, Python's ssl module cannot reach the system certificate store,
+so httpx (which uses certifi or the default SSL context) fails TLS
+handshakes.  This module detects Android at import time and builds an
+AsyncClient that disables certificate verification there, while keeping it
+enabled on every other platform.
+
+Every module that needs an HTTP client should call ``get_client()`` (or
+``get_client_with_headers()`` for per-source custom headers) instead of
+creating its own ``httpx.AsyncClient``.
+
+This module also owns the two helpers every source shares: HTML parsing
+(``parse_html``/``fetch_soup``) and user-facing error wording
+(``describe_error``).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import ssl
+from typing import Any
+
+import httpx
+from bs4 import BeautifulSoup
+
+
+def _is_android() -> bool:
+    try:
+        from kivy.utils import platform
+
+        return bool(platform == "android")
+    except Exception:
+        return False
+
+
+_ANDROID = _is_android()
+
+_DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+}
+
+
+def _make_ssl_context() -> ssl.SSLContext | None:
+    """Return an SSL context suitable for the current platform.
+
+    On Android the system CA bundle is not reachable from Python, so we
+    create a permissive context that skips certificate verification.
+    On desktop platforms we return None so httpx uses its own default
+    (which verifies certificates via certifi).
+    """
+    if not _ANDROID:
+        return None
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
+_ssl_ctx: ssl.SSLContext | None = _make_ssl_context()
+
+
+def get_client(
+    *,
+    headers: dict[str, str] | None = None,
+    timeout: int = 30,
+    follow_redirects: bool = True,
+) -> httpx.AsyncClient:
+    """Create an ``httpx.AsyncClient`` with Android-aware SSL settings.
+
+    Parameters
+    ----------
+    headers:
+        Extra headers merged on top of the sensible defaults.
+    timeout:
+        Request timeout in seconds.
+    follow_redirects:
+        Whether to follow HTTP redirects.
+    """
+    merged = dict(_DEFAULT_HEADERS)
+    if headers:
+        merged.update(headers)
+    return httpx.AsyncClient(
+        headers=merged,
+        follow_redirects=follow_redirects,
+        timeout=timeout,
+        # On Android the system CA store is unreachable; disable
+        # verification so TLS handshakes succeed.  On desktop the
+        # ssl_ctx is None and httpx verifies via certifi as usual.
+        verify=_ssl_ctx if _ssl_ctx is not None else True,
+    )
+
+
+def get_client_with_headers(
+    headers: dict[str, str],
+    *,
+    timeout: int = 30,
+    follow_redirects: bool = True,
+) -> httpx.AsyncClient:
+    """Convenience wrapper for sources that supply their own headers.
+
+    The provided *headers* override the defaults (e.g. a custom Referer).
+    """
+    return get_client(
+        headers=headers, timeout=timeout, follow_redirects=follow_redirects
+    )
+
+
+# --------------------------------------------------------------------------
+# HTML parsing
+# --------------------------------------------------------------------------
+
+
+def _detect_parser() -> str:
+    """Return the fastest bs4 tree builder available on this machine.
+
+    lxml parses roughly 1.6x faster than the pure-Python html.parser and
+    releases the GIL while doing so, but it is a compiled extension that the
+    Android APK does not bundle (see buildozer.spec).  Probe bs4 itself
+    rather than ``import lxml`` so a broken native library also falls back
+    instead of crashing on the first fetch.
+    """
+    for name in ("lxml", "html.parser"):
+        try:
+            BeautifulSoup("", name)
+        except Exception:
+            continue
+        return name
+    return "html.parser"
+
+
+HTML_PARSER: str = _detect_parser()
+
+
+async def parse_html(markup: str) -> BeautifulSoup:
+    """Parse HTML off the event loop and return a ``BeautifulSoup``."""
+    return await asyncio.to_thread(BeautifulSoup, markup, HTML_PARSER)
+
+
+# Transient statuses worth retrying.  404/403 and friends are deliberately
+# absent: retrying a removed novel or an anti-bot block only wastes the
+# user's time and makes the rate limiting worse.
+_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
+_MAX_ATTEMPTS = 3
+# Cap on a server-supplied Retry-After.  The GUI wraps most of these calls in
+# async_loop.run(..., timeout=30), so a large value would eat the whole budget
+# and surface as a timeout instead of a rate-limit message.
+_MAX_RETRY_AFTER = 8.0
+# Used when the server sends no usable Retry-After.
+_RETRY_BACKOFF = (1.0, 2.0)
+
+
+async def _sleep(seconds: float) -> None:
+    """Pause between retries.  Tests monkeypatch this to stay instant."""
+    await asyncio.sleep(seconds)
+
+
+def _retry_delay(response: httpx.Response, attempt: int) -> float:
+    """Seconds to wait before retrying *response* on attempt *attempt*.
+
+    A sane server-supplied ``Retry-After`` wins; anything larger is clamped to
+    the cap rather than honoured, and a missing or non-numeric one falls back
+    to exponential backoff.
+    """
+    try:
+        seconds = float(response.headers.get("Retry-After", ""))
+    except (TypeError, ValueError):
+        seconds = 0.0
+    if seconds > 0:
+        return min(seconds, _MAX_RETRY_AFTER)
+    index = attempt - 1
+    return _RETRY_BACKOFF[index] if index < len(_RETRY_BACKOFF) else _RETRY_BACKOFF[-1]
+
+
+async def _get_with_retry(
+    client: httpx.AsyncClient,
+    url: str,
+    *,
+    params: dict[str, Any] | None = None,
+    headers: dict[str, str] | None = None,
+) -> httpx.Response:
+    """GET *url*, retrying transient statuses, and return the response.
+
+    Raises via ``response.raise_for_status()`` for whatever is still non-2xx
+    once the attempts are spent.
+    """
+    response = await client.get(url, params=params, headers=headers)
+    for attempt in range(1, _MAX_ATTEMPTS):
+        if response.status_code not in _RETRY_STATUSES:
+            break
+        await _sleep(_retry_delay(response, attempt))
+        response = await client.get(url, params=params, headers=headers)
+    response.raise_for_status()
+    return response
+
+
+async def fetch_soup(
+    client: httpx.AsyncClient,
+    url: str,
+    *,
+    params: dict[str, Any] | None = None,
+    headers: dict[str, str] | None = None,
+) -> BeautifulSoup:
+    """GET *url*, raise on a non-2xx status, and parse the body off-loop.
+
+    Transient statuses are retried with backoff: novelfire rate-limits bursty
+    chapter paging, and without this every 429/500 became a dead end for the
+    caller.  Non-transient failures still raise on the first attempt.
+    """
+    response = await _get_with_retry(client, url, params=params, headers=headers)
+    return await parse_html(response.text)
+
+
+# --------------------------------------------------------------------------
+# User-facing error wording
+# --------------------------------------------------------------------------
+
+
+def _status_code(error: BaseException) -> int | None:
+    """Best-effort status code from an HTTP exception.
+
+    Works for ``httpx.HTTPStatusError`` and, because it reads ``.response``
+    structurally, for ``curl_cffi.requests.exceptions.HTTPError`` too — so
+    this module never has to import curl_cffi (which Android lacks).
+    """
+    response = getattr(error, "response", None)
+    code = getattr(response, "status_code", None)
+    return code if isinstance(code, int) else None
+
+
+def describe_error(error: BaseException, action: str) -> str:
+    """Turn an exception into a short user-facing message.
+
+    *action* is what the caller was doing, e.g. ``"Search failed"`` or
+    ``"Failed to fetch chapters"``.  Only genuine network failures are
+    described as a connection problem; anything else is named instead, so
+    a bug in the app never masquerades as "check your connection".
+    """
+    status = _status_code(error)
+    if status is not None:
+        if status == 404:
+            detail = "page not found (404)"
+        elif status == 403:
+            detail = "access denied (403)"
+        elif status == 429:
+            detail = "rate limited (429) — wait a moment and retry"
+        elif 500 <= status < 600:
+            detail = f"server error ({status}) — try again later"
+        else:
+            detail = f"HTTP {status}"
+        return f"{action} — {detail}."
+    if isinstance(error, httpx.TimeoutException):
+        return f"{action} — timed out. Check your connection."
+    if isinstance(error, httpx.TransportError):
+        return f"{action} — could not reach the server."
+    return f"{action} — unexpected error ({type(error).__name__})."