eaf-base-api 4.2.2__tar.gz → 4.2.3__tar.gz

{eaf_base_api-4.2.2 → eaf_base_api-4.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eaf_base_api
-Version: 4.2.2
+Version: 4.2.3
 Summary: A base API for EchterAlsFake's Porn APIs
 Author: Johannes Habel
 Author-email: Johannes Habel <EchterAlsFake@proton.me>

{eaf_base_api-4.2.2 → eaf_base_api-4.2.3}/base_api/base.py

@@ -3068,259 +3068,6 @@ class BaseCore:
                 raise DownloadFailed(f"Raw download failed for {url}: {error}", url=url) from error
 
     async def _legacy_download(self, url: str, configuration: DownloadConfigRAW) -> bool:
-        """
-        Download a file using streaming with stall tolerance and resume.
-        Supports fast concurrent range downloading if the server supports it and allow_multipart is True.
-        Assumes self.session is an AsyncSession.
-        """
-        path = configuration.path
-        max_retries = configuration.max_retries
-        read_timeout = configuration.read_timeout
-        stop_event = configuration.stop_event
-        allow_multipart = configuration.allow_multipart
-        callback = configuration.callback
-        chunk_size = configuration.chunk_size
-        max_workers = configuration.max_workers
-
-        self.logger.info(
-"""Legacy download start: url=%s path=%s
-max_retries=%s read_timeout=%s
-stop_event_set=%s
-allow_multipart=%s""", url, path, max_retries, read_timeout, bool(stop_event and stop_event.is_set()),
-        allow_multipart)
-
-        if stop_event is not None and stop_event.is_set():
-            self.logger.warning("Stop event already set; cancelling legacy download.")
-            raise DownloadCancelled("Download cancelled.")
-
-        # Ensure session is initialized
-        if self.session is None:
-            self.initialize_session()
-        session = self.session
-        assert session is not None
-
-        progress_bar = None
-        if callback is None:
-            progress_bar = Callback()
-            self.logger.debug("legacy_download: no callback provided, using default progress bar")
-
-        timeout = read_timeout
-
-        # 1. Check if the server supports Range requests and get file size (if multipart is allowed)
-        file_size = 0
-        accept_ranges = ""
-
-        if allow_multipart:
-            # We MUST request uncompressed content for range downloads, otherwise:
-            # 1) Content-Length from HEAD reflects the compressed size, not the real file size.
-            # 2) Mid-file Range requests on compressed streams cause libcurl error 61
-            #    ("incorrect header check") because partial gzip lacks a valid header.
-            no_compress = {"Accept-Encoding": "identity"}
-            try:
-                head_resp = await session.head(url, timeout=timeout, allow_redirects=True, headers=no_compress)
-                if getattr(head_resp, "url", None):
-                    url = str(head_resp.url)
-                if head_resp.status_code == 405:  # Method Not Allowed, fallback to streaming GET
-                    head_resp_stream = await session.request("GET", url, timeout=timeout, allow_redirects=True,
-                                                             stream=True, headers=no_compress)
-                    if getattr(head_resp_stream, "url", None):
-                        url = str(head_resp_stream.url)
-                    file_size = int(head_resp_stream.headers.get("Content-Length", 0))
-                    accept_ranges = head_resp_stream.headers.get("Accept-Ranges", "")
-                else:
-                    file_size = int(head_resp.headers.get("Content-Length", 0))
-                    accept_ranges = head_resp.headers.get("Accept-Ranges", "")
-            except Exception as e:
-                self.logger.warning("HEAD request failed for %s; falling back to streaming: %s", url, e, exc_info=True)
-
-        # 2. Execute Fast Multipart Download if supported and allowed
-        if allow_multipart and file_size > 0 and accept_ranges == "bytes":
-            self.logger.info("Server supports Range requests. Starting fast multipart download"
-                             "or %s bytes.", file_size)
-
-            # Pre-allocate file
-            def allocate_file() -> None:
-                if not os.path.exists(path):
-                    with open(path, "wb") as file_alloc:
-                        file_alloc.truncate(file_size)
-                elif os.path.getsize(path) != file_size:
-                    # File exists but size mismatch, truncate to correct size
-                    with open(path, "r+b") as file_alloc_size:
-                        file_alloc_size.truncate(file_size)
-            await asyncio.to_thread(allocate_file)
-
-            # We will use an array to track progress of chunks
-            # A chunk map: {chunk_index: bytes_downloaded}
-            chunk_progress = {}
-            total_downloaded = [0]  # List to allow modification in inner func
-            # Determine chunk sizes based on file size, but keep reasonable bounds
-            # For massive files, don't create 10,000 workers.
-            target_chunk_size = max(chunk_size, min(10 * 1024 * 1024, file_size // 10)) # Between 1MB and 10MB
-
-            semaphore = asyncio.Semaphore(max_workers)
-
-            async def download_chunk(start_chunk: int, end_chunk: int, chunk_idx_now: int) -> bool:
-                nonlocal total_downloaded
-                headers_chunk = {"Range": f"bytes={start_chunk}-{end_chunk}", "Accept-Encoding": "identity"}
-                chunk_progress[chunk_idx_now] = 0
-
-                for attempt_chunk in range(max_retries + 1):
-                    if stop_event is not None and stop_event.is_set():
-                        return False
-
-                    try:
-                        async with semaphore:
-                            resp = await cast(Any, session).request(
-                                "GET", url, headers=headers_chunk, timeout=timeout, allow_redirects=True, stream=True
-                            )
-                            resp.raise_for_status()
-
-                            # Open file once for this chunk download attempt
-                            file = await asyncio.to_thread(lambda: open(path, "rb+"))
-                            try:
-                                await asyncio.to_thread(file.seek, start_chunk + chunk_progress[chunk_idx_now])
-                                async for data in resp.aiter_content():
-                                    if stop_event is not None and stop_event.is_set():
-                                        return False
-
-                                    await asyncio.to_thread(cast(Any, file).write, data)
-
-                                    data_len = len(data)
-                                    chunk_progress[chunk_idx_now] += data_len
-                                    total_downloaded[0] += data_len
-
-                                    if callback:
-                                        callback(total_downloaded[0], file_size)
-                                    elif progress_bar:
-                                        progress_bar.text_progress_bar(downloaded=total_downloaded[0], total=file_size)
-                            finally:
-                                await asyncio.to_thread(file.close)
-
-                            return True # Chunk success
-
-                    except Exception as exc:
-                        if attempt_chunk < max_retries:
-                            self.logger.warning("Chunk %s failed for %s (attempt %s/%s): %s",
-                                                chunk_idx_now, url, attempt_chunk + 1, max_retries, exc, exc_info=True)
-                            # Reset progress for this chunk before retry
-                            total_downloaded[0] -= chunk_progress[chunk_idx_now]
-                            chunk_progress[chunk_idx_now] = 0
-                            await asyncio.sleep(1 * attempt_chunk)
-                        else:
-                            self.logger.error("Chunk %s permanently failed for %s: %s", chunk_idx_now, url, exc, exc_info=True)
-                            return False
-                return False
-
-            tasks = []
-            chunk_idx = 0
-            for start in range(0, file_size, target_chunk_size):
-                end = min(start + target_chunk_size - 1, file_size - 1)
-                tasks.append(download_chunk(start, end, chunk_idx))
-                chunk_idx += 1
-
-            results = await asyncio.gather(*tasks)
-
-            if progress_bar:
-                # We set it to None instead of del to avoid analyzer confusion about potential unassigned reference
-                progress_bar = None
-
-            if stop_event is not None and stop_event.is_set():
-                raise DownloadCancelled("Download cancelled.")
-
-            if not all(results):
-                raise NetworkRequestError("One or more chunks failed to download completely.")
-
-            self.logger.info("Fast multipart download complete: path=%s", path)
-            return True
-
-        # 3. Fallback to standard linear streaming download
-        if not allow_multipart:
-            self.logger.info("allow_multipart=False. Forcing linear streaming download.")
-        else:
-            self.logger.info("Server does not support Range requests or size is 0. Falling back to linear streaming.")
-
-        downloaded_so_far = 0
-        attempt = 0
-        etag = None
-
-        while True:
-            if stop_event is not None and stop_event.is_set():
-                self.logger.warning("Stop event set; cancelling legacy download.")
-                raise DownloadCancelled("Download cancelled.")
-            headers = {}
-            if downloaded_so_far:
-                headers["Range"] = f"bytes={downloaded_so_far}-"
-
-            try:
-                response = await cast(Any, session).request(
-                    "GET", url, headers=headers, allow_redirects=True, timeout=timeout, stream=True
-                )
-                if downloaded_so_far and response.status_code == 200:
-                    self.logger.warning("Server ignored Range request; restarting download from scratch.")
-                    downloaded_so_far = 0
-                response.raise_for_status()
-
-                etag_cur = response.headers.get("ETag")
-                if etag is None:
-                    etag = etag_cur
-                elif etag_cur and etag_cur != etag:
-                    raise RuntimeError("Remote content changed during download")
-
-                total = None
-                cr = response.headers.get("Content-Range")
-                if cr and "/" in cr:
-                    try: total = int(cr.rsplit("/", 1)[1])
-                    except ValueError: pass
-                if total is None:
-                    try: total = int(response.headers.get("Content-Length", "0")) or None
-                    except ValueError: pass
-
-                # Fix fallback if total size is still missing
-                if total is None:
-                    total = 0
-
-                mode = "ab" if downloaded_so_far else "wb"
-                f = await asyncio.to_thread(cast(Any, open), path, mode)
-                try:
-                    await asyncio.to_thread(f.seek, 0, 2)  # Move to EOF
-                    async for chunk in response.aiter_content():
-                        if stop_event is not None and stop_event.is_set():
-                            raise DownloadCancelled("Download cancelled.")
-                        if not chunk:
-                            continue
-                        await asyncio.to_thread(f.write, chunk)
-                        downloaded_so_far += len(chunk)
-
-                        if callback:
-                            callback(downloaded_so_far, total)
-                        elif progress_bar:
-                            progress_bar.text_progress_bar(downloaded=downloaded_so_far, total=total)
-                finally:
-                    await asyncio.to_thread(f.close)
-
-                if progress_bar:
-                    progress_bar = None
-                self.logger.info("Legacy download complete: bytes=%s path=%s", downloaded_so_far, path)
-                return True
-            except RequestsError as e:
-                err_str = str(e).lower()
-                if "timeout" in err_str or "read" in err_str or "reset" in err_str or "recv failure" in err_str or "connection" in err_str:
-                    attempt += 1
-                    if attempt > max_retries:
-                        raise
-                    backoff = min(2 ** attempt, 30)
-                    self.logger.warning("Download connection interrupted for %s (%s); retrying %s/%s in %ss", url, e, attempt, max_retries, backoff, exc_info=True)
-                    if stop_event is not None and stop_event.wait(backoff):
-                        raise DownloadCancelled("Download cancelled.") from e
-                    else:
-                        await asyncio.sleep(backoff)
-                    continue
-                else:
-                    raise NetworkRequestError(f"Stream for: {url} was closed or failed: {e}") from e
-            except DownloadCancelled:
-                raise
-            except Exception as exc:
-                error = traceback.format_exc()
-                raise NetworkRequestError(f"Unknown error for: {url} -->: {error}") from exc
+        from base_api.modules.raw_download import download_raw
 
-        return False
+        return await download_raw(self, url, configuration)

{eaf_base_api-4.2.2 → eaf_base_api-4.2.3}/base_api/modules/config.py

@@ -84,6 +84,7 @@ class DownloadConfigRAW(BaseConfigDownload):
     read_timeout: float = 120.0
     chunk_size: int = 1024
     max_retries: int = 5
+    resume_state_path: str | None = None
 
 
 @dataclass

eaf_base_api-4.2.3/base_api/modules/raw_download.py

@@ -0,0 +1,305 @@
+"""Resumable RAW transfers for BaseCore, including persisted multipart ranges."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import math
+import os
+import re
+from pathlib import Path
+
+from .errors import DownloadCancelled, NetworkRequestError
+from .progress_bars import Callback
+
+_RANGE = re.compile(r"bytes (\d+)-(\d+)/(\d+)")
+
+
+class _Checkpoint:
+    def __init__(self, path, source_url, output, mode, size, chunk_size, headers):
+        self.path = Path(path)
+        self.lock = asyncio.Lock()
+        self.restored = False
+        count = math.ceil(size / chunk_size) if mode == "multipart" else 1
+        self.data = {
+            "version": 1, "source": hashlib.sha256(source_url.encode()).hexdigest(),
+            "output": str(Path(output).resolve()), "mode": mode, "size": size,
+            "chunk_size": chunk_size, "progress": [0] * count,
+            "etag": headers.get("ETag"), "last_modified": headers.get("Last-Modified"),
+        }
+        try:
+            saved = json.loads(self.path.read_text())
+            if not isinstance(saved, dict):
+                return
+            if any(saved.get(key) != self.data[key] for key in ("version", "source", "output", "mode", "size")):
+                return
+            for name in ("etag", "last_modified"):
+                if saved.get(name) and saved[name] != self.data[name]:
+                    return
+            width = saved["chunk_size"]
+            progress = saved["progress"]
+            if mode == "multipart":
+                if not isinstance(width, int) or width <= 0 or len(progress) != math.ceil(size / width):
+                    return
+                limits = [min(width, size - index * width) for index in range(len(progress))]
+            else:
+                if len(progress) != 1:
+                    return
+                limits = [size or Path(output).stat().st_size]
+            if any(not isinstance(value, int) or not 0 <= value <= limit
+                   for value, limit in zip(progress, limits)):
+                return
+            length = Path(output).stat().st_size
+            if (mode == "multipart" and length != size) or (mode == "linear" and length < progress[0]):
+                return
+            self.data = saved
+            self.restored = True
+        except (OSError, ValueError, KeyError, TypeError, ZeroDivisionError):
+            pass
+
+    async def save(self, progress):
+        async with self.lock:
+            payload = dict(self.data, progress=list(progress))
+            await asyncio.to_thread(self._write, payload)
+
+    def _write(self, payload):
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = self.path.with_name(self.path.name + ".tmp")
+        temporary.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
+        os.replace(temporary, self.path)
+
+    def remove(self):
+        self.path.unlink(missing_ok=True)
+        self.path.with_name(self.path.name + ".tmp").unlink(missing_ok=True)
+
+
+def _stopped(event):
+    return event is not None and event.is_set()
+
+
+async def _retry_delay(event, seconds):
+    if event is None:
+        await asyncio.sleep(seconds)
+        return
+    try:
+        await asyncio.wait_for(event.wait(), seconds)
+    except TimeoutError:
+        return
+    raise DownloadCancelled("Download cancelled.")
+
+
+async def _close_response(response):
+    if response is None:
+        return
+    # curl-cffi's aclose() waits for the streaming task; signal its supported
+    # quit event first so an interrupted response stops receiving its body.
+    quit_event = getattr(response, "quit_now", None)
+    if quit_event is not None:
+        quit_event.set()
+    await response.aclose()
+
+
+def _range(response, start, end=None, total=None):
+    match = _RANGE.fullmatch(response.headers.get("Content-Range", ""))
+    if (response.status_code != 206 or match is None or int(match[1]) != start
+            or not start <= int(match[2]) < int(match[3])
+            or (end is not None and int(match[2]) != end)
+            or (total is not None and int(match[3]) != total)):
+        raise NetworkRequestError("Server did not return the requested byte range")
+    return int(match[3])
+
+
+async def download_raw(core, url, configuration):
+    """Download through the core's real session; checkpoints never infer sparse-file progress."""
+    output = Path(configuration.path)
+    checkpoint_path = configuration.resume_state_path or f"{output}.raw-state.json"
+    event = configuration.stop_event
+    if _stopped(event):
+        raise DownloadCancelled("Download cancelled.")
+    core.initialize_session()
+    session = core.session
+    output.parent.mkdir(parents=True, exist_ok=True)
+    original_url = url
+    size, ranges, headers = 0, False, {}
+    response = None
+    try:
+        response = await session.head(url, timeout=configuration.read_timeout,
+                                      headers={"Accept-Encoding": "identity"}, allow_redirects=True)
+        if response.status_code == 405:
+            await _close_response(response)
+            response = await session.request("GET", url, stream=True, timeout=configuration.read_timeout,
+                                             headers={"Accept-Encoding": "identity"}, allow_redirects=True)
+        response.raise_for_status()
+        headers = {name: response.headers.get(name) for name in ("ETag", "Last-Modified")}
+        size = int(response.headers.get("Content-Length", 0))
+        ranges = response.headers.get("Accept-Ranges", "").lower() == "bytes"
+        if response.url:
+            url = str(response.url)
+    except DownloadCancelled:
+        raise
+    except Exception:
+        # An outage must not turn a persisted partial download into a fresh file.
+        if Path(checkpoint_path).exists():
+            raise
+        core.logger.warning("RAW metadata unavailable; falling back to linear streaming", exc_info=True)
+    finally:
+        await _close_response(response)
+    mode = "multipart" if configuration.allow_multipart and ranges and size > 0 else "linear"
+    width = max(configuration.chunk_size, min(10 * 1024 * 1024, max(1, size // 10)))
+    state = _Checkpoint(checkpoint_path, original_url, output, mode, size, width, headers)
+    progress = list(state.data["progress"])
+    if not state.restored:
+        # Create the zero-progress checkpoint before preallocation. An existing
+        # full-size sparse file without a valid checkpoint is never considered complete.
+        await state.save(progress)
+        def allocate():
+            with output.open("wb") as file:
+                if mode == "multipart":
+                    file.truncate(size)
+        await asyncio.to_thread(allocate)
+    elif mode == "linear":
+        with output.open("r+b") as file:
+            file.truncate(progress[0])
+    core.logger.info("RAW %s: mode=%s retained_bytes=%s total=%s", output, mode, sum(progress), size)
+    bar = Callback() if configuration.callback is None else None
+
+    def notify(position, total):
+        if configuration.callback:
+            configuration.callback(position, total)
+        elif bar:
+            bar.text_progress_bar(downloaded=position, total=total)
+
+    if sum(progress):
+        notify(sum(progress), size)
+    validator = state.data.get("etag")
+    if not validator or validator.startswith("W/"):
+        validator = state.data.get("last_modified")
+
+    if mode == "multipart":
+        width = state.data["chunk_size"]
+        durable = list(progress)
+        semaphore = asyncio.Semaphore(max(1, configuration.max_workers))
+
+        async def chunk(index):
+            start = index * width
+            end = min(start + width, size) - 1
+            for attempt in range(configuration.max_retries + 1):
+                if _stopped(event):
+                    return False
+                offset = start + progress[index]
+                if offset > end:
+                    return True
+                response = None
+                try:
+                    async with semaphore:
+                        # Workers queued before a stop must not issue requests afterwards.
+                        if _stopped(event):
+                            return False
+                        request_headers = {"Range": f"bytes={offset}-{end}", "Accept-Encoding": "identity"}
+                        if validator:
+                            request_headers["If-Range"] = validator
+                        response = await session.request("GET", url, headers=request_headers, stream=True,
+                                                         timeout=configuration.read_timeout, allow_redirects=True)
+                        response.raise_for_status()
+                        _range(response, offset, end, size)
+                        file = await asyncio.to_thread(output.open, "r+b")
+                        try:
+                            await asyncio.to_thread(file.seek, offset)
+                            async for data in response.aiter_content():
+                                if _stopped(event):
+                                    return False
+                                if not data:
+                                    continue
+                                if start + progress[index] + len(data) > end + 1:
+                                    raise NetworkRequestError("Server exceeded the requested byte range")
+                                await asyncio.to_thread(file.write, data)
+                                progress[index] += len(data)
+                                notify(sum(progress), size)
+                        finally:
+                            await asyncio.to_thread(file.close)
+                        if start + progress[index] != end + 1:
+                            raise NetworkRequestError("Byte range ended before all requested bytes arrived")
+                        # Only closed/flushed ranges enter checkpoints while other
+                        # workers are active. Partial ranges are saved after all exit.
+                        durable[index] = progress[index]
+                        await state.save(durable)
+                        return True
+                except DownloadCancelled:
+                    return False
+                except Exception:
+                    if attempt >= configuration.max_retries:
+                        raise
+                    core.logger.warning("Retrying RAW range %s at byte %s", index, start + progress[index],
+                                        exc_info=True)
+                finally:
+                    await _close_response(response)
+                await _retry_delay(event, min(2 ** attempt, 30))
+            return False
+
+        tasks = [asyncio.create_task(chunk(index)) for index in range(len(progress))]
+        try:
+            results = await asyncio.gather(*tasks)
+        finally:
+            for task in tasks:
+                if not task.done():
+                    task.cancel()
+            await asyncio.gather(*tasks, return_exceptions=True)
+            await state.save(progress)
+        if _stopped(event):
+            raise DownloadCancelled("Download cancelled.")
+        if not all(results) or sum(progress) != size:
+            raise NetworkRequestError("RAW download did not complete every byte range")
+    else:
+        for attempt in range(configuration.max_retries + 1):
+            if _stopped(event):
+                raise DownloadCancelled("Download cancelled.")
+            offset = progress[0]
+            if size and offset == size:
+                break
+            response = None
+            request_headers = {"Accept-Encoding": "identity"}
+            if offset:
+                request_headers["Range"] = f"bytes={offset}-"
+                if validator:
+                    request_headers["If-Range"] = validator
+            try:
+                response = await session.request("GET", url, headers=request_headers, stream=True,
+                                                 timeout=configuration.read_timeout, allow_redirects=True)
+                response.raise_for_status()
+                if offset and response.status_code == 200:
+                    core.logger.warning("Server ignored RAW resume range; restarting from byte zero")
+                    progress[0] = 0
+                elif offset:
+                    size = _range(response, offset, total=size or None)
+                else:
+                    size = int(response.headers.get("Content-Length", 0))
+                state.data["size"] = size
+                file = await asyncio.to_thread(output.open, "ab" if progress[0] else "wb")
+                try:
+                    async for data in response.aiter_content():
+                        if _stopped(event):
+                            raise DownloadCancelled("Download cancelled.")
+                        if data:
+                            await asyncio.to_thread(file.write, data)
+                            progress[0] += len(data)
+                            notify(progress[0], size)
+                finally:
+                    await asyncio.to_thread(file.close)
+                    await state.save(progress)
+                if size and progress[0] != size:
+                    raise NetworkRequestError("RAW stream ended before its reported length")
+                break
+            except DownloadCancelled:
+                raise
+            except Exception:
+                if attempt >= configuration.max_retries:
+                    raise
+                core.logger.warning("Retrying RAW stream from byte %s", progress[0], exc_info=True)
+            finally:
+                await _close_response(response)
+            await _retry_delay(event, min(2 ** attempt, 30))
+    if _stopped(event):
+        raise DownloadCancelled("Download cancelled.")
+    state.remove()
+    core.logger.info("RAW download complete: path=%s bytes=%s", output, sum(progress))
+    return True

{eaf_base_api-4.2.2 → eaf_base_api-4.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "eaf_base_api"
-version = "4.2.2"
+version = "4.2.3"
 description = "A base API for EchterAlsFake's Porn APIs"
 requires-python = ">=3.12"
 license = "AGPL-3.0-or-later"

{eaf_base_api-4.2.2 → eaf_base_api-4.2.3}/pyproject.toml.orig

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "eaf_base_api"
-version = "4.2.2"
+version = "4.2.3"
 description = "A base API for EchterAlsFake's Porn APIs"
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.12"