eaf_base_api 4.0.0__tar.gz → 4.1.0__tar.gz

{eaf_base_api-4.0.0 → eaf_base_api-4.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eaf_base_api
-Version: 4.0.0
+Version: 4.1.0
 Summary: A base API for EchterAlsFake's Porn APIs
 Author: Johannes Habel
 Author-email: Johannes Habel <EchterAlsFake@proton.me>
@@ -136,15 +136,18 @@ extractor order is available with `ResultOrder.ORIGINAL`.
 
 ```python
 from base_api import Helper, ResultOrder
+from base_api.modules.config import IteratorConfig
 
 helper = Helper(core=core, constructor=Video)
 stream = helper.iterator(
     page_urls,
     extractor_videos,
-    max_page_concurrency=3,
-    max_item_concurrency=20,
-    load_fields=("title", "available_qualities"),
-    order=ResultOrder.COMPLETION,  # The default.
+    iterator_config=IteratorConfig(
+        max_page_concurrency=3,
+        max_item_concurrency=20,
+        load_specific_fields=("title", "available_qualities"),
+        order=ResultOrder.COMPLETION,  # The default.
+    ),
 )
 
 # The context manager guarantees immediate task cleanup if this loop breaks early.
@@ -156,11 +159,11 @@ async with stream:
         video = result.unwrap()
 ```
 
-Use `order=ResultOrder.ORIGINAL` when presentation order matters. Page and item
-failures independently support `ErrorMode.YIELD`, `ErrorMode.SKIP`, or
-`ErrorMode.RAISE`. `RetryPolicy` provides a strict maximum attempt count and
-optional exponential delay; error handlers return an `ErrorAction` and cannot
-create an unbounded retry loop.
+Use `IteratorConfig(order=ResultOrder.ORIGINAL)` when presentation order matters.
+Page and item failures independently support `ErrorMode.YIELD`, `ErrorMode.SKIP`,
+or `ErrorMode.RAISE`. `RetryPolicy` provides a strict maximum attempt count and
+optional exponential delay; the independent page and item handlers return an
+`ErrorAction` and cannot create an unbounded retry loop.
 
 # Can I use this for myself?
 Yes, you can, but I may change stuff here and there from time to time, and it would maybe break your project.

{eaf_base_api-4.0.0 → eaf_base_api-4.1.0}/README.md

@@ -116,15 +116,18 @@ extractor order is available with `ResultOrder.ORIGINAL`.
 
 ```python
 from base_api import Helper, ResultOrder
+from base_api.modules.config import IteratorConfig
 
 helper = Helper(core=core, constructor=Video)
 stream = helper.iterator(
     page_urls,
     extractor_videos,
-    max_page_concurrency=3,
-    max_item_concurrency=20,
-    load_fields=("title", "available_qualities"),
-    order=ResultOrder.COMPLETION,  # The default.
+    iterator_config=IteratorConfig(
+        max_page_concurrency=3,
+        max_item_concurrency=20,
+        load_specific_fields=("title", "available_qualities"),
+        order=ResultOrder.COMPLETION,  # The default.
+    ),
 )
 
 # The context manager guarantees immediate task cleanup if this loop breaks early.
@@ -136,11 +139,11 @@ async with stream:
         video = result.unwrap()
 ```
 
-Use `order=ResultOrder.ORIGINAL` when presentation order matters. Page and item
-failures independently support `ErrorMode.YIELD`, `ErrorMode.SKIP`, or
-`ErrorMode.RAISE`. `RetryPolicy` provides a strict maximum attempt count and
-optional exponential delay; error handlers return an `ErrorAction` and cannot
-create an unbounded retry loop.
+Use `IteratorConfig(order=ResultOrder.ORIGINAL)` when presentation order matters.
+Page and item failures independently support `ErrorMode.YIELD`, `ErrorMode.SKIP`,
+or `ErrorMode.RAISE`. `RetryPolicy` provides a strict maximum attempt count and
+optional exponential delay; the independent page and item handlers return an
+`ErrorAction` and cannot create an unbounded retry loop.
 
 # Can I use this for myself?
 Yes, you can, but I may change stuff here and there from time to time, and it would maybe break your project.

{eaf_base_api-4.0.0 → eaf_base_api-4.1.0}/base_api/base.py

@@ -5,7 +5,6 @@ import time
 import hashlib
 import string
 import shutil
-import random
 import asyncio
 import inspect
 import logging
@@ -14,7 +13,6 @@ import threading
 from collections import deque
 from collections.abc import Iterable, Mapping, Sequence
 from enum import StrEnum
-from functools import lru_cache
 from urllib.parse import urljoin
 from dataclasses import MISSING, dataclass, field, fields
 from curl_cffi import CurlOpt # Used for DNS over HTTPS
@@ -36,11 +34,9 @@ from base_api.modules.type_hints import (
 )
 from base_api.modules.static_functions import (
     load_segment_state, parse_retry_after, log_precondition_failed,
-    write_segment_state, build_segment_state, get_segment_index_width,
-    segment_file_path, is_video_playlist, height_from_variant,
-    pick_by_height, normalize_quality_value,
-    parse_challenge, other_challenge, least_factors,
-    collect_variants, pick_by_label
+    write_segment_state, build_segment_state, segment_file_path,
+    parse_challenge, other_challenge, least_factors, available_qualities,
+    choose_variant, collect_variants, get_segment_index_width
 )
 from base_api.modules.config import config, RuntimeConfig, DownloadConfigHLS, DownloadConfigRAW, IteratorConfig
 from base_api.modules.progress_bars import Callback
@@ -66,9 +62,6 @@ except (ModuleNotFoundError, ImportError):
     m3u8 = None  # type: ignore
 
 
-UA_DESKTOP_CHROME = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
-                     "Chrome/122.0.0.0 Safari/537.36")
-
 REGEX_CHALLENGE = re.compile(r'var p=(\d+); var s=(\d+);.*?(\d+):1;', re.DOTALL)
 
 
@@ -448,46 +441,47 @@ class BaseMedia:
         default_factory=dict, init=False, repr=False, compare=False
     )
 
-    def __getattribute__(self, name: str) -> Any:
-        """
-        Reject only the private unloaded sentinel, never a legitimate ``None``.
+    if not TYPE_CHECKING:
+        def __getattribute__(self, name: str) -> Any:
+            """
+            Reject only the private unloaded sentinel, never a legitimate ``None``.
 
-        Attribute access remains synchronous and therefore cannot initiate network
-        I/O.  The exception tells callers exactly which field and sources to pass
-        to ``load_fields`` or ``load_sources``.
-        """
-        value = object.__getattribute__(self, name)
-        if value is not _UNLOADED:
-            return value
+            Attribute access remains synchronous and therefore cannot initiate network
+            I/O.  The exception tells callers exactly which field and sources to pass
+            to ``load_fields`` or ``load_sources``.
+            """
+            value = object.__getattribute__(self, name)
+            if value is not _UNLOADED:
+                return value
 
-        # Because we override the __getattribute__ method, we need to call object.__getattribute__.
-        # If I'd use self.<something> we would get a recursion error as the function would call itself infinitely
+            # Because we override the __getattribute__ method, we need to call object.__getattribute__.
+            # If I'd use self.<something> we would get a recursion error as the function would call itself infinitely
 
-        model_type = type(self)
-        """
-        Explanation why model_type exists:
-        So, basically when we build the schema (down below) this takes some time. However, this function actually
-        caches the fully resolved dataclass. So if we are going over a thousand Video objects usually we would need
-        to reconstruct the thousand video objects, well, a thousand times.
-        
-        By giving the model_type with self, we can tell the cache: Yo, this is a Video object. See if you have
-        already processed this and if yes it will use this instead of processing it each time again.
-        Might only save a few milliseconds but I ain't Ubisoft XDDD
-        """
-        schema = _media_schema(model_type)
-        sources = schema.field_sources.get(name, ()) # E.g., ("api", "html") for name=title (PornHub API as an example)
-        url = object.__getattribute__(self, "url") # The actual URL to fetch e.g., https://example.com/video/id?=
-        all_source_errors = object.__getattribute__(self, "_source_errors")
-        relevant_errors = {
-            source: all_source_errors[source]
-            for source in sources
-            if source in all_source_errors
-        } # Basically just checks which sources had an error e.g., if html failed fetching this will return html
-        raise DataNotLoadedError(
-            model_type.__name__, name, url, sources, relevant_errors
-            # Raises a custom exception that tells you which attribute failed with its associated source and the error
-            # that happened along with the actual URL (so that I can replicate it when you report it)
-        )
+            model_type = type(self)
+            """
+            Explanation why model_type exists:
+            So, basically when we build the schema (down below) this takes some time. However, this function actually
+            caches the fully resolved dataclass. So if we are going over a thousand Video objects usually we would need
+            to reconstruct the thousand video objects, well, a thousand times.
+            
+            By giving the model_type with self, we can tell the cache: Yo, this is a Video object. See if you have
+            already processed this and if yes it will use this instead of processing it each time again.
+            Might only save a few milliseconds but I ain't Ubisoft XDDD
+            """
+            schema = _media_schema(model_type)
+            sources = schema.field_sources.get(name, ()) # E.g., ("api", "html") for name=title (PornHub API as an example)
+            url = object.__getattribute__(self, "url") # The actual URL to fetch e.g., https://example.com/video/id?=
+            all_source_errors = object.__getattribute__(self, "_source_errors")
+            relevant_errors = {
+                source: all_source_errors[source]
+                for source in sources
+                if source in all_source_errors
+            } # Basically just checks which sources had an error e.g., if html failed fetching this will return html
+            raise DataNotLoadedError(
+                model_type.__name__, name, url, sources, relevant_errors
+                # Raises a custom exception that tells you which attribute failed with its associated source and the error
+                # that happened along with the actual URL (so that I can replicate it when you report it)
+            )
 
     def __repr__(self) -> str:
         """Represent identity and load state without touching unresolved fields."""
@@ -1089,7 +1083,7 @@ class Helper(Generic[MediaT]):
         page_retry = iterator_config.page_retry
         item_retry = iterator_config.item_retry
         page_error_handler = iterator_config.page_error_handler
-        item_error_handler = iterator_config.page_error_handler
+        item_error_handler = iterator_config.item_error_handler
         load_fields = iterator_config.load_specific_fields
         load_sources = iterator_config.load_specific_sources
         page_request_method = iterator_config._page_request_method
@@ -1550,9 +1544,7 @@ class BaseCore:
         self.cache = cache if cache is not None else Cache(self.configuration)
         self.logger = configure_app_logging("BASE API - [BaseCore]", log_file=None, level=logging.ERROR)
         self.default_headers = {
-            "User-Agent": UA_DESKTOP_CHROME,
             "Accept-Language": self.configuration.locale,
-            "Accept-Encoding": "gzip, deflate, br"
         }
 
     async def __aenter__(self) -> Self:
@@ -1576,6 +1568,10 @@ class BaseCore:
                                    http_port=log_port)
 
     def initialize_session(self) -> None:
+        """Initialize the owned HTTP session once for the current lifecycle."""
+        if self.session is not None:
+            return
+
         verify = self.configuration.verify_ssl
 
         curl_options: Dict[CurlOpt, Union[bytes, int]] = {}
@@ -1609,19 +1605,20 @@ class BaseCore:
             proxy=proxy,
             timeout=self.configuration.timeout,
             verify=verify,
+            ja3=js3,
+            http_version=http_version,
             impersonate=impersonation,
             curl_options=curl_options,
-            http_version=http_version,
-            ja3=js3,
             proxy_auth=p_auth,
-            trust_env=trust_env
+            trust_env=trust_env,
+            cookies=self.configuration.cookies,
         )
         # Ensure our defaults are on the session
         assert self.session is not None
         self.session.headers.update(self.default_headers)
 
     async def enforce_delay(self) -> None:
-        """Enforces the specified delay in config.request_delay (only if > 0)."""
+        """Enforce this core's configured delay between requests, when enabled."""
         delay = self.configuration.request_delay
         if delay and delay > 0:
             async with self._delay_lock:
@@ -1714,6 +1711,7 @@ class BaseCore:
         exponential_wait = wait_exponential_jitter(
             initial=self.configuration.request_retry_initial_delay,
             max=self.configuration.request_retry_max_delay,
+            exp_base=self.configuration.request_multiplier,
             jitter=self.configuration.request_retry_jitter,
         )
 
@@ -2048,89 +2046,158 @@ class BaseCore:
         )
         return cast(bytes, response.content)
 
-
-    @lru_cache(maxsize=250)
-    async def get_m3u8_by_quality(self, m3u8_url: str, quality: Union[str, int]) -> str:
+    async def get_m3u8_by_quality(
+            self,
+            m3u8_url: str,
+            quality: str | int,
+    ) -> str:
         """
         Return the media-playlist URL for the requested quality.
 
-        quality:
-          - 'best' | 'half' | 'worst'
-          - 1080 / '1080' / '1080p' (and similar)
+        Supported preferences:
+            best
+            half
+            worst
+
+        Supported explicit qualities:
+            144
+            240
+            360
+            480
+            540
+            720
+            1080
+            1440
+            2160
+
+        Numeric strings such as "1080" and "1080p" are accepted too.
         """
-        if m3u8 is None:
-            raise ModuleNotFoundError(f"""
-Using m3u8 is optional depending whether you use HLS videos or static videos. It seems like you are trying to download
-from HLS. Please install m3u8 using: `pip install m3u8`.
 
-If this does not fix the issue, there's an import error related to your environment. In this case please create
-a new Python file, import only m3u8 and see what error you get. 
-""")
-
-        # Resolve master content
-        assert m3u8 is not None
+        if m3u8 is None:
+            raise ModuleNotFoundError(
+                "HLS support requires the 'm3u8' package."
+            )
 
-        if inspect.iscoroutinefunction(m3u8_url) or (callable(m3u8_url) and not isinstance(m3u8_url, str)):
+        # Resolve callable / awaitable sources.
+        if (
+                inspect.iscoroutinefunction(m3u8_url)
+                or (
+                callable(m3u8_url)
+                and not isinstance(m3u8_url, str)
+        )
+        ):
             m3u8_url = m3u8_url()
-        if inspect.iscoroutine(m3u8_url) or inspect.isawaitable(m3u8_url):
+
+        if inspect.isawaitable(m3u8_url):
             m3u8_url = await m3u8_url
 
+        if not isinstance(m3u8_url, str):
+            raise TypeError(
+                "m3u8_url must resolve to a string."
+            )
+
+        # Load master playlist.
         if m3u8_url.lstrip().startswith("#EXTM3U"):
             master = m3u8.loads(m3u8_url)
-            self.logger.debug("Resolved inline/custom m3u8 master content.")
-            base_for_join = ""  # URIs should be absolute in inline cases; join will handle if relative
+            base_url = None
+
+            self.logger.debug(
+                "Resolved inline/custom m3u8 master content."
+            )
+
         else:
-            content = await self.fetch_text(url=m3u8_url)
+            content = await self.fetch_text(
+                url=m3u8_url
+            )
+
             master = m3u8.loads(content)
-            base_for_join = m3u8_url
-            self.logger.debug("Resolved m3u8 master: %s", m3u8_url)
+            base_url = m3u8_url
+
+            self.logger.debug(
+                "Resolved m3u8 master: %s",
+                m3u8_url,
+            )
 
         if not master.is_variant:
-            raise PlaylistExtractionError(f"Playlist is not a master Playlist: {m3u8_url}")
+            raise PlaylistExtractionError(
+                f"Playlist is not a master playlist: {m3u8_url}"
+            )
 
         variants = collect_variants(master)
+
         if not variants:
-            raise PlaylistExtractionError(f"No usable variants found in master Playlist: {m3u8_url}, {master}")
+            raise PlaylistExtractionError(
+                f"No usable video variants found: {m3u8_url}"
+            )
 
-        q = normalize_quality_value(quality)
-        if isinstance(q, str):  # 'best'/'half'/'worst'
-            chosen = pick_by_label(variants, q)
-        else:  # numeric height like 1080, 720, etc.
-            chosen = pick_by_height(variants, q)
+        chosen = choose_variant(variants, quality)
 
-        full_url = urljoin(base_for_join or m3u8_url, chosen["uri"])
-        return full_url
+        uri = chosen["uri"]
 
-    async def list_available_qualities(self, m3u8_url: str) -> List[int]:
+        # Master was fetched from a URL.
+        if base_url is not None:
+            return urljoin(
+                base_url,
+                uri,
+            )
+
+        # Inline playlist containing an absolute variant URL.
+        if uri.startswith(("http://", "https://")):
+            return uri
+
+        raise PlaylistExtractionError(
+            "Inline HLS master contains relative variant URLs, "
+            "so they cannot be resolved without a base URL."
+        )
+
+    async def list_available_qualities(
+            self,
+            m3u8_url: str,
+    ) -> list[int]:
         """
-        Inspect the master playlist and return sorted unique heights (e.g., [240, 360, 480, 720, 1080]).
+        Inspect an HLS master playlist and return canonical,
+        sorted, unique qualities.
         """
-        assert m3u8 is not None
 
-        if inspect.iscoroutinefunction(m3u8_url) or (callable(m3u8_url) and not isinstance(m3u8_url, str)):
+        if m3u8 is None:
+            raise ModuleNotFoundError(
+                "HLS support requires the 'm3u8' package."
+            )
+
+        if (
+                inspect.iscoroutinefunction(m3u8_url)
+                or (
+                callable(m3u8_url)
+                and not isinstance(m3u8_url, str)
+        )
+        ):
             m3u8_url = m3u8_url()
-        if inspect.iscoroutine(m3u8_url) or inspect.isawaitable(m3u8_url):
+
+        if inspect.isawaitable(m3u8_url):
             m3u8_url = await m3u8_url
 
-        if not m3u8_url.startswith("https://"):
+        if not isinstance(m3u8_url, str):
+            raise TypeError(
+                "m3u8_url must resolve to a string."
+            )
+
+        if m3u8_url.lstrip().startswith("#EXTM3U"):
             master = m3u8.loads(m3u8_url)
-        else:
-            content = await self.fetch_text(url=m3u8_url)
+
+        elif m3u8_url.startswith(("http://", "https://")):
+            content = await self.fetch_text(
+                url=m3u8_url
+            )
+
             master = m3u8.loads(content)
 
+        else:
+            master = m3u8.loads(m3u8_url)
+
         if not master.is_variant:
             return []
 
-        heights = {h for h in (height_from_variant(v) for v in master.playlists) if h is not None}
-        if heights:
-            return sorted(heights)
-        # fallback: bandwidth-only (roughly infer tiers)
-        by_bw = sorted(
-            (getattr(v.stream_info, "bandwidth", 0) for v in master.playlists if is_video_playlist(v)),
-            key=int
-        )
-        # Return rank numbers instead of heights if we truly can't infer—kept simple:
-        return [i for i, _ in enumerate(by_bw, start=1)]
+        return available_qualities(collect_variants(master))
 
     async def get_segments(self, m3u8_url_master: str, quality: Union[str, int]) -> List[str]:
         assert m3u8 is not None
@@ -2217,7 +2284,7 @@ a new Python file, import only m3u8 and see what error you get.
             self.logger.debug("Failed to remove directory %s: %s", path, e)
 
     async def download_segment(self, url: str, timeout: int, stop_event:
-                                threading.Event | None = None) -> tuple[str, bytes, bool]:
+                                asyncio.Event | None = None) -> tuple[str, bytes, bool]:
         """
         Attempt to download a single segment.
         Returns (url, content, success).
@@ -2264,8 +2331,8 @@ a new Python file, import only m3u8 and see what error you get.
         return await self.threaded_download(
             configuration=configuration,
             pre_resolved_m3u8=m3u8_url,
-            timeout=config.timeout,
-            max_workers=config.max_workers_download
+            timeout=self.configuration.timeout,
+            max_workers=self.configuration.max_workers_download,
         )
 
     async def threaded_download(
@@ -2455,6 +2522,8 @@ a new Python file, import only m3u8 and see what error you get.
                 else:
                     self.logger.debug(f"Writing segments to disk. segment_dir={segment_dir} tmp_path={tmp_path}")
 
+                segment_tasks: set[asyncio.Task[Tuple[int, bool, bytes]]] = set()
+                stop_waiter: asyncio.Task[bool] | None = None
                 try:
                     # Use asyncio.gather to fetch segments concurrently instead of ThreadPoolExecutor
 
@@ -2489,78 +2558,110 @@ a new Python file, import only m3u8 and see what error you get.
                                     )
                             return idx, False, b""
 
-                    tasks = [fetch_segment_with_semaphore(i, segments[i]) for i in target_indices]
-
-                    # Use asyncio.as_completed to process results as they come in, similar to wait(FIRST_COMPLETED)
-                    for coro in asyncio.as_completed(tasks):
-                        if stop_event is not None and stop_event.is_set():
-                            cancelled = True
-                            # The remaining tasks will see the event set and exit quickly
-                            continue
+                    segment_tasks = {
+                        asyncio.create_task(
+                            fetch_segment_with_semaphore(i, segments[i]),
+                            name=f"hls-segment-{i}",
+                        )
+                        for i in target_indices
+                    }
+                    stop_waiter = (
+                        asyncio.create_task(stop_event.wait(), name="hls-stop-waiter")
+                        if stop_event is not None
+                        else None
+                    )
 
-                        i, success, data = await coro
+                    while segment_tasks:
+                        waiters = set(segment_tasks)
+                        if stop_waiter is not None:
+                            waiters.add(stop_waiter)
+                        done, _ = await asyncio.wait(
+                            waiters,
+                            return_when=asyncio.FIRST_COMPLETED,
+                        )
 
-                        if cancelled:
-                            continue
+                        if stop_waiter is not None and stop_waiter in done:
+                            cancelled = True
+                            for task in segment_tasks:
+                                task.cancel()
+                            await asyncio.gather(*segment_tasks, return_exceptions=True)
+                            segment_tasks.clear()
+                            self.logger.info("Cancelled all in-flight HLS segment requests.")
+                            break
+
+                        completed_tasks = done.intersection(segment_tasks)
+                        for task in completed_tasks:
+                            segment_tasks.remove(task)
+                            i, success, data = task.result()
+
+                            if success and data:
+                                downloaded[i] = True # Successfully got segment, mark it as done
+                                downloaded_count += 1
+                                if segment_dir:
+                                    # Write to a temp path (good for resuming, but not I/O efficient)
+                                    seg_path = segment_file_path(segment_dir, i, width)
+                                    tmp_seg = f"{seg_path}.part"
+                                    # Offload segment file writing to a thread
+                                    def write_part(ts_path: str, t_data: bytes) -> None:
+                                        with open(ts_path, "wb") as f:
+                                            f.write(t_data)
+                                    await asyncio.to_thread(write_part, tmp_seg, data)
+                                    os.replace(tmp_seg, seg_path)
+                                else:
+                                    assert parts is not None
+                                    parts[i] = data # Keep in memory (I/O efficient)
+
+                                progressed += 1 # Fetched +1 segment, so we give back callback
+                                if callback:
+                                    callback(progressed, n)
+                                if progressed >= next_progress_log or progressed == n:
+                                    remaining = n - downloaded_count
+                                    self.logger.debug(
+                                        f"Segment progress: processed={progressed}/{n} "
+                                        f"downloaded={downloaded_count} remaining={remaining}"
+                                    )
+                                    next_progress_log += progress_log_step
 
-                        if success and data:
-                            downloaded[i] = True # Successfully got segment, mark it as done
-                            downloaded_count += 1
-                            if segment_dir:
-                                # Write to a temp path (good for resuming, but not I/O efficient)
-                                seg_path = segment_file_path(segment_dir, i, width)
-                                tmp_seg = f"{seg_path}.part"
-                                # Offload segment file writing to a thread
-                                def write_part(ts_path: str, t_data: bytes) -> None:
-                                    with open(ts_path, "wb") as f:
-                                        f.write(t_data)
-                                await asyncio.to_thread(write_part, tmp_seg, data)
-                                os.replace(tmp_seg, seg_path)
                             else:
-                                assert parts is not None
-                                parts[i] = data # Keep in memory (I/O efficient)
-
-                            progressed += 1 # Fetched +1 segment, so we give back callback
-                            if callback:
-                                callback(progressed, n)
-                            if progressed >= next_progress_log or progressed == n:
-                                remaining = n - downloaded_count
-                                self.logger.debug(
-                                    f"Segment progress: processed={progressed}/{n} "
-                                    f"downloaded={downloaded_count} remaining={remaining}"
-                                )
-                                next_progress_log += progress_log_step
-
-                        else:
-                            # Handling failure (already retried in fetch_segment_with_semaphore)
-                            progressed += 1
-                            if callback:
-                                callback(progressed, n)
-                            if progressed >= next_progress_log or progressed == n:
-                                remaining = n - downloaded_count
-                                self.logger.debug(
-                                    f"Segment progress: processed={progressed}/{n} "
-                                    f"downloaded={downloaded_count} remaining={remaining}"
-                                )
-                                next_progress_log += progress_log_step
-
-                        if not segment_dir and parts is not None:
-                            chunks_to_write = []
-                            while next_to_write < n and parts[next_to_write] is not None:
-                                if parts[next_to_write]:
-                                    chunks_to_write.append(parts[next_to_write])
-                                next_to_write += 1
-                            if chunks_to_write:
-                                # Write memory chunks to thread to prevent IO block
-                                def write_chunks(fp: Any, list_of_data: List[bytes]) -> None:
-                                    for c_data in list_of_data:
-                                        fp.write(c_data)
-                                await asyncio.to_thread(write_chunks, cast(Any, out_fp), chunks_to_write)
+                                # Handling failure (already retried in fetch_segment_with_semaphore)
+                                progressed += 1
+                                if callback:
+                                    callback(progressed, n)
+                                if progressed >= next_progress_log or progressed == n:
+                                    remaining = n - downloaded_count
+                                    self.logger.debug(
+                                        f"Segment progress: processed={progressed}/{n} "
+                                        f"downloaded={downloaded_count} remaining={remaining}"
+                                    )
+                                    next_progress_log += progress_log_step
+
+                            if not segment_dir and parts is not None:
+                                chunks_to_write = []
+                                while next_to_write < n and parts[next_to_write] is not None:
+                                    if parts[next_to_write]:
+                                        chunks_to_write.append(parts[next_to_write])
+                                    next_to_write += 1
+                                if chunks_to_write:
+                                    # Write memory chunks to thread to prevent IO block
+                                    def write_chunks(fp: Any, list_of_data: List[bytes]) -> None:
+                                        for c_data in list_of_data:
+                                            fp.write(c_data)
+                                    await asyncio.to_thread(write_chunks, cast(Any, out_fp), chunks_to_write)
 
                 finally:
+                    if stop_waiter is not None:
+                        stop_waiter.cancel()
+                        await asyncio.gather(stop_waiter, return_exceptions=True)
+                    if segment_tasks:
+                        for task in segment_tasks:
+                            task.cancel()
+                        await asyncio.gather(*segment_tasks, return_exceptions=True)
                     if out_fp is not None:
                         out_fp.close()
 
+            if stop_event is not None and stop_event.is_set():
+                cancelled = True
+
             missing = [i for i, ok in enumerate(downloaded) if not ok] # Missing segments
             missing_urls = [segments[i] for i in missing] # Missing URLs of segments
             self.logger.info(

{eaf_base_api-4.0.0 → eaf_base_api-4.1.0}/base_api/modules/static_functions.py

@@ -3,11 +3,13 @@ import re
 import math
 import json
 import unicodedata
+from collections.abc import Iterable
+from dataclasses import asdict
 from pathlib import PurePath
 from .type_hints import DownloadState
 from datetime import timezone, datetime
 from curl_cffi.requests import Response
-from typing import Dict, Any, cast, List, Callable, Tuple, Union
+from typing import Dict, Any, cast, List, Callable, Literal, Union
 from email.utils import parsedate_to_datetime
 
 
@@ -68,196 +70,290 @@ def least_factors(n: int) -> int:
     return n
 
 
-def normalize_quality_value(quality: Union[str, int]) -> Union[str, int]:
+type QualityPreference = int | Literal["best", "half", "worst"]
+
+QUALITY_LABELS = frozenset({"best", "half", "worst"})
+
+
+def is_video_playlist(variant: Any) -> bool:
+    """Filter out I-frames/audio-only playlists."""
+    # m3u8 lib sometimes sets is_iframe if EXT-X-I-FRAME-STREAM-INF is present.
+    if getattr(variant, "is_iframe", False):
+        return False
+
+    # If codecs known and contain only audio (mp4-a, ac-3, ec-3, etc.)
+    codecs = getattr(variant.stream_info, "codecs", None) if getattr(variant, "stream_info", None) else False
+    if codecs:
+        # very light heuristic: if no video codec substring, probably audio-only.
+        # video: avc1, hvc1, hev1, vp9, av01, dvh
+        codecs_text = str(codecs).lower()
+        if not any(v in codecs_text for v in ("avc1", "hvc1", "hev1", "av01", "vp9", "dvh")):
+            return False
+
+    return True
+
+
+def get_segment_index_width(total: int) -> int:
+    return max(6, len(str(max(0, total - 1))))
+
+
+COMMON_QUALITIES = frozenset({
+    144,
+    240,
+    250,
+    360,
+    480,
+    540,
+    720,
+    1080,
+    1440,
+    2160,
+})
+# Kept as an alias for callers that imported the name during the 4.0 rollout.
+ALLOWED_QUALITIES = COMMON_QUALITIES
+
+
+def validate_quality(value: int) -> int:
+    """Validate a normalized quality without restricting provider-specific tiers."""
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise TypeError(f"Invalid quality type: {type(value).__name__}")
+    if value <= 0:
+        raise ValueError(f"Invalid video quality: {value!r}")
+
+    return value
+
+
+def normalize_quality(value: str | int) -> int:
     """
-    quality: represents the quality value that should be normalized
+    Convert a quality into a canonical integer.
+
+    Accepted:
+        720
+        "720"
+        "720p"
+
+    Rejected:
+        "best"
+        "half"
+        "worst"
+        "720p60"
+        zero or negative values
     """
-    if isinstance(quality, int):
-        return quality # If the quality value is already an int, just return it directly
 
-    quality = str(quality).lower().strip() # Convert to string, lower and remove white spaces
+    if isinstance(value, bool):
+        raise TypeError("A boolean is not a valid video quality.")
+
+    if isinstance(value, int):
+        quality = value
+
+    elif isinstance(value, str):
+        value = value.strip().lower()
+
+        match = re.fullmatch(r"(\d+)[pP]?", value)
+
+        if not match:
+            raise ValueError(
+                f"Invalid video quality: {value!r}"
+            )
+
+        quality = int(match.group(1))
+
+    else:
+        raise TypeError(
+            f"Invalid quality type: {type(value).__name__}"
+        )
+
+    return validate_quality(quality)
+
+def normalize_quality_preference(
+    value: str | int,
+) -> QualityPreference:
+
+    if isinstance(value, str):
+        value = value.strip().lower()
 
-    if quality in {"best", "half", "worst"}:
-        return quality # best, half and worst are also accepted values and will be further resolved in other functions
+        if value in QUALITY_LABELS:
+            return cast(QualityPreference, value)
+
+    return normalize_quality(value)
 
-    m = re.search(r'(\d{3,4})', quality) # Search for int values that fit 144p-2160p values. return as int
-    if m:
-        return int(m.group(1))
-    raise ValueError(f"Invalid quality: {quality}")
 
 
 def choose_quality_from_list(
-        available: List[Union[str, int]],
-        target: Union[str, int],
-        default_fallback: int | str | None = None
+    available: Iterable[str | int],
+    target: str | int,
+    default_fallback: str | int | None = None,
 ) -> int:
-    """
-    Selects a video quality from a list based on a target integer< or label.
+    """Choose a quality, falling back to the nearest available numeric tier."""
 
-    :param available: List of available qualities (e.g., [240, "360", "1080p"]).
-    :param target: Numeric target (highest ≤ target) or label ("best", "worst", "half").
-    :param default_fallback: Optional integer to return if all parsing fails.
-    :return: The selected quality as an integer.
-    """
+    available_ints = normalize_qualities(available)
 
-    # 1. Edge Case: Empty List
-    if not available:
+    if not available_ints:
         if default_fallback is not None:
-            return default_fallback
-        raise ValueError("The 'available' list cannot be empty.")
+            return normalize_quality(default_fallback)
+        raise ValueError(
+            "No valid video qualities are available."
+        )
 
-    # 2. Data Sanitization: Filter out unparseable values safely
-    valid_ints = set()
-    for x in available:
-        try:
-            # Handle common string formats gracefully (e.g., "1080p" -> 1080)
-            if isinstance(x, str):
-                x = x.lower().rstrip('p')
-            valid_ints.add(int(x))
-        except (ValueError, TypeError):
-            continue  # Skip unparseable garbage rather than crashing
-
-    if not valid_ints:
-        if default_fallback is not None:
-            return default_fallback
-        raise ValueError("No valid numeric qualities found in 'available'.")
-
-    # Sort the sanitized list (e.g., [144, 240, 360, 720, 1080])
-    available_ints = sorted(valid_ints)
-
-    # 3. Edge Case: User passes a numeric string as a target (e.g., "1080")
-    if isinstance(target, str) and target.isdigit():
-        target = int(target)
-
-    # 4. Handle String Targets
-    if isinstance(target, str):
-        target = target.lower()
-        if target == "best":
-            return available_ints[-1]
-        if target == "worst":
-            return available_ints[0]
-        if target == "half":
-            # Middle index. Examples: len=3 (idx 1), len=4 (idx 2, rounds up)
-            return available_ints[len(available_ints) // 2]
-
-        # Fallback for unrecognized string labels
+    try:
+        preference = normalize_quality_preference(target)
+    except (TypeError, ValueError):
         if default_fallback is not None:
-            return default_fallback
-        raise ValueError(f"Invalid label: '{target}'. Expected 'best', 'worst', 'half', or a number.")
+            return normalize_quality(default_fallback)
+        raise
 
-    # 5. Handle Numeric Targets (highest ≤ target, else closest)
-    if isinstance(target, (int, float)):
-        le = [h for h in available_ints if h <= target]
-        if le:
-            return le[-1]
+    if preference == "best":
+        return available_ints[-1]
 
-        # Fallback closest: if target is 144 but only 240+ is available,
-        # all available qualities are > target. The closest is the lowest available.
+    if preference == "worst":
         return available_ints[0]
 
-    # Catch-all for entirely wrong target types (e.g., lists, dicts)
-    if default_fallback is not None:
-        return default_fallback
-    raise TypeError("Target must be a string or integer.")
+    if preference == "half":
+        return available_ints[len(available_ints) // 2]
 
+    # Prefer the higher tier when two variants are equally close.
+    return min(
+        available_ints,
+        key=lambda quality: (abs(quality - preference), -quality),
+    )
 
-def height_from_variant(variant: Any) -> int | None:
-    """Extract height from a variant:
-    1) stream_info.resolution (w, h)
-    2) URI pattern like .../720p/...
+
+def normalize_qualities(
+    values: Iterable[str | int],
+) -> list[int]:
+    """
+    Return canonical, unique qualities sorted worst -> best.
     """
-    if getattr(variant, "stream_info", None) and variant.stream_info.resolution:
-        _, h = variant.stream_info.resolution  # (width, height)
-        return int(h) # -> returns the height of a variant of a m3u8 master playlist
 
-    # Fallback to search with a regex pattern
-    if variant.uri:
-        m = HEIGHT_FROM_URI.search(variant.uri)
-        if m:
-            return int(m.group(1))
+    qualities: set[int] = set()
+    for value in values:
+        try:
+            qualities.add(normalize_quality(value))
+        except (TypeError, ValueError):
+            # Provider data can contain labels such as "auto" alongside real
+            # qualities.  They should not make the usable variants disappear.
+            continue
 
-    # If nothing is found, though this shouldn't happen
-    return None
+    return sorted(qualities)
 
-def is_video_playlist(variant: Any) -> bool:
-    """Filter out I-frames/audio-only playlists."""
-    # m3u8 lib sometimes sets is_iframe if EXT-X-I-FRAME-STREAM-INF is present.
-    if getattr(variant, "is_iframe", False):
-        return False
 
-    # If codecs known and contain only audio (mp4-a, ac-3, ec-3, etc.)
-    codecs = getattr(variant.stream_info, "codecs", None) if getattr(variant, "stream_info", None) else False
-    if codecs:
-        # very light heuristic: if no video codec substring, probably audio-only.
-        # video: avc1, hvc1, hev1, vp9, av01, dvh
-        assert isinstance(codecs, str)
-        if not any(v in codecs.lower() for v in ("avc1", "hvc1", "hev1", "av01", "vp9", "dvh")):
-            return False
+def quality_from_variant(variant: Any) -> int | None:
+    """Extract a quality tier from a landscape or portrait HLS variant."""
+    stream_info = getattr(variant, "stream_info", None)
+    resolution = getattr(stream_info, "resolution", None)
+    if resolution:
+        try:
+            width, height = resolution
+            # Quality names describe the shorter side: 1920x1080 and
+            # 1080x1920 are both 1080p.
+            return normalize_quality(min(int(width), int(height)))
+        except (TypeError, ValueError):
+            pass
+
+    uri = getattr(variant, "uri", None)
+    if isinstance(uri, str):
+        match = HEIGHT_FROM_URI.search(uri)
+        if match:
+            try:
+                return normalize_quality(match.group(1))
+            except (TypeError, ValueError):
+                pass
 
-    return True
+    return None
 
-def collect_variants(master: Any) -> List[Dict[str, Any]]:
-    """Normalize playlist variants to a comparable list."""
-    items: List[Dict[str, Any]] = []
-    for v in master.playlists:
-        if not is_video_playlist(v):
+
+def collect_variants(master: Any) -> list[dict[str, Any]]:
+    """Return video variants with normalized, comparable metadata."""
+    variants: list[dict[str, Any]] = []
+    for variant in getattr(master, "playlists", ()):
+        if not is_video_playlist(variant):
             continue
 
-        h = height_from_variant(v)
-        bw = getattr(v.stream_info, "bandwidth", 0) if getattr(v, "stream_info", None) else 0
-        fr = getattr(v.stream_info, "frame_rate", 0.0) if getattr(v, "stream_info", None) else 0.0
-        items.append({
-            "uri": v.uri,
-            "height": h,                 # may be None
-            "bandwidth": int(bw or 0),
-            "frame_rate": float(fr or 0.0),
-            "resolution": getattr(v.stream_info, "resolution", None) if getattr(v, "stream_info", None) else None,
-            "raw": v
+        stream_info = getattr(variant, "stream_info", None)
+        variants.append({
+            "uri": getattr(variant, "uri", ""),
+            "quality": quality_from_variant(variant),
+            "bandwidth": int(getattr(stream_info, "bandwidth", 0) or 0),
+            "frame_rate": float(getattr(stream_info, "frame_rate", 0.0) or 0.0),
+            "resolution": getattr(stream_info, "resolution", None),
+            "raw": variant,
         })
-    return items
 
-def pick_by_label(variants: List[Dict[str, Any]], label: str) -> Dict[str, Any]:
-    """best / worst / half based on a combined rank by (height, bandwidth)."""
-    # rank by height first, then bandwidth as tiebreaker
-    def key_fn(v: Dict[str, Any]) -> Tuple[int, int]:
-        return v["height"] or 0, v["bandwidth"]
-    ordered = sorted(variants, key=key_fn)
+    return variants
 
-    if not ordered:
-        raise ValueError("No video variants available in master playlist.")
 
-    elif label == "worst":
+def available_qualities(variants: Iterable[dict[str, Any]]) -> list[int]:
+    """Return sorted, unique integer qualities from normalized variants."""
+    return normalize_qualities(
+        variant["quality"]
+        for variant in variants
+        if variant.get("quality") is not None
+    )
+
+
+def choose_variant(
+    variants: Iterable[dict[str, Any]],
+    target: str | int,
+) -> dict[str, Any]:
+    """Select an HLS variant with quality and bandwidth fallbacks."""
+    candidates = list(variants)
+    if not candidates:
+        raise ValueError("No video variants are available.")
+
+    preference = normalize_quality_preference(target)
+    qualities = available_qualities(candidates)
+    if qualities:
+        selected_quality = choose_quality_from_list(qualities, preference)
+        matching = [
+            variant
+            for variant in candidates
+            if variant.get("quality") == selected_quality
+        ]
+        return max(
+            matching,
+            key=lambda variant: (
+                variant.get("bandwidth", 0),
+                variant.get("frame_rate", 0.0),
+            ),
+        )
+
+    # Some masters expose only bandwidth. Labels can still be ranked, while a
+    # numeric request falls back to the highest-bandwidth usable variant.
+    ordered = sorted(
+        candidates,
+        key=lambda variant: (
+            variant.get("bandwidth", 0),
+            variant.get("frame_rate", 0.0),
+        ),
+    )
+    if preference == "worst":
         return ordered[0]
-    elif label == "half":
-        return ordered[len(ordered)//2]
-    elif label == "best":
-        return ordered[-1]
-    else:
-        raise ValueError("Invalid quality label.")
+    if preference == "half":
+        return ordered[len(ordered) // 2]
+    return ordered[-1]
 
 
-def pick_by_height(variants: List[Dict[str, Any]], target: int) -> Dict[str, Any]:
-    """Choose the highest height ≤ target; else closest by absolute diff (ties -> higher)."""
-    with_height = [v for v in variants if v["height"] is not None]
-    if with_height:
-        # Prefer height <= target
-        below_eq = [v for v in with_height if v["height"] <= target]
-        if below_eq:
-            # Among same height, prefer higher bandwidth then higher fps
-            best = sorted(below_eq, key=lambda v: (v["height"], v["bandwidth"], v["frame_rate"]))[-1]
-            return best
+# Compatibility wrappers for callers of the pre-4.0 helper names.
+def normalize_quality_value(quality: str | int) -> QualityPreference:
+    return normalize_quality_preference(quality)
 
-        # Fallback: closest by absolute diff; ties -> higher height
-        def diff_key(v: Dict[str, Any]) -> Tuple[int, int, int, float]:
-            return abs((v["height"] or 0) - target), -(v["height"] or 0), v["bandwidth"], v["frame_rate"]
-        return sorted(with_height, key=diff_key)[0]
 
-    # If we have no heights at all, fall back to bandwidth ranking
-    return sorted(variants, key=lambda v: v["bandwidth"])[-1]
+def height_from_variant(variant: Any) -> int | None:
+    return quality_from_variant(variant)
 
 
-def get_segment_index_width(total: int) -> int:
-    return max(6, len(str(max(0, total - 1))))
+def pick_by_label(
+    variants: list[dict[str, Any]],
+    label: str,
+) -> dict[str, Any]:
+    return choose_variant(variants, label)
+
+
+def pick_by_height(
+    variants: list[dict[str, Any]],
+    target: int,
+) -> dict[str, Any]:
+    return choose_variant(variants, target)
 
 
 def segment_file_path(segment_dir, index: int, width: int) -> str:
@@ -266,8 +362,12 @@ def segment_file_path(segment_dir, index: int, width: int) -> str:
 
 def write_segment_state(state_path: str, state: DownloadState) -> None:
     tmp_path = f"{state_path}.tmp"
+    payload = asdict(state)
+    for path_key in ("output_path", "segment_dir"):
+        if isinstance(payload[path_key], PurePath):
+            payload[path_key] = str(payload[path_key])
     with open(tmp_path, "w", encoding="utf-8") as fp:
-        json.dump(state, fp, ensure_ascii=True, indent=2, sort_keys=True)
+        json.dump(payload, fp, ensure_ascii=True, indent=2, sort_keys=True)
     os.replace(tmp_path, state_path)
 
 
@@ -479,4 +579,4 @@ def strip_title(
     sanitized = sanitized[:max_length].rstrip(" .")
 
     # 8. Return default fallback if sanitization leaves an empty string
-    return sanitized if sanitized else default_name
+    return sanitized if sanitized else default_name

{eaf_base_api-4.0.0 → eaf_base_api-4.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "eaf_base_api"
-version = "4.0.0"
+version = "4.1.0"
 description = "A base API for EchterAlsFake's Porn APIs"
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.12"