eaf-base-api 4.2.0__tar.gz → 4.2.2__tar.gz

{eaf_base_api-4.2.0 → eaf_base_api-4.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eaf_base_api
-Version: 4.2.0
+Version: 4.2.2
 Summary: A base API for EchterAlsFake's Porn APIs
 Author: Johannes Habel
 Author-email: Johannes Habel <EchterAlsFake@proton.me>
@@ -11,7 +11,6 @@ Requires-Dist: cachetools>=7.0.0
 Requires-Dist: curl-cffi
 Requires-Dist: tenacity>=9.1.2
 Requires-Dist: m3u8 ; extra == 'hls'
-Requires-Dist: av ; python_full_version >= '3.12' and extra == 'hls'
 Requires-Python: >=3.12
 Project-URL: Homepage, https://github.com/EchterAlsFake/eaf_base_api
 Project-URL: Repository, https://github.com/EchterAlsFake/eaf_base_api
@@ -90,6 +89,35 @@ Site-specific parsing, quality selection, and fallback behavior stay in each
 provider. Updated providers require `eaf-base-api>=4.2.0`; update the base library
 alongside them.
 
+## FFmpeg remuxing and single-pass metadata tagging (4.3)
+
+HLS TS→MP4 remuxing (`DownloadConfigHLS(remux=True)`) now runs directly via an
+`ffmpeg` binary (discovered on `PATH` or configured via `RuntimeConfig.ffmpeg_path`)
+instead of `PyAV`. You can also pass `metadata` (a provider `Video` object or `dict`)
+and `thumbnail` (`bytes` or path) on `DownloadConfigHLS` so `eaf_base_api` embeds
+MP4 container tags and cover art in the **same single FFmpeg remux pass** without
+rewriting the file a second time:
+
+```python
+from base_api import DownloadConfigHLS, write_metadata_tags
+
+config = DownloadConfigHLS(
+    quality="best",
+    path="./downloads",
+    remux=True,
+    metadata=video,           # Writes title, author, genre, date, description, url, comment
+    thumbnail=cover_bytes,    # Optional JPEG/PNG cover art embedded in the same pass
+    return_report=True,
+)
+report = await video.download(configuration=config)
+print(report.metadata_written)  # True when single-pass metadata/cover tagging succeeded
+```
+
+For direct MP4 downloads (`DownloadConfigRAW`) or existing MP4 files, use
+`write_metadata_tags(path, metadata=video, thumbnail=cover_bytes)` from
+`base_api.modules.ffmpeg` (or `base_api`). If `ffmpeg` is not installed when
+`remux=True`, `eaf_base_api` logs a warning and preserves the raw MPEG-TS stream.
+
 # Documentation (IMPORTANT!)
 > [!IMPORTANT]
 > Configuring eaf_base_api is necessary if you use any of my Porn APIs, because they all depend on this project.

{eaf_base_api-4.2.0 → eaf_base_api-4.2.2}/README.md

@@ -70,6 +70,35 @@ Site-specific parsing, quality selection, and fallback behavior stay in each
 provider. Updated providers require `eaf-base-api>=4.2.0`; update the base library
 alongside them.
 
+## FFmpeg remuxing and single-pass metadata tagging (4.3)
+
+HLS TS→MP4 remuxing (`DownloadConfigHLS(remux=True)`) now runs directly via an
+`ffmpeg` binary (discovered on `PATH` or configured via `RuntimeConfig.ffmpeg_path`)
+instead of `PyAV`. You can also pass `metadata` (a provider `Video` object or `dict`)
+and `thumbnail` (`bytes` or path) on `DownloadConfigHLS` so `eaf_base_api` embeds
+MP4 container tags and cover art in the **same single FFmpeg remux pass** without
+rewriting the file a second time:
+
+```python
+from base_api import DownloadConfigHLS, write_metadata_tags
+
+config = DownloadConfigHLS(
+    quality="best",
+    path="./downloads",
+    remux=True,
+    metadata=video,           # Writes title, author, genre, date, description, url, comment
+    thumbnail=cover_bytes,    # Optional JPEG/PNG cover art embedded in the same pass
+    return_report=True,
+)
+report = await video.download(configuration=config)
+print(report.metadata_written)  # True when single-pass metadata/cover tagging succeeded
+```
+
+For direct MP4 downloads (`DownloadConfigRAW`) or existing MP4 files, use
+`write_metadata_tags(path, metadata=video, thumbnail=cover_bytes)` from
+`base_api.modules.ffmpeg` (or `base_api`). If `ffmpeg` is not installed when
+`remux=True`, `eaf_base_api` logs a warning and preserves the raw MPEG-TS stream.
+
 # Documentation (IMPORTANT!)
 > [!IMPORTANT]
 > Configuring eaf_base_api is necessary if you use any of my Porn APIs, because they all depend on this project.

{eaf_base_api-4.2.0 → eaf_base_api-4.2.2}/base_api/__init__.py

@@ -1,4 +1,8 @@
 __all__ = [
+    "append_url_path",
+    "profile_base_url",
+    "replace_url_path",
+    "update_url_query",
     "BaseCore",
     "BaseMedia",
     "Cache",
@@ -55,15 +59,21 @@ __all__ = [
     "ProxyError",
     "UnknownNetworkError",
     "DownloadFailed",
+    "MetadataWriteError",
     "VideoUnavailable",
     "ResourceGone",
     "LoginFailed",
     "RegionBlocked",
     "configure_app_logging",
+    "find_ffmpeg",
+    "has_ffmpeg",
+    "remux_ts_to_mp4",
+    "write_metadata_tags",
 ]
 
 
 from base_api.modules import errors
+from base_api.modules.urls import append_url_path, profile_base_url, replace_url_path, update_url_query
 from base_api.modules.progress_bars import Callback
 from base_api.modules.errors import (
     DataNotLoadedError,
@@ -87,11 +97,18 @@ from base_api.modules.errors import (
     ProxyError,
     UnknownNetworkError,
     DownloadFailed,
+    MetadataWriteError,
     VideoUnavailable,
     ResourceGone,
     LoginFailed,
     RegionBlocked,
 )
+from base_api.modules.ffmpeg import (
+    find_ffmpeg,
+    has_ffmpeg,
+    remux_ts_to_mp4,
+    write_metadata_tags,
+)
 from base_api.modules.logger import configure_app_logging
 from base_api.base import (
     BaseCore,

{eaf_base_api-4.2.0 → eaf_base_api-4.2.2}/base_api/base.py

@@ -5,6 +5,8 @@ import time
 import hashlib
 import string
 import shutil
+import subprocess
+from pathlib import Path
 import asyncio
 import inspect
 import logging
@@ -27,6 +29,7 @@ from typing import (
 
 
 # 1. Standardize on relative imports
+from base_api.modules.urls import FetchedText
 from base_api.modules.errors import *
 from base_api.modules.type_hints import (
     DownloadReport, ResultOrder, ErrorMode, ErrorAction, ScrapeStage, 
@@ -39,30 +42,26 @@ from base_api.modules.static_functions import (
     choose_variant, collect_variants, get_segment_index_width
 )
 from base_api.modules.config import config, RuntimeConfig, DownloadConfigHLS, DownloadConfigRAW, IteratorConfig, make_iterator_config
+from base_api.modules.ffmpeg import find_ffmpeg, remux_ts_to_mp4, write_metadata_tags
 from base_api.modules.progress_bars import Callback
 from base_api.modules.logger import configure_app_logging, get_logger, log_context, ContextLogger
 
 # 2. Handle optional dependencies cleanly
 try:
     import m3u8
-except ImportError:
-    m3u8 = None
+    # Needed for all videos that use HLS streaming. Some do not and use mp4 containers / files instead
+except (ModuleNotFoundError, ImportError):
+    m3u8 = None  # type: ignore
 
-# 3. Handle specific runtime imports
 if TYPE_CHECKING:
-    from av.audio.codeccontext import AudioCodecContext
     import m3u8
 
-# The following imports are optional, because they depend on per API and I want to be as memory efficient as possible
 
-try:
-    import m3u8
-    # Needed for all videos that use HLS streaming. Some do not and use mp4 containers / files instead
-except (ModuleNotFoundError, ImportError):
-    m3u8 = None  # type: ignore
+REGEX_CHALLENGE = re.compile(r'var p=(\d+); var s=(\d+);.*?(\d+):1;', re.DOTALL)
 
 
-REGEX_CHALLENGE = re.compile(r'var p=(\d+); var s=(\d+);.*?(\d+):1;', re.DOTALL)
+class _ChallengeRetry(NetworkRequestError):
+    """Replay with an updated challenge cookie without network backoff."""
 
 
 class CachePolicy(StrEnum):
@@ -102,46 +101,6 @@ def _contains_http_status(error: BaseException, status_code: int) -> bool:
     return False
 
 
-def _normalize_packet_timestamps(
-    packet: Any,
-    offsets: dict[int, int],
-    last_dts: dict[int, int],
-    last_durations: dict[int, int],
-) -> int:
-    """Keep remuxed packet timestamps continuous across HLS discontinuities."""
-    if packet.dts is None:
-        return 0
-
-    stream_index = packet.stream.index
-    raw_dts = packet.dts
-    offset = offsets.get(stream_index, 0)
-    normalized_dts = raw_dts + offset
-    previous_dts = last_dts.get(stream_index)
-    correction = 0
-
-    if previous_dts is not None:
-        step = max(1, last_durations.get(stream_index, packet.duration or 1))
-        expected_dts = previous_dts + step
-        try:
-            ten_seconds = max(1, int(10 / float(packet.time_base)))
-        except (TypeError, ValueError, ZeroDivisionError):
-            ten_seconds = step * 240
-
-        discontinuity_threshold = max(step * 10, ten_seconds)
-        delta = normalized_dts - expected_dts
-        if normalized_dts <= previous_dts or abs(delta) > discontinuity_threshold:
-            correction = -delta
-            offset += correction
-            offsets[stream_index] = offset
-            normalized_dts = raw_dts + offset
-
-    packet.dts = normalized_dts
-    if packet.pts is not None:
-        packet.pts += offset
-    last_dts[stream_index] = normalized_dts
-    last_durations[stream_index] = max(1, packet.duration or 1)
-    return correction
-
 
 @dataclass(frozen=True, slots=True)
 class RequestCacheKey:
@@ -464,6 +423,9 @@ class BaseMedia:
     url: str
     core: object
 
+    # Transport metadata is separate from lazily loaded fields and the caller's URL.
+    resolved_url: str | None = field(default=None, init=False, repr=False, compare=False)
+
     loader_methods: ClassVar[Mapping[str, str]] = {}
 
     _load_lock: asyncio.Lock = field(
@@ -1619,6 +1581,8 @@ class BaseCore:
         self._inflight_text_requests: dict[RequestCacheKey, asyncio.Future[str]] = {}
         self.latest_key: str | None = None
         self.latest_key_time: float = 0.0
+        self._challenge_generation = 0
+        self._challenge_ready_at = 0.0
         self.last_request_time: float | None = None
         self.total_requests: int = 0  # Tracks how many requests have been made
         self.session: AsyncSession | None = None
@@ -1838,6 +1802,8 @@ class BaseCore:
 
         def retry_wait(retry_state: Any) -> float:
             error = retry_state.outcome.exception() if retry_state.outcome else None
+            if isinstance(error, _ChallengeRetry):
+                return 0.0
             if isinstance(error, RateLimitError) and error.retry_after is not None:
                 return max(0.0, error.retry_after)
             return cast(float, exponential_wait(retry_state))
@@ -1855,13 +1821,23 @@ class BaseCore:
                     try:
                         await self.enforce_delay()
                         req_headers = self._merged_headers(headers)
-                        req_cookies = self._merged_cookies(cookies)
-
-                        current_time = asyncio.get_running_loop().time()
-                        latest_key = self.latest_key
-                        if "KEY" not in session.cookies and latest_key is not None:
-                            if current_time - self.latest_key_time < 10:
-                                session.cookies.set("KEY", latest_key, domain=".pornhub.com", path="/")
+                        async with self.lock:
+                            # Pornhub requires a cooldown after computing KEY.
+                            # All requests share its deadline; late responses do
+                            # not add another sleep. Keeping the deadline also
+                            # preserves the cooldown if its waiter is cancelled.
+                            loop = asyncio.get_running_loop()
+                            while (remaining := self._challenge_ready_at - loop.time()) > 0:
+                                await asyncio.sleep(remaining)
+                            current_time = asyncio.get_running_loop().time()
+                            latest_key = self.latest_key
+                            if "KEY" not in session.cookies and latest_key is not None:
+                                if current_time - self.latest_key_time < 10:
+                                    session.cookies.set("KEY", latest_key, domain=".pornhub.com", path="/")
+                            # Snapshot before sending: a late response may belong
+                            # to the cookie generation preceding another solve.
+                            challenge_generation = self._challenge_generation
+                            req_cookies = self._merged_cookies(cookies)
 
                         self.total_requests += 1
                         response = await cast(Any, session).request(
@@ -1886,15 +1862,13 @@ class BaseCore:
                             resp_text = cast(bytes, response.content).decode(enc, errors="replace")
 
                             if 'onload="go()"' in resp_text:
-                                local_latest = getattr(self, "latest_key", None)
                                 async with self.lock:
-                                    if getattr(self, "latest_key", None) != local_latest:
+                                    if self._challenge_generation != challenge_generation:
                                         self.logger.info("Another task already resolved the challenge! Retrying request with the new cookie.")
                                         if self.latest_key:
                                             session.cookies.set("KEY", self.latest_key, domain=".pornhub.com", path="/")
 
-                                        await asyncio.sleep(1.5)
-                                        raise NetworkRequestError("Retrying request with the new cookie.")
+                                        raise _ChallengeRetry("Retrying request with the new cookie.")
 
                                     self.logger.info("Challenge page detected! Solving...")
                                     get_challenge = re.compile(r'go\(\).*?{(.*?)n=l.*?KEY.*?s\+":(\d+):', re.DOTALL)
@@ -1923,7 +1897,9 @@ class BaseCore:
 
                                             self.latest_key = cookie_value
                                             self.latest_key_time = asyncio.get_running_loop().time()
+                                            self._challenge_ready_at = self.latest_key_time + 1.5
                                             session.cookies.set("KEY", cookie_value, domain=".pornhub.com", path="/")
+                                            self._challenge_generation += 1
                                             self.logger.info("RESOLVED CHALLENGE! Injected cookie: %s", cookie_value)
 
                                             try:
@@ -1931,8 +1907,7 @@ class BaseCore:
                                             except (KeyError, Exception):
                                                 pass
 
-                                            await asyncio.sleep(1.5)
-                                            raise NetworkRequestError("Retrying request after solving challenge.")
+                                            raise _ChallengeRetry("Retrying request after solving challenge.")
                                         except (NetworkRequestError, SecurityAbort):
                                             raise
                                         except Exception as challenge_error:
@@ -1940,7 +1915,6 @@ class BaseCore:
 
                                     else:
                                         self.logger.error("Detected challenge page, but the regex failed to extract data.")
-                                        await asyncio.sleep(1.5)
                                         raise ChallengeRegexError("Detected Challenge, but regex couldn't extract, report this!")
 
 
@@ -1979,6 +1953,10 @@ class BaseCore:
                             f"HTTP {status} for {url}", status_code=status, url=url
                         )
 
+                    except _ChallengeRetry:
+                        # Cookie refresh is expected control flow, not a network
+                        # failure. The retry controller still bounds replays.
+                        raise
                     except RequestsError as e:
                         err_str = str(e).lower()
                         self.logger.error("Request error for URL %s: %s", url, e, exc_info=True)
@@ -2123,7 +2101,10 @@ class BaseCore:
                 params=params,
                 retry_non_idempotent=retry_non_idempotent,
             )
-            content = self._decode_response(response, url, self.logger)
+            content = FetchedText(
+                self._decode_response(response, url, self.logger),
+                url=getattr(response, "url", None) or url,
+            )
             if key is not None:
                 self.cache.set_response(key, content)
                 assert pending is not None
@@ -2947,14 +2928,25 @@ class BaseCore:
                 self.logger.info(f"Remuxing TS to MP4: input={tmp_path} output={path}")
                 # Offload heavy CPU/IO bound task
                 try:
-                    await asyncio.to_thread(self._convert_ts_to_mp4, tmp_path, path, callback_remux, ios_support)
+                    report.metadata_written = bool(
+                        await asyncio.to_thread(
+                            self._convert_ts_to_mp4,
+                            tmp_path,
+                            path,
+                            callback_remux,
+                            ios_support,
+                            configuration.metadata,
+                            configuration.thumbnail,
+                        )
+                    )
+                    configuration.metadata_written = report.metadata_written
                 except Exception:
                     self._safe_remove(path)
                     raise
                 # This is important, because not all players can play MPEG-TS AND I want to write
                 # metadata to the files, and this doesn't work without a container.
                 self._safe_remove(tmp_path)
-                self.logger.info(f"Remux completed: output={path}")
+                self.logger.info(f"Remux completed: output={path} metadata_written={report.metadata_written}")
 
             else:
                 self.logger.debug("Remux disabled; moving temporary file into place.")
@@ -2993,8 +2985,16 @@ class BaseCore:
             raise DownloadFailed(f"HLS download failed for {pre_resolved_m3u8}: {e}",
                                  url=pre_resolved_m3u8) from e
 
-    def _convert_ts_to_mp4(self, input_path: str, output_path: str,
-                           callback: Callable[[int, int], None] | None = None, ios_support: bool = False) -> None:
+    def _convert_ts_to_mp4(
+        self,
+        input_path: str,
+        output_path: str,
+        callback: Callable[[int, int], None] | None = None,
+        ios_support: bool = False,
+        metadata: Any = None,
+        thumbnail: Any = None,
+    ) -> bool:
+        del ios_support  # Kept for signature compatibility; FFmpeg copies AAC/H.264 with +faststart
         start_ts = time.perf_counter()
         self.logger.info("Remux start: input=%s output=%s", input_path, output_path)
 
@@ -3002,167 +3002,59 @@ class BaseCore:
             input_size = os.path.getsize(input_path)
             self.logger.debug("Remux input size: %s bytes", input_size)
         except Exception as e:
+            input_size = 0
             self.logger.debug("Remux input size unavailable for %s: %s", input_path, e, exc_info=True)
 
-        try:
-            from av import open as av_open  # type: ignore[import-not-found]
-            from av.audio.resampler import AudioResampler  # type: ignore[import-not-found]
-            import av.audio.frame  # Used for runtime isinstance check
-        except (ModuleNotFoundError, ImportError) as e:
-            self.logger.error("PyAV import failed for remux: %s", e, exc_info=True)
-            raise ModuleNotFoundError(
-                f"PyAV is required for remuxing. Install with pip install av. Not supported on Termux! {e}") from e
-
-        self.logger.debug("Opening input for remux: %s", input_path)
-        input_ = av_open(input_path)
-        fmt_name = (input_.format.name or "").lower()
-        self.logger.info("Input format detected: %s", fmt_name or '<unknown>')
-
-        if fmt_name == "mpegts":
-            # Fix 1: Suppress the stub mismatch for av.open
-            output = av_open(output_path, mode="w", format="mp4",
-                             options={"movflags": "faststart"})  # type: ignore[arg-type]
-
-            # --- VIDEO ---
-            in_video = input_.streams.video[0]
-            out_video = output.add_stream_from_template(template=in_video)
-            self.logger.debug(
-                "Video stream: codec=%s bit_rate=%s",
-                getattr(in_video.codec_context, 'name', None), getattr(in_video.codec_context, 'bit_rate', None)
+        ffmpeg_bin = find_ffmpeg(getattr(self.configuration, "ffmpeg_path", None))
+        if not ffmpeg_bin:
+            raise FileNotFoundError(
+                "FFmpeg executable not found for remuxing. Install FFmpeg on PATH, set "
+                "PORN_FETCH_FFMPEG_BINARY, or bundle it in third_party/."
             )
 
-            # --- AUDIO ---
-            in_audio = next((s for s in input_.streams if s.type == "audio"), None)
-            out_audio = None
-            transcode_audio = False
-            resampler = None
-
-            if in_audio:
-                # Fix 3: Explicitly narrow out None
-                assert in_audio is not None
-
-                # Fix 2: Cast context to AudioCodecContext so IDE knows about sample_rate and layout
-                audio_ctx = cast('AudioCodecContext', in_audio.codec_context)
-
-                copy_ok = {"aac"} if ios_support else {"aac", "alac", "mp3"}
-                codec_name = (audio_ctx.name or "").lower()
-                sample_rate = audio_ctx.sample_rate or 0
-                layout_name = audio_ctx.layout.name if getattr(audio_ctx, "layout", None) else "unknown"
-
-                self.logger.debug(
-                    "Audio stream: codec=%s sample_rate=%s layout=%s", codec_name, sample_rate, layout_name
-                )
-
-                if codec_name in copy_ok:
-                    out_audio = output.add_stream_from_template(template=in_audio)
-                    self.logger.info("Audio codec MP4-compatible; remuxing without transcoding.")
-                else:
-                    transcode_audio = True
-                    sample_rate = audio_ctx.sample_rate or 48000
-                    layout = audio_ctx.layout.name if getattr(audio_ctx, "layout", None) else "stereo"
-
-                    out_audio = output.add_stream("aac", rate=sample_rate)
-                    self.logger.info("Transcoding audio to AAC: sample_rate=%s layout=%s"), sample_rate, layout
-
-                    try:
-                        out_audio.layout = layout
-                    except Exception as exc:
-                        self.logger.warning("Failed to set audio layout while remuxing %s to %s: %s", input_path, output_path, exc, exc_info=True)
-                        pass
-
-                    resampler = AudioResampler(format="fltp", layout=layout, rate=sample_rate)
-            else:
-                self.logger.info("No audio stream detected; remuxing video only.")
-
-            # --- DEMUX ---
-            demux_streams = [in_video] + ([in_audio] if in_audio else [])
-            packets = input_.demux(demux_streams)
+        self.logger.info("Using FFmpeg binary for TS to MP4 remux: %s", ffmpeg_bin)
+        metadata_written = False
+        has_meta = metadata is not None or thumbnail is not None
 
+        if has_meta:
             try:
-                total = os.path.getsize(input_path)
+                remux_ts_to_mp4(
+                    input_path,
+                    output_path,
+                    ffmpeg_bin=ffmpeg_bin,
+                    metadata=metadata,
+                    thumbnail=thumbnail,
+                )
+                metadata_written = True
             except Exception as exc:
-                self.logger.warning("Failed to read input size for remux progress: %s: %s", input_path, exc, exc_info=True)
-                total = 100
-
-            self.logger.info("Demuxing packets: total_bytes=%s", total)
-            progress_step = max(1, total // 10) if total else 0
-            next_progress_log = progress_step if progress_step else 0
-            current_progress = 0
-            timestamp_offsets: dict[int, int] = {}
-            last_dts: dict[int, int] = {}
-            last_durations: dict[int, int] = {}
-
-            for idx, packet in enumerate(packets):
-                pkt_size = getattr(packet, "size", 0) or 0
-                current_progress += pkt_size
-
-                if packet.dts is None:
-                    if callback:
-                        callback(current_progress, total)
-                    continue
-
-                timestamp_correction = _normalize_packet_timestamps(
-                    packet,
-                    timestamp_offsets,
-                    last_dts,
-                    last_durations,
+                self.logger.warning(
+                    "Single-pass remux with metadata failed (%s); falling back to plain remux.",
+                    exc,
                 )
-                if timestamp_correction:
-                    self.logger.info(
-                        "Normalized HLS timestamp discontinuity: stream=%s correction=%s time_base=%s",
-                        packet.stream.index,
-                        timestamp_correction,
-                        packet.time_base,
-                    )
-
-                if packet.stream == in_video:
-                    packet.stream = out_video
-                    output.mux(packet)
+                self._safe_remove(output_path)
 
-                elif in_audio and packet.stream == in_audio:
-                    if not transcode_audio:
-                        packet.stream = out_audio
-                        output.mux(packet)
-                    else:
-                        assert out_audio is not None
-                        for frame in packet.decode():
-                            # Fix 4: Ensure the frame is recognized as an AudioFrame
-                            if not isinstance(frame, av.audio.frame.AudioFrame):
-                                continue
-
-                            frames = resampler.resample(frame) if resampler else [frame]
-                            for f in frames:
-                                for enc_pkt in out_audio.encode(f):
-                                    output.mux(enc_pkt)
-
-                if callback:
-                    callback(current_progress, total)
-                if progress_step and current_progress >= next_progress_log:
-                    self.logger.debug("Remux progress: bytes=%s/%s", current_progress, total)
-                    next_progress_log += progress_step
-
-            if transcode_audio and out_audio:
-                self.logger.debug("Flushing AAC encoder.")
-                for enc_pkt in out_audio.encode(None):
-                    output.mux(enc_pkt)
-
-            input_.close()
-            output.close()
-            elapsed = time.perf_counter() - start_ts
+        if not metadata_written:
+            remux_ts_to_mp4(
+                input_path,
+                output_path,
+                ffmpeg_bin=ffmpeg_bin,
+            )
 
+        if callback:
             try:
-                out_size = os.path.getsize(output_path)
-                self.logger.info("Remux complete: output=%s size=%s bytes elapsed=%s.2f", output_path, out_size,
-                                 elapsed)
-            except Exception as e:
-                self.logger.info("Remux complete: output=%s elapsed=%s.2fs (size unavailable: %s)", output_path,
-                                 elapsed, e, exc_info=True)
+                total = input_size or os.path.getsize(output_path)
+                callback(total, total)
+            except Exception:
+                pass
 
-        else:
-            self.logger.info("Stream seems to be already in MP4! Skipping remux...")
-            os.rename(input_path, output_path)
-            elapsed = time.perf_counter() - start_ts
-            self.logger.info("Remux skipped; file moved. elapsed=%s.2f", elapsed)
+        elapsed = time.perf_counter() - start_ts
+        self.logger.info(
+            "Remux complete: output=%s metadata_written=%s elapsed=%.2fs",
+            output_path,
+            metadata_written,
+            elapsed,
+        )
+        return metadata_written
 
     async def legacy_download(self, url: str, configuration: DownloadConfigRAW) -> bool:
         """Download a raw stream, preserving the URL and original failure traceback."""

{eaf_base_api-4.2.0 → eaf_base_api-4.2.2}/base_api/modules/config.py

@@ -40,6 +40,7 @@ class RuntimeConfig:
         self.videos_concurrency: int = 5
         self.pages_concurrency: int = 2
         self.interface: str | None = None # IP Address of the network interface you want to bind to
+        self.ffmpeg_path: str | None = None # Optional explicit path to an FFmpeg executable
         self.ip_resolve: int | None = (
             int(os.environ["CURL_IPRESOLVE"])
             if "CURL_IPRESOLVE" in os.environ and os.environ["CURL_IPRESOLVE"].isdigit()
@@ -57,6 +58,9 @@ class BaseConfigDownload:
     callback: callback_hint = None
     no_title: bool = False
     stop_event: asyncio.Event | None = None
+    metadata: Any = None
+    thumbnail: Any = None
+    metadata_written: bool = False
 
 
 @dataclass

{eaf_base_api-4.2.0 → eaf_base_api-4.2.2}/base_api/modules/errors.py

@@ -388,6 +388,12 @@ class DownloadFailed(ScraperException):
     pass
 
 
+class MetadataWriteError(BaseScraperError):
+    """Raised when writing MP4 metadata tags or cover art fails."""
+    pass
+
+
+
 class VideoUnavailable(ScraperException):
     """Raised when a video is unavailable or has been removed."""
     pass

eaf_base_api-4.2.2/base_api/modules/ffmpeg.py

@@ -0,0 +1,359 @@
+"""FFmpeg binary discovery, MPEG-TS to MP4 remuxing, and MP4 metadata tagging."""
+from __future__ import annotations
+
+import inspect
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from collections.abc import Mapping
+from datetime import date, datetime
+from pathlib import Path
+from typing import Any
+
+from base_api.modules.config import config
+from base_api.modules.errors import MetadataWriteError
+
+logger = logging.getLogger("base_api.ffmpeg")
+
+
+def _find_original_executable() -> Path | None:
+    """Return the true executable path, ignoring onefile or temp extractions."""
+    try:
+        import __compiled__  # type: ignore[import-not-found]
+        orig = getattr(__compiled__, "original_argv0", None)
+        if orig and Path(orig).exists():
+            return Path(orig).resolve()
+    except (ImportError, OSError, TypeError):
+        pass
+
+    if appimage := os.environ.get("APPIMAGE"):
+        if Path(appimage).exists():
+            return Path(appimage).resolve()
+
+    if sys.argv and sys.argv[0] and Path(sys.argv[0]).exists():
+        return Path(sys.argv[0]).resolve()
+
+    return None
+
+
+def _is_executable_file(candidate: Path) -> bool:
+    return candidate.is_file() and (sys.platform == "win32" or os.access(candidate, os.X_OK))
+
+
+def find_ffmpeg(
+    ffmpeg_bin: Path | str | None = None,
+    *,
+    original_executable: Path | None = None,
+    extra_roots: tuple[Path, ...] = (),
+) -> Path | None:
+    """Discover an explicit, bundled, or system FFmpeg executable."""
+    binary_name = "ffmpeg.exe" if sys.platform == "win32" else "ffmpeg"
+
+    # 1. Explicit argument, runtime config, or environment variable override
+    for explicit in (
+        ffmpeg_bin,
+        getattr(config, "ffmpeg_path", None),
+        os.environ.get("PORN_FETCH_FFMPEG_BINARY"),
+        os.environ.get("EAF_FFMPEG_BINARY"),
+    ):
+        if explicit:
+            candidate = Path(explicit).expanduser()
+            if _is_executable_file(candidate):
+                return candidate.resolve()
+
+    candidates: list[Path] = []
+
+    # 2. Paths relative to current running executable
+    orig = original_executable if original_executable is not None else _find_original_executable()
+    if orig is not None:
+        exec_dir = orig.parent
+        candidates.append(exec_dir / "third_party" / binary_name)
+        candidates.append(exec_dir / binary_name)
+        # macOS App bundle: Contents/MacOS/
+        candidates.append(exec_dir / "Contents" / "MacOS" / "third_party" / binary_name)
+        candidates.append(exec_dir / "Contents" / "MacOS" / binary_name)
+        if exec_dir.name == "MacOS" and exec_dir.parent.name == "Contents":
+            candidates.append(exec_dir.parent.parent.parent / "third_party" / binary_name)
+
+    # 3. Project / caller roots and working directory
+    roots: list[Path] = list(extra_roots)
+    roots.append(Path.cwd())
+    roots.append(Path(__file__).resolve().parents[2])
+    if sys.executable:
+        roots.append(Path(sys.executable).resolve().parent)
+
+    for root in roots:
+        candidates.append(root / "third_party" / binary_name)
+        candidates.append(root / binary_name)
+
+    for cand in candidates:
+        if _is_executable_file(cand):
+            return cand.resolve()
+
+    # 4. System PATH fallback
+    if in_path := shutil.which(binary_name) or shutil.which("ffmpeg"):
+        p = Path(in_path)
+        if p.is_file():
+            return p.resolve()
+
+    return None
+
+
+def has_ffmpeg(
+    ffmpeg_bin: Path | str | None = None,
+    *,
+    original_executable: Path | None = None,
+    extra_roots: tuple[Path, ...] = (),
+) -> bool:
+    """Return True if an FFmpeg binary is available on the system or bundled."""
+    return (
+        find_ffmpeg(
+            ffmpeg_bin,
+            original_executable=original_executable,
+            extra_roots=extra_roots,
+        )
+        is not None
+    )
+
+
+def _safe_getattr(obj: Any, name: str) -> Any:
+    """Safely read an attribute without triggering unawaited coroutine warnings or lazy-load errors."""
+    try:
+        val = getattr(obj, name, None)
+    except Exception:
+        return None
+    if inspect.iscoroutine(val):
+        val.close()
+        return None
+    if callable(val):
+        return None
+    return val
+
+
+def _format_metadata_value(value: Any) -> str | None:
+    if value is None:
+        return None
+    if isinstance(value, (datetime, date)):
+        return value.isoformat()
+    if isinstance(value, Mapping):
+        name = value.get("name") or value.get("username") or value.get("title")
+        return _format_metadata_value(name)
+    name_attr = _safe_getattr(value, "name") or _safe_getattr(value, "username")
+    if name_attr is not None and not isinstance(value, (str, bytes)):
+        return _format_metadata_value(name_attr)
+    text = str(value).strip()
+    return text if text else None
+
+
+def _extract_metadata_fields(data: Any) -> dict[str, str]:
+    """Extract normalized FFmpeg metadata key-value pairs from an object or mapping."""
+    if data is None:
+        return {}
+
+    if isinstance(data, Mapping):
+        raw_fields = {
+            "title": data.get("title"),
+            "artist": data.get("artist") if "artist" in data else data.get("author"),
+            "genre": data.get("genre", "XXX"),
+            "date": data.get("date") if "date" in data else data.get("publish_date"),
+        }
+        for key, val in data.items():
+            if key not in {"title", "artist", "author", "genre", "date", "publish_date", "thumbnail_data", "thumbnail"}:
+                raw_fields[str(key)] = val
+    else:
+        artist = (
+            _safe_getattr(data, "artist")
+            or _safe_getattr(data, "author")
+            or _safe_getattr(data, "author_information")
+        )
+        raw_fields = {
+            "title": _safe_getattr(data, "title"),
+            "artist": artist,
+            "genre": _safe_getattr(data, "genre") or "XXX",
+            "date": _safe_getattr(data, "date") or _safe_getattr(data, "publish_date"),
+        }
+
+    resolved: dict[str, str] = {}
+    for key, val in raw_fields.items():
+        formatted = _format_metadata_value(val)
+        if formatted is not None:
+            resolved[key] = formatted
+    return resolved
+
+
+def _extract_thumbnail_bytes(data: Any, explicit_thumbnail: bytes | Path | str | None = None) -> bytes | None:
+    candidate = explicit_thumbnail
+    if candidate is None and data is not None:
+        if isinstance(data, Mapping):
+            candidate = data.get("thumbnail_data") or data.get("thumbnail")
+        else:
+            candidate = _safe_getattr(data, "thumbnail_data")
+
+    if isinstance(candidate, (str, Path)):
+        p = Path(candidate)
+        if p.is_file():
+            return p.read_bytes()
+        return None
+
+    if isinstance(candidate, bytes) and len(candidate) > 0:
+        return candidate
+    return None
+
+
+def remux_ts_to_mp4(
+    input_path: Path | str,
+    output_path: Path | str,
+    ffmpeg_bin: Path | str | None = None,
+    *,
+    metadata: Any = None,
+    thumbnail: bytes | Path | str | None = None,
+    extra_roots: tuple[Path, ...] = (),
+) -> bool:
+    """
+    Remux an MPEG-TS container into a faststart MP4 container using FFmpeg.
+
+    When *metadata* or *thumbnail* is supplied, tags and attached cover art are
+    written in the same single FFmpeg pass to avoid a second full-file rewrite.
+    """
+    ffmpeg = find_ffmpeg(ffmpeg_bin, extra_roots=extra_roots)
+    if not ffmpeg:
+        raise FileNotFoundError("FFmpeg executable not found. Install FFmpeg or bundle it in third_party/.")
+
+    in_p = Path(input_path).resolve()
+    out_p = Path(output_path).resolve()
+    out_p.parent.mkdir(parents=True, exist_ok=True)
+
+    thumb_bytes = _extract_thumbnail_bytes(metadata, thumbnail)
+    meta_fields = _extract_metadata_fields(metadata)
+    temporary_thumb: Path | None = None
+
+    try:
+        cmd = [
+            str(ffmpeg),
+            "-y",
+            "-v", "error",
+            "-i", str(in_p),
+        ]
+
+        if thumb_bytes:
+            suffix = ".png" if thumb_bytes.startswith(b"\x89PNG\r\n\x1a\n") else ".jpg"
+            with tempfile.NamedTemporaryFile(dir=out_p.parent, delete=False, suffix=suffix) as tf:
+                tf.write(thumb_bytes)
+                temporary_thumb = Path(tf.name)
+
+            cmd.extend([
+                "-i", str(temporary_thumb),
+                "-map", "0:v:0",
+                "-map", "0:a?",
+                "-map", "1:v:0",
+                "-c", "copy",
+                "-disposition:v:1", "attached_pic",
+            ])
+        else:
+            cmd.extend(["-c", "copy"])
+
+        for key, val in meta_fields.items():
+            cmd.extend(["-metadata", f"{key}={val}"])
+
+        cmd.extend([
+            "-movflags", "+faststart",
+            str(out_p),
+        ])
+
+        proc = subprocess.run(cmd, capture_output=True, text=True)
+        if proc.returncode != 0:
+            raise RuntimeError(f"FFmpeg stream-copy remux failed: {proc.stderr.strip()}")
+
+        return True
+    finally:
+        if temporary_thumb and temporary_thumb.exists():
+            try:
+                temporary_thumb.unlink(missing_ok=True)
+            except OSError:
+                pass
+
+
+def write_metadata_tags(
+    path: str | Path,
+    data: Any,
+    ffmpeg_bin: Path | str | None = None,
+    *,
+    thumbnail: bytes | Path | str | None = None,
+    extra_roots: tuple[Path, ...] = (),
+) -> bool:
+    """Write tags and optional cover thumbnail into an existing MP4 file using FFmpeg."""
+    ffmpeg = find_ffmpeg(ffmpeg_bin, extra_roots=extra_roots)
+    if not ffmpeg:
+        logger.warning("FFmpeg not found; skipping MP4 metadata tagging.")
+        return False
+
+    target_path = Path(path).resolve()
+    if not target_path.is_file():
+        return False
+
+    directory = target_path.parent
+    temporary_output = directory / f".tmp_meta_{target_path.name}"
+    temporary_thumb: Path | None = None
+
+    thumb_bytes = _extract_thumbnail_bytes(data, thumbnail)
+    meta_fields = _extract_metadata_fields(data)
+
+    try:
+        cmd = [
+            str(ffmpeg),
+            "-y",
+            "-v", "error",
+            "-i", str(target_path),
+        ]
+
+        if thumb_bytes:
+            suffix = ".png" if thumb_bytes.startswith(b"\x89PNG\r\n\x1a\n") else ".jpg"
+            with tempfile.NamedTemporaryFile(dir=directory, delete=False, suffix=suffix) as tf:
+                tf.write(thumb_bytes)
+                temporary_thumb = Path(tf.name)
+
+            cmd.extend([
+                "-i", str(temporary_thumb),
+                "-map", "0:v:0",
+                "-map", "0:a?",
+                "-map", "1:v:0",
+                "-c", "copy",
+                "-disposition:v:1", "attached_pic",
+            ])
+        else:
+            cmd.extend([
+                "-map", "0",
+                "-c", "copy",
+            ])
+
+        for key, val in meta_fields.items():
+            cmd.extend(["-metadata", f"{key}={val}"])
+
+        cmd.extend(["-movflags", "+faststart", str(temporary_output)])
+
+        proc = subprocess.run(cmd, capture_output=True, text=True)
+        if proc.returncode != 0:
+            raise MetadataWriteError(f"FFmpeg tagging failed: {proc.stderr.strip()}")
+
+        os.replace(temporary_output, target_path)
+        return True
+
+    except Exception as error:
+        if isinstance(error, MetadataWriteError):
+            raise
+        raise MetadataWriteError(str(error)) from error
+
+    finally:
+        if temporary_output.exists():
+            try:
+                temporary_output.unlink(missing_ok=True)
+            except OSError:
+                pass
+        if temporary_thumb and temporary_thumb.exists():
+            try:
+                temporary_thumb.unlink(missing_ok=True)
+            except OSError:
+                pass

{eaf_base_api-4.2.0 → eaf_base_api-4.2.2}/base_api/modules/provider.py

@@ -7,6 +7,7 @@ from functools import wraps
 
 from base_api.modules import errors
 from base_api.modules.logger import class_name, get_logger, log_context
+from base_api.modules.urls import same_site_url
 
 
 async def fetch_content(core, url: str, *, logger, owner=None, get_json=False,
@@ -20,6 +21,10 @@ async def fetch_content(core, url: str, *, logger, owner=None, get_json=False,
     with log_context(owner, url):
         try:
             content = await core.fetch_text(url)
+            resolved = getattr(content, "url", None)
+            if (isinstance(resolved, str) and owner is not None
+                    and getattr(owner, "url", None) == url and same_site_url(url, resolved)):
+                owner.resolved_url = resolved
             return json.loads(content, strict=False) if get_json else content
         except errors.DownloadCancelled:
             raise

{eaf_base_api-4.2.0 → eaf_base_api-4.2.2}/base_api/modules/type_hints.py

@@ -36,6 +36,7 @@ class DownloadReport:
     segment_state_path: Path | str | None
     start_segment: int
     quality: str | int
+    metadata_written: bool = False
 
     def __getitem__(self, key: str) -> Any:
         return getattr(self, key)

eaf_base_api-4.2.2/base_api/modules/urls.py

@@ -0,0 +1,51 @@
+"""Small URL operations shared by provider pagination and profile adapters."""
+
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+
+def update_url_query(url: str, **changes: str | int | None) -> str:
+    """Replace named query keys without losing filters or treating anchors as HTTP."""
+    parsed = urlsplit(url)
+    pairs = [(key, value) for key, value in parse_qsl(parsed.query, keep_blank_values=True) if key not in changes]
+    pairs.extend((key, str(value)) for key, value in changes.items() if value is not None)
+    return urlunsplit(parsed._replace(query=urlencode(pairs), fragment=""))
+
+
+def replace_url_path(url: str, path: str) -> str:
+    parsed = urlsplit(url)
+    return urlunsplit(parsed._replace(path=path, fragment=""))
+
+
+def append_url_path(url: str, *parts: str | int) -> str:
+    path = urlsplit(url).path.rstrip("/")
+    return replace_url_path(url, path + "/" + "/".join(str(part).strip("/") for part in parts))
+
+
+def profile_base_url(url: str, *, depth: int = 2) -> str:
+    """Remove a provider's tab/page suffix, retaining its host, slug and filters."""
+    parsed = urlsplit(url)
+    parts = [part for part in parsed.path.split("/") if part]
+    if len(parts) < depth:
+        raise ValueError("An individual profile URL with a name or ID is required")
+    return update_url_query(replace_url_path(url, "/" + "/".join(parts[:depth])), page=None)
+
+
+def same_site_url(original: str, resolved: str) -> bool:
+    """Accept response metadata on the same site, including localized subdomains."""
+    before, after = urlsplit(original), urlsplit(resolved)
+    if after.scheme not in {"http", "https"} or not before.hostname or not after.hostname:
+        return False
+    # The supported providers use these two-label domains, except RedTube Brazil.
+    depth = 3 if before.hostname.endswith(".com.br") else 2
+    return before.hostname.split(".")[-depth:] == after.hostname.split(".")[-depth:]
+
+
+class FetchedText(str):
+    """A string that retains response identity through text caches and parsers."""
+
+    url: str
+
+    def __new__(cls, content: str, *, url: str):
+        value = super().__new__(cls, content)
+        value.url = url
+        return value

eaf_base_api-4.2.2/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["uv_build>=0.9.30,<0.10.0"]
+build-backend = "uv_build"
+
+[project]
+name = "eaf_base_api"
+version = "4.2.2"
+description = "A base API for EchterAlsFake's Porn APIs"
+requires-python = ">=3.12"
+license = "AGPL-3.0-or-later"
+license-files = ["LICENSE*"]
+classifiers = ["Programming Language :: Python"]
+dependencies = [
+    "cachetools>=7.0.0",
+    "curl_cffi",
+    "tenacity>=9.1.2",
+]
+
+[project.readme]
+file = "README.md"
+content-type = "text/markdown"
+
+[[project.authors]]
+name = "Johannes Habel"
+email = "EchterAlsFake@proton.me"
+
+[project.optional-dependencies]
+hls = ["m3u8"]
+
+[project.urls]
+Homepage = "https://github.com/EchterAlsFake/eaf_base_api"
+Repository = "https://github.com/EchterAlsFake/eaf_base_api"
+
+[tool.uv]
+publish-url = "https://upload.pypi.org/legacy/"
+
+[tool.uv.build-backend]
+module-root = ""
+module-name = "base_api"
+
+[dependency-groups]
+dev = [
+    "mypy",
+    "types-m3u8",
+    "pylint",
+    "pytest",
+    "pytest-asyncio",
+]

eaf_base_api-4.2.0/pyproject.toml → eaf_base_api-4.2.2/pyproject.toml.orig

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "eaf_base_api"
-version = "4.2.0"
+version = "4.2.2"
 description = "A base API for EchterAlsFake's Porn APIs"
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.12"
@@ -28,8 +28,6 @@ dependencies = [
 [project.optional-dependencies]
 hls = [
   "m3u8",
-  # av needs Python >= 3.10, so guard it (otherwise `pip install ...[hls]` fails on 3.8/3.9)
-  "av; python_version >= '3.12'",
 ]
 
 [project.urls]