ytp-dl 0.6.9__py3-none-any.whl → 0.7.0__py3-none-any.whl

scripts/api.py

@@ -1,15 +1,21 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 
+import json
 import os
+import re
 import shutil
 import tempfile
 import time
 from threading import BoundedSemaphore, Lock
 
-from flask import Flask, request, send_file, jsonify
+from flask import Flask, Response, jsonify, request, send_file, stream_with_context
 
-from .downloader import validate_environment, download_video
+from .downloader import (
+    download_video,
+    download_video_stream,
+    validate_environment,
+)
 
 app = Flask(__name__)
 
@@ -30,6 +36,8 @@ STALE_JOB_TTL_S = int(os.environ.get("YTPDL_STALE_JOB_TTL_S", "3600"))
 
 _ALLOWED_EXTENSIONS = {"mp3", "mp4", "best"}
 
+_JOB_ID_RX = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
+
 
 def _cleanup_stale_jobs() -> None:
     now = time.time()
@@ -48,6 +56,14 @@ def _cleanup_stale_jobs() -> None:
         pass
 
 
+def _safe_job_id(raw: str | None) -> str:
+    s = (raw or "").strip()
+    if _JOB_ID_RX.match(s):
+        return s
+    # fallback to timestamp-based id (stable & filesystem-safe)
+    return str(int(time.time() * 1000))
+
+
 def _try_acquire_job_slot() -> bool:
     global _in_use
     if not _sem.acquire(blocking=False):
@@ -67,6 +83,11 @@ def _release_job_slot() -> None:
 
 @app.route("/api/download", methods=["POST"])
 def handle_download():
+    """
+    Backwards-compatible direct download endpoint:
+      - runs yt-dlp
+      - returns the file body directly
+    """
     _cleanup_stale_jobs()
 
     if not _try_acquire_job_slot():
@@ -145,6 +166,169 @@ def handle_download():
         return jsonify(error=f"Download failed: {str(e)}"), 500
 
 
+@app.route("/api/download_sse", methods=["POST"])
+def handle_download_sse():
+    """
+    New SSE endpoint:
+      - streams raw yt-dlp stdout lines in real-time (SSE)
+      - stores result.json with the final file path in a stable job dir
+      - releases concurrency slot immediately when yt-dlp completes
+      - client fetches the final file via /api/fetch/<job_id>
+    """
+    _cleanup_stale_jobs()
+
+    if not _try_acquire_job_slot():
+        return jsonify(error="Server busy, try again later"), 503
+
+    released = False
+
+    def _release_once() -> None:
+        nonlocal released
+        if not released:
+            released = True
+            _release_job_slot()
+
+    try:
+        data = request.get_json(force=True)
+        url = (data.get("url") or "").strip()
+        resolution = data.get("resolution")
+        extension = (data.get("extension") or "mp4").strip().lower()
+
+        if not url:
+            _release_once()
+            return jsonify(error="Missing 'url'"), 400
+
+        if extension not in _ALLOWED_EXTENSIONS:
+            _release_once()
+            return jsonify(
+                error=f"Invalid 'extension'. Allowed: {sorted(_ALLOWED_EXTENSIONS)}"
+            ), 400
+
+        job_id = _safe_job_id(data.get("job_id"))
+        job_dir = os.path.join(BASE_DOWNLOAD_DIR, job_id)
+        shutil.rmtree(job_dir, ignore_errors=True)
+        os.makedirs(job_dir, exist_ok=True)
+
+        result_path = os.path.join(job_dir, "result.json")
+        err_path = os.path.join(job_dir, "error.txt")
+
+        def gen():
+            file_path: str | None = None
+            try:
+                # optional "ready" marker
+                yield f"data: [vps_ready] {job_id}\n\n"
+
+                dl = download_video_stream(
+                    url=url,
+                    resolution=resolution,
+                    extension=extension,
+                    out_dir=job_dir,
+                )
+
+                # manually iterate so we can capture StopIteration.value (the final path)
+                while True:
+                    try:
+                        line = next(dl)
+                    except StopIteration as si:
+                        file_path = si.value
+                        break
+
+                    if line:
+                        yield f"data: {line}\n\n"
+
+                if not (file_path and os.path.exists(file_path)):
+                    raise RuntimeError("Download completed but output file not found")
+
+                # Persist result for /api/fetch/<job_id>
+                try:
+                    with open(result_path, "w", encoding="utf-8") as f:
+                        json.dump(
+                            {
+                                "job_id": job_id,
+                                "file_path": os.path.abspath(file_path),
+                                "basename": os.path.basename(file_path),
+                                "created_ts": time.time(),
+                            },
+                            f,
+                            ensure_ascii=False,
+                        )
+                except Exception:
+                    pass
+
+                # Release slot as soon as yt-dlp is done (do NOT hold slot for fetch transfer)
+                _release_once()
+
+                yield f"data: [vps_done] {os.path.basename(file_path)}\n\n"
+                yield "data: [vps_end]\n\n"
+                return
+
+            except Exception as e:
+                try:
+                    with open(err_path, "w", encoding="utf-8") as f:
+                        f.write(str(e))
+                except Exception:
+                    pass
+
+                _release_once()
+                msg = str(e)
+                # keep it as a single SSE line
+                yield f"data: [vps_error] {msg}\n\n"
+                yield "data: [vps_end]\n\n"
+                return
+
+            finally:
+                # ensure slot is released even if client disconnects mid-stream
+                _release_once()
+
+        resp = Response(stream_with_context(gen()), content_type="text/event-stream")
+        resp.headers["Cache-Control"] = "no-cache"
+        resp.headers["X-Accel-Buffering"] = "no"
+        return resp
+
+    except Exception as e:
+        _release_once()
+        return jsonify(error=f"Download failed: {str(e)}"), 500
+
+
+@app.route("/api/fetch/<job_id>", methods=["GET"])
+def fetch_job_file(job_id: str):
+    """
+    Fetch the finished file for a job_id created by /api/download_sse.
+    Cleans up the job dir after response is fully consumed.
+    """
+    job_id = _safe_job_id(job_id)
+    job_dir = os.path.join(BASE_DOWNLOAD_DIR, job_id)
+    result_path = os.path.join(job_dir, "result.json")
+
+    if not os.path.exists(result_path):
+        return jsonify(error="Job not found or not complete"), 404
+
+    try:
+        with open(result_path, "r", encoding="utf-8") as f:
+            meta = json.load(f)
+        file_path = meta.get("file_path") or ""
+    except Exception:
+        return jsonify(error="Job metadata unreadable"), 500
+
+    file_path = os.path.abspath(file_path)
+    if not file_path.startswith(os.path.abspath(job_dir) + os.sep):
+        return jsonify(error="Invalid job file path"), 500
+
+    if not os.path.exists(file_path):
+        return jsonify(error="Job file missing"), 404
+
+    response = send_file(file_path, as_attachment=True)
+
+    def _cleanup() -> None:
+        try:
+            shutil.rmtree(job_dir, ignore_errors=True)
+        except Exception:
+            pass
+
+    response.call_on_close(_cleanup)
+    return response
+
+
 @app.route("/healthz", methods=["GET"])
 def healthz():
     with _in_use_lock:

scripts/downloader.py

@@ -6,7 +6,8 @@ import shlex
 import shutil
 import subprocess
 import time
-from typing import Optional, List, Tuple
+from collections import deque
+from typing import Deque, Generator, List, Optional, Tuple
 
 # =========================
 # Config / constants
@@ -60,6 +61,13 @@ def _tail(out: str) -> str:
     return txt.strip()
 
 
+def _tail_deque(lines: Deque[str]) -> str:
+    txt = "\n".join(list(lines)[-_MAX_ERR_LINES:])
+    if len(txt) > _MAX_ERR_CHARS:
+        txt = txt[-_MAX_ERR_CHARS:]
+    return txt.strip()
+
+
 def _is_youtube_url(url: str) -> bool:
     u = (url or "").lower()
     return any(h in u for h in ("youtube.com", "youtu.be", "youtube-nocookie.com"))
@@ -129,14 +137,19 @@ def _common_flags() -> List[str]:
     # --no-playlist prevents accidental channel/playlist pulls (and disk blowups)
     return [
         "--no-playlist",
-        "--retries", "10",
-        "--fragment-retries", "10",
-        "--retry-sleep", "exp=1:30",
-        "--user-agent", MODERN_UA,
+        "--retries",
+        "10",
+        "--fragment-retries",
+        "10",
+        "--retry-sleep",
+        "exp=1:30",
+        "--user-agent",
+        MODERN_UA,
         "--no-cache-dir",
         "--ignore-config",
         "--embed-metadata",
-        "--sleep-interval", "1",
+        "--sleep-interval",
+        "1",
     ]
 
 
@@ -249,11 +262,14 @@ def _download_with_format(
 
     argv = [
         YTDLP_BIN,
-        "-f", fmt,
+        "-f",
+        fmt,
         *(_common_flags()),
-        "--output", out_tpl,
+        "--output",
+        out_tpl,
         # Ensure we can reliably pick the final output path.
-        "--print", "after_move:filepath",
+        "--print",
+        "after_move:filepath",
     ]
 
     if extract_mp3:
@@ -292,6 +308,108 @@ def _fmt_best(cap: int) -> str:
     return f"bv*[height<={cap}]+ba/b[height<={cap}]"
 
 
+def _download_with_format_stream(
+    url: str,
+    out_dir: str,
+    fmt: str,
+    merge_output_format: Optional[str] = None,
+    extract_mp3: bool = False,
+) -> Generator[str, None, str]:
+    """
+    Stream yt-dlp stdout lines in real-time.
+    Yields: raw stdout lines
+    Returns (StopIteration.value): absolute final output path
+    """
+    out_dir = os.path.abspath(out_dir)
+    os.makedirs(out_dir, exist_ok=True)
+    out_tpl = os.path.join(out_dir, "%(title)s.%(ext)s")
+
+    argv: List[str] = [
+        YTDLP_BIN,
+        "-f",
+        fmt,
+        *(_common_flags()),
+        # ensure progress is emitted as real lines (no carriage-return overwrites)
+        "--progress",
+        "--newline",
+        "--output",
+        out_tpl,
+        "--print",
+        "after_move:filepath",
+    ]
+
+    if extract_mp3:
+        argv.extend(["--extract-audio", "--audio-format", "mp3"])
+
+    if merge_output_format:
+        argv.extend(["--merge-output-format", merge_output_format])
+
+    argv.append(url)
+
+    tail_lines: Deque[str] = deque(maxlen=_MAX_ERR_LINES)
+    # Keep only a bounded capture for path extraction.
+    cap_lines: Deque[str] = deque(maxlen=2000)
+
+    proc = subprocess.Popen(
+        argv,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+        universal_newlines=True,
+    )
+
+    try:
+        assert proc.stdout is not None
+        for raw in iter(proc.stdout.readline, ""):
+            line = (raw or "").rstrip("\n")
+            if not line:
+                continue
+
+            tail_lines.append(line)
+            cap_lines.append(line)
+            yield line
+
+        proc.stdout.close()
+        rc = proc.wait()
+
+        out_joined = "\n".join(cap_lines)
+        path = _extract_final_path(out_joined, out_dir)
+
+        if path and os.path.exists(path):
+            return os.path.abspath(path)
+
+        # fallback: newest non-temp file in out_dir
+        try:
+            best_path = None
+            best_mtime = -1.0
+            for name in os.listdir(out_dir):
+                if name.endswith((".part", ".ytdl", ".tmp")):
+                    continue
+                full = os.path.join(out_dir, name)
+                if not os.path.isfile(full):
+                    continue
+                mt = os.path.getmtime(full)
+                if mt > best_mtime:
+                    best_mtime = mt
+                    best_path = full
+            if best_path and os.path.exists(best_path):
+                return os.path.abspath(best_path)
+        except Exception:
+            pass
+
+        if rc != 0:
+            raise RuntimeError(f"yt-dlp failed (format: {fmt})\n{_tail_deque(tail_lines)}")
+        raise RuntimeError(f"Download completed but output file not found (format: {fmt})\n{_tail_deque(tail_lines)}")
+
+    finally:
+        try:
+            if proc.poll() is None:
+                proc.kill()
+        except Exception:
+            pass
+
+
 # =========================
 # Public API
 # =========================
@@ -361,3 +479,71 @@ def download_video(
     finally:
         if _mullvad_present():
             _run_argv(["mullvad", "disconnect"], check=False)
+
+
+def download_video_stream(
+    url: str,
+    resolution: int | None = 1080,
+    extension: Optional[str] = None,
+    out_dir: str = DEFAULT_OUT_DIR,
+) -> Generator[str, None, str]:
+    """
+    Streaming variant for SSE usage.
+    Yields raw yt-dlp stdout lines; returns final file path as StopIteration.value.
+    """
+    if not url:
+        raise RuntimeError("Missing URL")
+
+    out_dir = os.path.abspath(out_dir)
+    os.makedirs(out_dir, exist_ok=True)
+
+    validate_environment()
+
+    require_mullvad_login()
+    mullvad_connect(MULLVAD_LOCATION)
+    if not mullvad_wait_connected():
+        raise RuntimeError("Mullvad connection failed")
+
+    try:
+        mode = (extension or "mp4").lower().strip()
+
+        if mode == "mp3":
+            return (yield from _download_with_format_stream(
+                url=url,
+                out_dir=out_dir,
+                fmt="bestaudio",
+                merge_output_format=None,
+                extract_mp3=True,
+            ))
+
+        cap = int(resolution or 1080)
+
+        if mode == "best":
+            try:
+                return (yield from _download_with_format_stream(
+                    url=url,
+                    out_dir=out_dir,
+                    fmt=_fmt_best(cap),
+                    merge_output_format=None,
+                    extract_mp3=False,
+                ))
+            except Exception:
+                return (yield from _download_with_format_stream(
+                    url=url,
+                    out_dir=out_dir,
+                    fmt=_fmt_mp4_apple_safe(cap),
+                    merge_output_format="mp4",
+                    extract_mp3=False,
+                ))
+
+        return (yield from _download_with_format_stream(
+            url=url,
+            out_dir=out_dir,
+            fmt=_fmt_mp4_apple_safe(cap),
+            merge_output_format="mp4",
+            extract_mp3=False,
+        ))
+
+    finally:
+        if _mullvad_present():
+            _run_argv(["mullvad", "disconnect"], check=False)

{ytp_dl-0.6.9.dist-info → ytp_dl-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ytp-dl
-Version: 0.6.9
+Version: 0.7.0
 Summary: YouTube video downloader with Mullvad VPN integration and Flask API
 Home-page: https://github.com/yourusername/ytp-dl
 Author: dumgum82
@@ -57,7 +57,7 @@ A lightweight YouTube downloader with Mullvad VPN integration and an HTTP API.
 ## Installation
 
 ```bash
-pip install ytp-dl==0.6.9 yt-dlp[default]
+pip install ytp-dl==0.7.0 yt-dlp[default]
 ```
 
 Requirements:
@@ -242,7 +242,7 @@ When Mullvad connects/disconnects, Linux routing can change in a way that breaks
 
 * Installs Python, FFmpeg, Mullvad CLI, and Deno
 * Creates a virtualenv at `/opt/yt-dlp-mullvad/venv`
-* Installs `ytp-dl==0.6.9` + `yt-dlp[default]` + `gunicorn`
+* Installs `ytp-dl==0.7.0` + `yt-dlp[default]` + `gunicorn`
 * Installs a policy-routing oneshot service to keep the public API reachable
 * Sets up a systemd service on port 5000
 * Runs Gunicorn with `gthread` (threaded) workers
@@ -258,7 +258,7 @@ Note: `gthread` is a built-in Gunicorn worker class (no extra Python dependency)
 #   - Installs Deno system-wide (JS runtime required for modern YouTube extraction via yt-dlp)
 #   - Configures policy routing so the public API stays reachable while Mullvad toggles
 #   - Creates a virtualenv at /opt/yt-dlp-mullvad/venv
-#   - Installs ytp-dl==0.6.9 + yt-dlp[default] + gunicorn in that venv
+#   - Installs ytp-dl==0.7.0 + yt-dlp[default] + gunicorn in that venv
 #   - Creates a systemd service ytp-dl-api.service on port 5000
 #
 # Mullvad connect/disconnect is handled per-job by downloader.py.
@@ -394,7 +394,7 @@ mkdir -p "${APP_DIR}"
 python3 -m venv "${VENV_DIR}"
 source "${VENV_DIR}/bin/activate"
 pip install --upgrade pip
-pip install "ytp-dl==0.6.9" "yt-dlp[default]" gunicorn
+pip install "ytp-dl==0.7.0" "yt-dlp[default]" gunicorn
 deactivate
 
 echo "==> 3) API environment file (/etc/default/ytp-dl-api)"

ytp_dl-0.7.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+scripts/__init__.py,sha256=EbAplfCcyLD3Q_9sxemm6owCc5_UJv53vmlxy810p2s,152
+scripts/api.py,sha256=Lwu-Bb1fg6fjKBECyNOtvt3Q6Kna5qwmYUT50X51vUk,10356
+scripts/downloader.py,sha256=Zpjoxcm8HJ2_j7xMXQysfEKXanOHKtbLAFwuYkzqc4w,16049
+ytp_dl-0.7.0.dist-info/METADATA,sha256=KG6IndrPKUI5xOzA3Ldx0iKTqBbwjoqAYDEIxVtXKdU,14547
+ytp_dl-0.7.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ytp_dl-0.7.0.dist-info/entry_points.txt,sha256=QqjqZZAEt3Y7RGrleqZ312sjjboUpbMLdo7qFxuCH30,48
+ytp_dl-0.7.0.dist-info/top_level.txt,sha256=rmzd5mewlrJy4sT608KPib7sM7edoY75AeqJeY3SPB4,8
+ytp_dl-0.7.0.dist-info/RECORD,,

ytp_dl-0.6.9.dist-info/RECORD

@@ -1,8 +0,0 @@
-scripts/__init__.py,sha256=EbAplfCcyLD3Q_9sxemm6owCc5_UJv53vmlxy810p2s,152
-scripts/api.py,sha256=cyLjHmelLwzh8-GOjqXsQdhm6wLX8bOkADZ_qU1naRQ,4331
-scripts/downloader.py,sha256=vvHasu-41DGPDUzOTA4kz52tijTkaii1NnuU4cHQxg8,10825
-ytp_dl-0.6.9.dist-info/METADATA,sha256=Z4H5XVsQIQgU3hFXdF-Lp2Yf6-sSzTjwpHAptrGeMYg,14547
-ytp_dl-0.6.9.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-ytp_dl-0.6.9.dist-info/entry_points.txt,sha256=QqjqZZAEt3Y7RGrleqZ312sjjboUpbMLdo7qFxuCH30,48
-ytp_dl-0.6.9.dist-info/top_level.txt,sha256=rmzd5mewlrJy4sT608KPib7sM7edoY75AeqJeY3SPB4,8
-ytp_dl-0.6.9.dist-info/RECORD,,