StreamingCommunity 3.3.9__py3-none-any.whl → 3.4.0__py3-none-any.whl

StreamingCommunity/Lib/Downloader/DASH/downloader.py

@@ -1,7 +1,6 @@
 # 25.07.25
 
 import os
-import time
 import shutil
 
 
@@ -13,7 +12,7 @@ from rich.table import Table
 
 # Internal utilities
 from StreamingCommunity.Util.config_json import config_manager
-from StreamingCommunity.Util.os import os_manager, internet_manager
+from StreamingCommunity.Util.os import os_manager, internet_manager, get_wvd_path
 from StreamingCommunity.Util.http_client import create_client
 from StreamingCommunity.Util.headers import get_userAgent
 
@@ -32,11 +31,11 @@ from ...FFmpeg import print_duration_table, join_audios, join_video, join_subtit
 # Config
 DOWNLOAD_SPECIFIC_AUDIO = config_manager.get_list('M3U8_DOWNLOAD', 'specific_list_audio')
 DOWNLOAD_SPECIFIC_SUBTITLE = config_manager.get_list('M3U8_DOWNLOAD', 'specific_list_subtitles')
-ENABLE_SUBTITLE = config_manager.get_bool('M3U8_DOWNLOAD', 'download_subtitle')
 MERGE_SUBTITLE = config_manager.get_bool('M3U8_DOWNLOAD', 'merge_subs')
 FILTER_CUSTOM_REOLUTION = str(config_manager.get('M3U8_CONVERSION', 'force_resolution')).strip().lower()
 CLEANUP_TMP = config_manager.get_bool('M3U8_DOWNLOAD', 'cleanup_tmp_folder')
 RETRY_LIMIT = config_manager.get_int('REQUESTS', 'max_retry')
+EXTENSION_OUTPUT = config_manager.get("M3U8_CONVERSION", "extension")
 
 
 # Variable
@@ -44,18 +43,17 @@ console = Console()
 
 
 class DASH_Downloader:
-    def __init__(self, cdm_device, license_url, mpd_url, mpd_sub_list: list = None, output_path: str = None):
+    def __init__(self, license_url, mpd_url, mpd_sub_list: list = None, output_path: str = None):
         """
         Initialize the DASH Downloader with necessary parameters.
 
         Parameters:
-            - cdm_device (str): Path to the CDM device for decryption.
             - license_url (str): URL to obtain the license for decryption.
             - mpd_url (str): URL of the MPD manifest file.
             - mpd_sub_list (list): List of subtitle dicts with keys: 'language', 'url', 'format'.
             - output_path (str): Path to save the final output file.
         """
-        self.cdm_device = cdm_device
+        self.cdm_device = get_wvd_path()
         self.license_url = license_url
         self.mpd_url = mpd_url
         self.mpd_sub_list = mpd_sub_list or []
@@ -196,49 +194,28 @@ class DASH_Downloader:
         Download subtitle files based on configuration with retry mechanism.
         Returns True if successful or if no subtitles to download, False on critical error.
         """
-        if not ENABLE_SUBTITLE or not self.selected_subs:
-            return True
-        
-        headers = {'User-Agent': get_userAgent()}
-        client = create_client(headers=headers)
+        client = create_client(headers={'User-Agent': get_userAgent()})
         
         for sub in self.selected_subs:
-            language = sub.get('language', 'unknown')
-            url = sub.get('url')
-            fmt = sub.get('format', 'vtt')
-            
-            if not url:
-                console.print(f"[yellow]Warning: No URL for subtitle {language}[/yellow]")
-                continue
-            
-            # Retry mechanism for downloading subtitles
-            success = False
-            for attempt in range(RETRY_LIMIT):
-                try:
-                    # Download subtitle
-                    response = client.get(url)
-                    response.raise_for_status()
-                    
-                    # Save subtitle file
-                    sub_filename = f"{language}.{fmt}"
-                    sub_path = os.path.join(self.subs_dir, sub_filename)
-                    
-                    with open(sub_path, 'wb') as f:
-                        f.write(response.content)
-                    
-                    success = True
-                    break
+            try:
+                language = sub.get('language', 'unknown')
+                fmt = sub.get('format', 'vtt')
+
+                # Download subtitle
+                response = client.get(sub.get('url'))
+                response.raise_for_status()
+                
+                # Save subtitle file and make request
+                sub_filename = f"{language}.{fmt}"
+                sub_path = os.path.join(self.subs_dir, sub_filename)
+                
+                with open(sub_path, 'wb') as f:
+                    f.write(response.content)
                     
-                except Exception as e:
-                    if attempt < RETRY_LIMIT - 1:
-                        console.print(f"[yellow]Attempt {attempt + 1}/{RETRY_LIMIT} failed for subtitle {language}: {e}. Retrying...[/yellow]")
-                        time.sleep(1.5 ** attempt)
-                    else:
-                        console.print(f"[yellow]Warning: Failed to download subtitle {language} after {RETRY_LIMIT} attempts: {e}[/yellow]")
+            except Exception as e:
+                console.print(f"[red]Error downloading subtitle {language}: {e}[/red]")
+                return False
             
-            if not success:
-                continue
-        
         return True
 
     def download_and_decrypt(self, custom_headers=None, custom_payload=None):
@@ -256,7 +233,6 @@ class DASH_Downloader:
 
         # Fetch keys immediately after obtaining PSSH
         if not self.parser.pssh:
-            console.print("[red]No PSSH found: segments are not encrypted, skipping decryption.")
             self.download_segments(clear=True)
             return True
 
@@ -316,7 +292,7 @@ class DASH_Downloader:
                 # Decrypt video
                 decrypted_path = os.path.join(self.decrypted_dir, "video.mp4")
                 result_path = decrypt_with_mp4decrypt(
-                    encrypted_path, KID, KEY, output_path=decrypted_path
+                    "Video", encrypted_path, KID, KEY, output_path=decrypted_path
                 )
 
                 if not result_path:
@@ -365,7 +341,7 @@ class DASH_Downloader:
                 # Decrypt audio
                 decrypted_path = os.path.join(self.decrypted_dir, "audio.mp4")
                 result_path = decrypt_with_mp4decrypt(
-                    encrypted_path, KID, KEY, output_path=decrypted_path
+                    f"Audio {audio_language}", encrypted_path, KID, KEY, output_path=decrypted_path
                 )
 
                 if not result_path:
@@ -381,9 +357,110 @@ class DASH_Downloader:
         return True
 
     def download_segments(self, clear=False):
-        # Download segments and concatenate them
-        # clear=True: no decryption needed
-        pass
+        """
+        Download video/audio segments without decryption (for clear content).
+        
+        Parameters:
+            clear (bool): If True, content is not encrypted and doesn't need decryption
+        """
+        if not clear:
+            console.print("[yellow]Warning: download_segments called with clear=False[/yellow]")
+            return False
+        
+        video_segments_count = 0
+        
+        # Download subtitles
+        self.download_subtitles()
+        
+        # Download video
+        video_rep = self.get_representation_by_type("video")
+        if video_rep:
+            encrypted_path = os.path.join(self.encrypted_dir, f"{video_rep['id']}_encrypted.m4s")
+            
+            # If m4s file doesn't exist, start downloading
+            if not os.path.exists(encrypted_path):
+                video_downloader = MPD_Segments(
+                    tmp_folder=self.encrypted_dir,
+                    representation=video_rep,
+                    pssh=self.parser.pssh
+                )
+                
+                try:
+                    result = video_downloader.download_streams(description="Video")
+                    
+                    # Store the video segment count for limiting audio
+                    video_segments_count = video_downloader.get_segments_count()
+                    
+                    # Check for interruption or failure
+                    if result.get("stopped"):
+                        self.stopped = True
+                        self.error = "Download interrupted"
+                        return False
+                    
+                    if result.get("nFailed", 0) > 0:
+                        self.error = f"Failed segments: {result['nFailed']}"
+                        return False
+                    
+                except Exception as ex:
+                    self.error = str(ex)
+                    console.print(f"[red]Error downloading video: {ex}[/red]")
+                    return False
+            
+            # NO DECRYPTION: just copy/move to decrypted folder
+            decrypted_path = os.path.join(self.decrypted_dir, "video.mp4")
+            if os.path.exists(encrypted_path) and not os.path.exists(decrypted_path):
+                shutil.copy2(encrypted_path, decrypted_path)
+
+        else:
+            self.error = "No video found"
+            console.print(f"[red]{self.error}[/red]")
+            return False
+        
+        # Download audio with segment limiting
+        audio_rep = self.get_representation_by_type("audio")
+        if audio_rep:
+            encrypted_path = os.path.join(self.encrypted_dir, f"{audio_rep['id']}_encrypted.m4s")
+            
+            # If m4s file doesn't exist, start downloading
+            if not os.path.exists(encrypted_path):
+                audio_language = audio_rep.get('language', 'Unknown')
+                
+                audio_downloader = MPD_Segments(
+                    tmp_folder=self.encrypted_dir,
+                    representation=audio_rep,
+                    pssh=self.parser.pssh,
+                    limit_segments=video_segments_count if video_segments_count > 0 else None
+                )
+                
+                try:
+                    result = audio_downloader.download_streams(description=f"Audio {audio_language}")
+                    
+                    # Check for interruption or failure
+                    if result.get("stopped"):
+                        self.stopped = True
+                        self.error = "Download interrupted"
+                        return False
+                    
+                    if result.get("nFailed", 0) > 0:
+                        self.error = f"Failed segments: {result['nFailed']}"
+                        return False
+                    
+                except Exception as ex:
+                    self.error = str(ex)
+                    console.print(f"[red]Error downloading audio: {ex}[/red]")
+                    return False
+            
+            # NO DECRYPTION: just copy/move to decrypted folder
+            decrypted_path = os.path.join(self.decrypted_dir, "audio.mp4")
+            if os.path.exists(encrypted_path) and not os.path.exists(decrypted_path):
+                shutil.copy2(encrypted_path, decrypted_path)
+                
+        else:
+            self.error = "No audio found"
+            console.print(f"[red]{self.error}[/red]")
+            return False
+        
+        return True
 
     def finalize_output(self):
         """
@@ -415,8 +492,8 @@ class DASH_Downloader:
             console.print("[red]Video file missing, cannot export[/red]")
             return None
         
-        # Merge subtitles if enabled and available
-        if MERGE_SUBTITLE and ENABLE_SUBTITLE and self.selected_subs:
+        # Merge subtitles if available
+        if MERGE_SUBTITLE and self.selected_subs:
 
             # Check which subtitle files actually exist
             existing_sub_tracks = []
@@ -455,7 +532,7 @@ class DASH_Downloader:
         
         # Handle failed sync case
         if use_shortest:
-            new_filename = output_file.replace(".mp4", "_failed_sync.mp4")
+            new_filename = output_file.replace(EXTENSION_OUTPUT, f"_failed_sync{EXTENSION_OUTPUT}")
             if os.path.exists(output_file):
                 os.rename(output_file, new_filename)
                 output_file = new_filename

StreamingCommunity/Lib/Downloader/DASH/parser.py

@@ -1,6 +1,7 @@
 # 25.07.25
 
 import re
+import logging
 from urllib.parse import urljoin
 import xml.etree.ElementTree as ET
 from typing import List, Dict, Optional, Tuple, Any
@@ -456,6 +457,7 @@ class MPDParser:
                 )
                 
                 response.raise_for_status()
+                logging.info(f"Successfully fetched MPD: {response.content}")
                 self.root = ET.fromstring(response.content)
                 break
 
@@ -480,9 +482,6 @@ class MPDParser:
                 self.pssh = pssh_element.text
                 break
 
-        if not self.pssh:
-            console.print("[bold red]PSSH not found in MPD![/bold red]")
-
     def _parse_representations(self) -> None:
         """Parse all representations from the MPD"""
         base_url = self._get_initial_base_url()

StreamingCommunity/Lib/Downloader/DASH/segments.py

@@ -24,6 +24,7 @@ DEFAULT_VIDEO_WORKERS = config_manager.get_int('M3U8_DOWNLOAD', 'default_video_w
 DEFAULT_AUDIO_WORKERS = config_manager.get_int('M3U8_DOWNLOAD', 'default_audio_workers')
 SEGMENT_MAX_TIMEOUT = config_manager.get_int("M3U8_DOWNLOAD", "segment_timeout")
 LIMIT_SEGMENT = config_manager.get_int('M3U8_DOWNLOAD', 'limit_segment')
+ENABLE_RETRY = config_manager.get_bool('M3U8_DOWNLOAD', 'enable_retry')
 
 
 # Variable
@@ -50,7 +51,8 @@ class MPD_Segments:
             self.limit_segments = LIMIT_SEGMENT if LIMIT_SEGMENT > 0 else None
         else:
             self.limit_segments = limit_segments
-            
+        
+        self.enable_retry = ENABLE_RETRY
         self.download_interrupted = False
         self.info_nFailed = 0
         
@@ -63,8 +65,8 @@ class MPD_Segments:
         self._last_progress_update = 0
         self._progress_update_interval = 0.1
         
-        # Segment tracking
-        self.segment_files = {}
+        # Segment tracking - store only metadata, not content
+        self.segment_status = {}  # {idx: {'downloaded': bool, 'size': int}}
         self.segments_lock = asyncio.Lock()
 
     def get_concat_path(self, output_dir: str = None):
@@ -114,7 +116,7 @@ class MPD_Segments:
 
     async def download_segments(self, output_dir: str = None, concurrent_downloads: int = None, description: str = "DASH"):
         """
-        Download and concatenate all segments (including init) asynchronously and in order.
+        Download segments to temporary files, then concatenate them in order.
         
         Parameters:
             - output_dir (str): Output directory for segments
@@ -150,7 +152,7 @@ class MPD_Segments:
         # Initialize estimator
         estimator = M3U8_Ts_Estimator(total_segments=len(segment_urls) + 1)
 
-        self.segment_files = {}
+        self.segment_status = {}
         self.downloaded_segments = set()
         self.info_nFailed = 0
         self.download_interrupted = False
@@ -166,18 +168,19 @@ class MPD_Segments:
                 # Download init segment
                 await self._download_init_segment(client, init_url, concat_path, estimator, progress_bar)
 
-                # Download all segments (first batch) - writes to temp files
+                # Download all segments to temp files
                 await self._download_segments_batch(
                     client, segment_urls, temp_dir, semaphore, REQUEST_MAX_RETRY, estimator, progress_bar
                 )
 
-                # Retry failed segments 
-                await self._retry_failed_segments(
-                    client, segment_urls, temp_dir, semaphore, REQUEST_MAX_RETRY, estimator, progress_bar
-                )
+                # Retry failed segments only if enabled
+                if self.enable_retry:
+                    await self._retry_failed_segments(
+                        client, segment_urls, temp_dir, semaphore, REQUEST_MAX_RETRY, estimator, progress_bar
+                    )
 
-                # Concatenate all segment files in order
-                await self._concatenate_segments(concat_path, len(segment_urls))
+                # Concatenate all segments IN ORDER
+                await self._concatenate_segments_in_order(temp_dir, concat_path, len(segment_urls))
 
         except KeyboardInterrupt:
             self.download_interrupted = True
@@ -225,29 +228,33 @@ class MPD_Segments:
 
     async def _download_segments_batch(self, client, segment_urls, temp_dir, semaphore, max_retry, estimator, progress_bar):
         """
-        Download a batch of segments and write them to temp files immediately.
+        Download segments to temporary files - write immediately to disk, not memory.
         """
         async def download_single(url, idx):
             async with semaphore:
                 headers = {'User-Agent': get_userAgent()}
+                temp_file = os.path.join(temp_dir, f"seg_{idx:06d}.tmp")
                 
                 for attempt in range(max_retry):
                     if self.download_interrupted:
-                        return idx, False, attempt
+                        return idx, False, attempt, 0
                         
                     try:
                         timeout = min(SEGMENT_MAX_TIMEOUT, 10 + attempt * 3)
                         resp = await client.get(url, headers=headers, follow_redirects=True, timeout=timeout)
 
-                        # Write to temp file immediately
+                        # Write directly to temp file
                         if resp.status_code == 200:
-                            temp_file = os.path.join(temp_dir, f"seg_{idx:06d}.tmp")
+                            content_size = len(resp.content)
+                            with open(temp_file, 'wb') as f:
+                                f.write(resp.content)
+                            
+                            # Update status
                             async with self.segments_lock:
-                                with open(temp_file, 'wb') as f:
-                                    f.write(resp.content)
-                                self.segment_files[idx] = temp_file
+                                self.segment_status[idx] = {'downloaded': True, 'size': content_size}
+                                self.downloaded_segments.add(idx)
                             
-                            return idx, True, attempt, len(resp.content)
+                            return idx, True, attempt, content_size
                         else:
                             if attempt < 2:
                                 sleep_time = 0.5 + attempt * 0.5
@@ -258,19 +265,21 @@ class MPD_Segments:
                     except Exception:
                         sleep_time = min(2.0, 1.1 * (2 ** attempt))
                         await asyncio.sleep(sleep_time)
+                
+                # Mark as failed
+                async with self.segments_lock:
+                    self.segment_status[idx] = {'downloaded': False, 'size': 0}
                         
                 return idx, False, max_retry, 0
 
-        # Initial download attempt
+        # Download all segments concurrently
         tasks = [download_single(url, i) for i, url in enumerate(segment_urls)]
 
         for coro in asyncio.as_completed(tasks):
             try:
                 idx, success, nretry, size = await coro
                 
-                if success:
-                    self.downloaded_segments.add(idx)
-                else:
+                if not success:
                     self.info_nFailed += 1
                 
                 if nretry > self.info_maxRetry:
@@ -283,7 +292,7 @@ class MPD_Segments:
 
             except KeyboardInterrupt:
                 self.download_interrupted = True
-                print("\n[red]Download interrupted by user (Ctrl+C).")
+                console.print("\n[red]Download interrupted by user (Ctrl+C).")
                 break
 
     async def _retry_failed_segments(self, client, segment_urls, temp_dir, semaphore, max_retry, estimator, progress_bar):
@@ -301,6 +310,7 @@ class MPD_Segments:
             async def download_single(url, idx):
                 async with semaphore:
                     headers = {'User-Agent': get_userAgent()}
+                    temp_file = os.path.join(temp_dir, f"seg_{idx:06d}.tmp")
 
                     for attempt in range(max_retry):
                         if self.download_interrupted:
@@ -310,15 +320,17 @@ class MPD_Segments:
                             timeout = min(SEGMENT_MAX_TIMEOUT, 15 + attempt * 5)
                             resp = await client.get(url, headers=headers, timeout=timeout)
                             
-                            # Write to temp file immediately
+                            # Write directly to temp file
                             if resp.status_code == 200:
-                                temp_file = os.path.join(temp_dir, f"seg_{idx:06d}.tmp")
+                                content_size = len(resp.content)
+                                with open(temp_file, 'wb') as f:
+                                    f.write(resp.content)
+                                
                                 async with self.segments_lock:
-                                    with open(temp_file, 'wb') as f:
-                                        f.write(resp.content)
-                                    self.segment_files[idx] = temp_file
+                                    self.segment_status[idx] = {'downloaded': True, 'size': content_size}
+                                    self.downloaded_segments.add(idx)
                                 
-                                return idx, True, attempt, len(resp.content)
+                                return idx, True, attempt, content_size
                             else:
                                 await asyncio.sleep(1.5 * (2 ** attempt))
 
@@ -334,9 +346,7 @@ class MPD_Segments:
                 try:
                     idx, success, nretry, size = await coro
 
-                    if success:
-                        self.downloaded_segments.add(idx)
-                    else:
+                    if not success:
                         nFailed_this_round += 1
 
                     if nretry > self.info_maxRetry:
@@ -355,20 +365,24 @@ class MPD_Segments:
             self.info_nFailed = nFailed_this_round
             global_retry_count += 1
 
-    async def _concatenate_segments(self, concat_path, total_segments):
+    async def _concatenate_segments_in_order(self, temp_dir, concat_path, total_segments):
         """
-        Concatenate all segment files in order to the final output file.
-        Skip missing segments and continue with available ones.
+        Concatenate all segment files IN ORDER to the final output file.
         """
-        successful_segments = 0
         with open(concat_path, 'ab') as outfile:
             for idx in range(total_segments):
-                if idx in self.segment_files:
-                    temp_file = self.segment_files[idx]
-                    if os.path.exists(temp_file):
-                        with open(temp_file, 'rb') as infile:
-                            outfile.write(infile.read())
-                        successful_segments += 1
+                temp_file = os.path.join(temp_dir, f"seg_{idx:06d}.tmp")
+                
+                # Only concatenate successfully downloaded segments
+                if idx in self.downloaded_segments and os.path.exists(temp_file):
+                    with open(temp_file, 'rb') as infile:
+                        
+                        # Read and write in chunks to avoid memory issues
+                        while True:
+                            chunk = infile.read(8192)  # 8KB chunks
+                            if not chunk:
+                                break
+                            outfile.write(chunk)
 
     def _get_bar_format(self, description: str) -> str:
         """
@@ -435,7 +449,8 @@ class MPD_Segments:
         # Delete temp segment files
         if temp_dir and os.path.exists(temp_dir):
             try:
-                for temp_file in self.segment_files.values():
+                for idx in range(len(self.selected_representation.get('segment_urls', []))):
+                    temp_file = os.path.join(temp_dir, f"seg_{idx:06d}.tmp")
                     if os.path.exists(temp_file):
                         os.remove(temp_file)
                 os.rmdir(temp_dir)
@@ -447,7 +462,7 @@ class MPD_Segments:
             self._display_error_summary()
             
         # Clear memory
-        self.segment_files = {}
+        self.segment_status = {}
 
     def _display_error_summary(self) -> None:
         """
@@ -455,11 +470,8 @@ class MPD_Segments:
         """
         total_segments = len(self.selected_representation.get('segment_urls', []))
         failed_indices = [i for i in range(total_segments) if i not in self.downloaded_segments]
-        successful_segments = len(self.downloaded_segments)
-
-        console.print(f"[green]Download Summary: "
-              f"[cyan]Successful: [red]{successful_segments}/{total_segments} "
-              f"[cyan]Max retries: [red]{getattr(self, 'info_maxRetry', 0)} "
-              f"[cyan]Total retries: [red]{getattr(self, 'info_nRetry', 0)} "
-              f"[cyan]Failed segments: [red]{getattr(self, 'info_nFailed', 0)} "
-              f"[cyan]Failed indices: [red]{failed_indices} \n")
+
+        console.print(f" [cyan]Max retries: [red]{getattr(self, 'info_maxRetry', 0)} [white]| "
+            f"[cyan]Total retries: [red]{getattr(self, 'info_nRetry', 0)} [white]| "
+            f"[cyan]Failed segments: [red]{getattr(self, 'info_nFailed', 0)} [white]| "
+            f"[cyan]Failed indices: [red]{failed_indices}")