karaoke-gen 0.55.0__tar.gz → 0.57.0__tar.gz

{karaoke_gen-0.55.0 → karaoke_gen-0.57.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: karaoke-gen
-Version: 0.55.0
+Version: 0.57.0
 Summary: Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens.
 License: MIT
 Author: Andrew Beveridge
@@ -25,14 +25,16 @@ Requires-Dist: google-auth-httplib2
 Requires-Dist: google-auth-oauthlib
 Requires-Dist: kbputils (>=0.0.16,<0.0.17)
 Requires-Dist: lyrics-converter (>=0.2.1)
-Requires-Dist: lyrics-transcriber (>=0.54)
+Requires-Dist: lyrics-transcriber (>=0.61)
 Requires-Dist: lyricsgenius (>=3)
+Requires-Dist: modal (>=1.0.5,<2.0.0)
 Requires-Dist: numpy (>=2)
 Requires-Dist: pillow (>=10.1)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
 Requires-Dist: pyinstaller (>=6.3)
 Requires-Dist: pyperclip
-Requires-Dist: pytest-asyncio (>=0.23.5,<0.24.0)
+Requires-Dist: pytest-asyncio
+Requires-Dist: python-multipart (>=0.0.20,<0.0.21)
 Requires-Dist: requests (>=2)
 Requires-Dist: thefuzz (>=0.22)
 Requires-Dist: toml (>=0.10)

{karaoke_gen-0.55.0 → karaoke_gen-0.57.0}/karaoke_gen/audio_processor.py

@@ -72,11 +72,11 @@ class AudioProcessor:
 
         for file in output_files:
             if "(Vocals)" in file:
-                self.logger.info(f"Renaming Vocals file {file} to {vocals_path}")
-                os.rename(file, vocals_path)
+                self.logger.info(f"Moving Vocals file {file} to {vocals_path}")
+                shutil.move(file, vocals_path)
             elif "(Instrumental)" in file:
-                self.logger.info(f"Renaming Instrumental file {file} to {instrumental_path}")
-                os.rename(file, instrumental_path)
+                self.logger.info(f"Moving Instrumental file {file} to {instrumental_path}")
+                shutil.move(file, instrumental_path)
             elif model_name in file:
                 # Example filename 1: "Freddie Jackson - All I'll Ever Ask (feat. Najee) (Local)_(Piano)_htdemucs_6s.flac"
                 # Example filename 2: "Freddie Jackson - All I'll Ever Ask (feat. Najee) (Local)_(Guitar)_htdemucs_6s.flac"
@@ -86,8 +86,8 @@ class AudioProcessor:
                 stem_name = stem_name.strip("()")  # Remove parentheses if present
 
                 other_stem_path = os.path.join(track_output_dir, f"{artist_title} ({stem_name} {model_name}).{self.lossless_output_format}")
-                self.logger.info(f"Renaming other stem file {file} to {other_stem_path}")
-                os.rename(file, other_stem_path)
+                self.logger.info(f"Moving other stem file {file} to {other_stem_path}")
+                shutil.move(file, other_stem_path)
 
             elif model_name_no_extension in file:
                 # Example filename 1: "Freddie Jackson - All I'll Ever Ask (feat. Najee) (Local)_(Piano)_htdemucs_6s.flac"
@@ -98,8 +98,8 @@ class AudioProcessor:
                 stem_name = stem_name.strip("()")  # Remove parentheses if present
 
                 other_stem_path = os.path.join(track_output_dir, f"{artist_title} ({stem_name} {model_name}).{self.lossless_output_format}")
-                self.logger.info(f"Renaming other stem file {file} to {other_stem_path}")
-                os.rename(file, other_stem_path)
+                self.logger.info(f"Moving other stem file {file} to {other_stem_path}")
+                shutil.move(file, other_stem_path)
 
         self.logger.info(f"Separation complete! Output file(s): {vocals_path} {instrumental_path}")
 
@@ -262,10 +262,10 @@ class AudioProcessor:
 
             for file in clean_output_files:
                 if "(Vocals)" in file and not self._file_exists(vocals_path):
-                    os.rename(file, vocals_path)
+                    shutil.move(file, vocals_path)
                     result["vocals"] = vocals_path
                 elif "(Instrumental)" in file and not self._file_exists(instrumental_path):
-                    os.rename(file, instrumental_path)
+                    shutil.move(file, instrumental_path)
                     result["instrumental"] = instrumental_path
         else:
             result["vocals"] = vocals_path
@@ -298,7 +298,7 @@ class AudioProcessor:
                     new_filename = f"{artist_title} ({stem_name} {model}).{self.lossless_output_format}"
                     other_stem_path = os.path.join(stems_dir, new_filename)
                     if not self._file_exists(other_stem_path):
-                        os.rename(file, other_stem_path)
+                        shutil.move(file, other_stem_path)
                     result[model][stem_name] = other_stem_path
 
         return result
@@ -318,10 +318,10 @@ class AudioProcessor:
 
                 for file in backing_vocals_output:
                     if "(Vocals)" in file and not self._file_exists(lead_vocals_path):
-                        os.rename(file, lead_vocals_path)
+                        shutil.move(file, lead_vocals_path)
                         result[model]["lead_vocals"] = lead_vocals_path
                     elif "(Instrumental)" in file and not self._file_exists(backing_vocals_path):
-                        os.rename(file, backing_vocals_path)
+                        shutil.move(file, backing_vocals_path)
                         result[model]["backing_vocals"] = backing_vocals_path
             else:
                 result[model]["lead_vocals"] = lead_vocals_path

{karaoke_gen-0.55.0 → karaoke_gen-0.57.0}/karaoke_gen/file_handler.py

@@ -39,28 +39,64 @@ class FileHandler:
 
         return copied_file_name
 
-    def download_video(self, url, output_filename_no_extension):
+    def download_video(self, url, output_filename_no_extension, cookies_str=None):
         self.logger.debug(f"Downloading media from URL {url} to filename {output_filename_no_extension} + (as yet) unknown extension")
 
         ydl_opts = {
             "quiet": True,
             "format": "bv*+ba/b",  # if a combined video + audio format is better than the best video-only format use the combined format
             "outtmpl": f"{output_filename_no_extension}.%(ext)s",
-            "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
+            # Enhanced anti-detection options
+            "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            "referer": "https://www.youtube.com/",
+            "sleep_interval": 1,
+            "max_sleep_interval": 3,
+            "fragment_retries": 3,
+            "extractor_retries": 3,
+            "retries": 3,
+            # Headers to appear more human
+            "http_headers": {
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "en-us,en;q=0.5",
+                "Accept-Encoding": "gzip, deflate",
+                "DNT": "1",
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+            },
         }
 
-        with ydl(ydl_opts) as ydl_instance:
-            ydl_instance.download([url])
-
-            # Search for the file with any extension
-            downloaded_files = glob.glob(f"{output_filename_no_extension}.*")
-            if downloaded_files:
-                downloaded_file_name = downloaded_files[0]  # Assume the first match is the correct one
-                self.logger.info(f"Download finished, returning downloaded filename: {downloaded_file_name}")
-                return downloaded_file_name
-            else:
-                self.logger.error("No files found matching the download pattern.")
-                return None
+        # Add cookies if provided
+        if cookies_str:
+            self.logger.info("Using provided cookies for enhanced YouTube download access")
+            # Save cookies to a temporary file
+            import tempfile
+            with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f:
+                f.write(cookies_str)
+                ydl_opts['cookiefile'] = f.name
+        else:
+            self.logger.info("No cookies provided for download - attempting standard download")
+
+        try:
+            with ydl(ydl_opts) as ydl_instance:
+                ydl_instance.download([url])
+
+                # Search for the file with any extension
+                downloaded_files = glob.glob(f"{output_filename_no_extension}.*")
+                if downloaded_files:
+                    downloaded_file_name = downloaded_files[0]  # Assume the first match is the correct one
+                    self.logger.info(f"Download finished, returning downloaded filename: {downloaded_file_name}")
+                    return downloaded_file_name
+                else:
+                    self.logger.error("No files found matching the download pattern.")
+                    return None
+        finally:
+            # Clean up temporary cookie file if it was created
+            if cookies_str and 'cookiefile' in ydl_opts:
+                try:
+                    import os
+                    os.unlink(ydl_opts['cookiefile'])
+                except:
+                    pass
 
     def extract_still_image_from_video(self, input_filename, output_filename_no_extension):
         output_filename = output_filename_no_extension + ".png"

{karaoke_gen-0.55.0 → karaoke_gen-0.57.0}/karaoke_gen/karaoke_finalise/karaoke_finalise.py

@@ -44,6 +44,8 @@ class KaraokeFinalise:
         cdg_styles=None,
         keep_brand_code=False,
         non_interactive=False,
+        user_youtube_credentials=None,  # Add support for pre-stored credentials
+        server_side_mode=False,  # New parameter for server-side deployment
     ):
         self.log_level = log_level
         self.log_formatter = log_formatter
@@ -99,6 +101,8 @@ class KaraokeFinalise:
 
         self.skip_notifications = False
         self.non_interactive = non_interactive
+        self.user_youtube_credentials = user_youtube_credentials
+        self.server_side_mode = server_side_mode
 
         self.suffixes = {
             "title_mov": " (Title).mov",
@@ -146,6 +150,10 @@ class KaraokeFinalise:
         if self.non_interactive:
             self.ffmpeg_base_command += " -y"
 
+        # Detect and configure hardware acceleration
+        self.nvenc_available = self.detect_nvenc_support()
+        self.configure_hardware_acceleration()
+
     def check_input_files_exist(self, base_name, with_vocals_file, instrumental_audio_file):
         self.logger.info(f"Checking required input files exist...")
 
@@ -256,12 +264,17 @@ class KaraokeFinalise:
             self.discord_notication_enabled = True
 
         # Enable folder organisation if brand prefix and target directory are provided and target directory is valid
+        # In server-side mode, we skip the local folder organization but may still need brand codes
         if self.brand_prefix is not None and self.organised_dir is not None:
-            if not os.path.isdir(self.organised_dir):
+            if not self.server_side_mode and not os.path.isdir(self.organised_dir):
                 raise Exception(f"Target directory does not exist: {self.organised_dir}")
 
-            self.logger.debug(f"Brand prefix and target directory provided, enabling folder organisation")
-            self.folder_organisation_enabled = True
+            if not self.server_side_mode:
+                self.logger.debug(f"Brand prefix and target directory provided, enabling local folder organisation")
+                self.folder_organisation_enabled = True
+            else:
+                self.logger.debug(f"Server-side mode: brand prefix provided for remote organization")
+                self.folder_organisation_enabled = False  # Disable local folder organization in server mode
 
         # Enable public share copy if public share directory is provided and is valid directory with MP4 and CDG subdirectories
         if self.public_share_dir is not None:
@@ -292,18 +305,61 @@ class KaraokeFinalise:
         self.logger.info(f" Public share copy: {self.public_share_copy_enabled}")
         self.logger.info(f" Public share rclone: {self.public_share_rclone_enabled}")
 
-        self.prompt_user_confirmation_or_raise_exception(
-            f"Confirm features enabled log messages above match your expectations for finalisation?",
-            "Refusing to proceed without user confirmation they're happy with enabled features.",
-            allow_empty=True,
-        )
+        # Skip user confirmation in non-interactive mode for Modal deployment
+        if not self.non_interactive:
+            self.prompt_user_confirmation_or_raise_exception(
+                f"Confirm features enabled log messages above match your expectations for finalisation?",
+                "Refusing to proceed without user confirmation they're happy with enabled features.",
+                allow_empty=True,
+            )
+        else:
+            self.logger.info("Non-interactive mode: automatically confirming enabled features")
 
     def authenticate_youtube(self):
-        """Authenticate and return a YouTube service object."""
-        credentials = None
-        youtube_token_file = "/tmp/karaoke-finalise-youtube-token.pickle"
-
+        """Authenticate with YouTube and return service object."""
+        from google.auth.transport.requests import Request
+        from google.oauth2.credentials import Credentials
+        from googleapiclient.discovery import build
+        from google_auth_oauthlib.flow import InstalledAppFlow
+        import pickle
+        import os
+
+        # Check if we have pre-stored credentials (for non-interactive mode)
+        if self.user_youtube_credentials and self.non_interactive:
+            try:
+                # Create credentials object from stored data
+                credentials = Credentials(
+                    token=self.user_youtube_credentials['token'],
+                    refresh_token=self.user_youtube_credentials.get('refresh_token'),
+                    token_uri=self.user_youtube_credentials.get('token_uri'),
+                    client_id=self.user_youtube_credentials.get('client_id'),
+                    client_secret=self.user_youtube_credentials.get('client_secret'),
+                    scopes=self.user_youtube_credentials.get('scopes')
+                )
+                
+                # Refresh token if needed
+                if credentials.expired and credentials.refresh_token:
+                    credentials.refresh(Request())
+                
+                # Build YouTube service with credentials
+                youtube = build('youtube', 'v3', credentials=credentials)
+                self.logger.info("Successfully authenticated with YouTube using pre-stored credentials")
+                return youtube
+                
+            except Exception as e:
+                self.logger.error(f"Failed to authenticate with pre-stored credentials: {str(e)}")
+                # Fall through to original authentication if pre-stored credentials fail
+        
+        # Original authentication code for interactive mode
+        if self.non_interactive:
+            raise Exception("YouTube authentication required but running in non-interactive mode. Please pre-authenticate or disable YouTube upload.")
+        
         # Token file stores the user's access and refresh tokens for YouTube.
+        youtube_token_file = "/tmp/karaoke-finalise-youtube-token.pickle"
+        
+        credentials = None
+        
+        # Check if we have saved credentials
         if os.path.exists(youtube_token_file):
             with open(youtube_token_file, "rb") as token:
                 credentials = pickle.load(token)
@@ -313,6 +369,9 @@ class KaraokeFinalise:
             if credentials and credentials.expired and credentials.refresh_token:
                 credentials.refresh(Request())
             else:
+                if self.non_interactive:
+                    raise Exception("YouTube authentication required but running in non-interactive mode. Please pre-authenticate or disable YouTube upload.")
+                    
                 flow = InstalledAppFlow.from_client_secrets_file(
                     self.youtube_client_secrets_file, scopes=["https://www.googleapis.com/auth/youtube"]
                 )
@@ -631,74 +690,130 @@ class KaraokeFinalise:
         return base_name, artist, title
 
     def execute_command(self, command, description):
-        self.logger.info(description)
+        """Execute a shell command and log the output. For general commands (rclone, etc.)"""
+        self.logger.info(f"{description}")
+        self.logger.debug(f"Executing command: {command}")
+        
         if self.dry_run:
-            self.logger.info(f"DRY RUN: Would run command: {command}")
-        else:
-            self.logger.info(f"Running command: {command}")
-            os.system(command)
+            self.logger.info(f"DRY RUN: Would execute: {command}")
+            return
+        
+        try:
+            result = subprocess.run(command, shell=True, capture_output=True, text=True, timeout=600)
+            
+            # Log command output for debugging
+            if result.stdout and result.stdout.strip():
+                self.logger.debug(f"Command STDOUT: {result.stdout.strip()}")
+            if result.stderr and result.stderr.strip():
+                self.logger.debug(f"Command STDERR: {result.stderr.strip()}")
+            
+            if result.returncode != 0:
+                error_msg = f"Command failed with exit code {result.returncode}"
+                self.logger.error(error_msg)
+                self.logger.error(f"Command: {command}")
+                if result.stdout:
+                    self.logger.error(f"STDOUT: {result.stdout}")
+                if result.stderr:
+                    self.logger.error(f"STDERR: {result.stderr}")
+                raise Exception(f"{error_msg}: {command}")
+            else:
+                self.logger.info(f"✓ Command completed successfully")
+                
+        except subprocess.TimeoutExpired:
+            error_msg = f"Command timed out after 600 seconds"
+            self.logger.error(error_msg)
+            raise Exception(f"{error_msg}: {command}")
+        except Exception as e:
+            if "Command failed" not in str(e):
+                error_msg = f"Command failed with exception: {e}"
+                self.logger.error(error_msg)
+                raise Exception(f"{error_msg}: {command}")
+            else:
+                raise
 
     def remux_with_instrumental(self, with_vocals_file, instrumental_audio, output_file):
         """Remux the video with instrumental audio to create karaoke version"""
-        # fmt: off
+        # This operation is primarily I/O bound (remuxing), so hardware acceleration doesn't provide significant benefit
+        # Keep the existing approach but use the new execute method
         ffmpeg_command = (
             f'{self.ffmpeg_base_command} -an -i "{with_vocals_file}" '
             f'-vn -i "{instrumental_audio}" -c:v copy -c:a pcm_s16le "{output_file}"'
         )
-        # fmt: on
         self.execute_command(ffmpeg_command, "Remuxing video with instrumental audio")
 
     def convert_mov_to_mp4(self, input_file, output_file):
-        """Convert MOV file to MP4 format"""
-        # fmt: off
-        ffmpeg_command = (
+        """Convert MOV file to MP4 format with hardware acceleration support"""
+        # Hardware-accelerated version
+        gpu_command = (
+            f'{self.ffmpeg_base_command} {self.hwaccel_decode_flags} -i "{input_file}" '
+            f'-c:v {self.video_encoder} {self.get_nvenc_quality_settings("high")} -c:a {self.aac_codec} {self.mp4_flags} "{output_file}"'
+        )
+        
+        # Software fallback version
+        cpu_command = (
             f'{self.ffmpeg_base_command} -i "{input_file}" '
             f'-c:v libx264 -c:a {self.aac_codec} {self.mp4_flags} "{output_file}"'
         )
-        # fmt: on
-        self.execute_command(ffmpeg_command, "Converting MOV video to MP4")
+        
+        self.execute_command_with_fallback(gpu_command, cpu_command, "Converting MOV video to MP4")
 
     def encode_lossless_mp4(self, title_mov_file, karaoke_mp4_file, env_mov_input, ffmpeg_filter, output_file):
-        """Create the final MP4 with PCM audio (lossless)"""
-        # fmt: off
-        ffmpeg_command = (
+        """Create the final MP4 with PCM audio (lossless) using hardware acceleration when available"""
+        # Hardware-accelerated version
+        gpu_command = (
+            f"{self.ffmpeg_base_command} {self.hwaccel_decode_flags} -i {title_mov_file} "
+            f"{self.hwaccel_decode_flags} -i {karaoke_mp4_file} {env_mov_input} "
+            f'{ffmpeg_filter} -map "[outv]" -map "[outa]" -c:v {self.video_encoder} '
+            f'{self.get_nvenc_quality_settings("lossless")} -c:a pcm_s16le {self.mp4_flags} "{output_file}"'
+        )
+        
+        # Software fallback version
+        cpu_command = (
             f"{self.ffmpeg_base_command} -i {title_mov_file} -i {karaoke_mp4_file} {env_mov_input} "
             f'{ffmpeg_filter} -map "[outv]" -map "[outa]" -c:v libx264 -c:a pcm_s16le '
             f'{self.mp4_flags} "{output_file}"'
         )
-        # fmt: on
-        self.execute_command(ffmpeg_command, "Creating MP4 version with PCM audio")
+        
+        self.execute_command_with_fallback(gpu_command, cpu_command, "Creating MP4 version with PCM audio")
 
     def encode_lossy_mp4(self, input_file, output_file):
         """Create MP4 with AAC audio (lossy, for wider compatibility)"""
-        # fmt: off
+        # This is primarily an audio re-encoding operation, video is copied
+        # Hardware acceleration doesn't provide significant benefit for copy operations
         ffmpeg_command = (
             f'{self.ffmpeg_base_command} -i "{input_file}" '
             f'-c:v copy -c:a {self.aac_codec} -b:a 320k {self.mp4_flags} "{output_file}"'
         )
-        # fmt: on
         self.execute_command(ffmpeg_command, "Creating MP4 version with AAC audio")
 
     def encode_lossless_mkv(self, input_file, output_file):
         """Create MKV with FLAC audio (for YouTube)"""
-        # fmt: off
+        # This is primarily an audio re-encoding operation, video is copied
+        # Hardware acceleration doesn't provide significant benefit for copy operations
         ffmpeg_command = (
             f'{self.ffmpeg_base_command} -i "{input_file}" '
             f'-c:v copy -c:a flac "{output_file}"'
         )
-        # fmt: on
         self.execute_command(ffmpeg_command, "Creating MKV version with FLAC audio for YouTube")
 
     def encode_720p_version(self, input_file, output_file):
-        """Create 720p MP4 with AAC audio (for smaller file size)"""
-        # fmt: off
-        ffmpeg_command = (
+        """Create 720p MP4 with AAC audio (for smaller file size) using hardware acceleration when available"""
+        # Hardware-accelerated version with GPU scaling and encoding
+        gpu_command = (
+            f'{self.ffmpeg_base_command} {self.hwaccel_decode_flags} -i "{input_file}" '
+            f'-c:v {self.video_encoder} -vf "{self.scale_filter}=1280:720" '
+            f'{self.get_nvenc_quality_settings("medium")} -b:v 2000k '
+            f'-c:a {self.aac_codec} -b:a 128k {self.mp4_flags} "{output_file}"'
+        )
+        
+        # Software fallback version
+        cpu_command = (
             f'{self.ffmpeg_base_command} -i "{input_file}" '
-            f'-c:v libx264 -vf "scale=1280:720" -b:v 200k -preset medium -tune animation '
+            f'-c:v libx264 -vf "scale=1280:720" -b:v 2000k -preset medium -tune animation '
             f'-c:a {self.aac_codec} -b:a 128k {self.mp4_flags} "{output_file}"'
         )
-        # fmt: on
-        self.execute_command(ffmpeg_command, "Encoding 720p version of the final video")
+        
+        self.execute_command_with_fallback(gpu_command, cpu_command, "Encoding 720p version of the final video")
 
     def prepare_concat_filter(self, input_files):
         """Prepare the concat filter and additional input for end credits if present"""
@@ -749,17 +864,21 @@ class KaraokeFinalise:
         self.encode_lossless_mkv(output_files["final_karaoke_lossless_mp4"], output_files["final_karaoke_lossless_mkv"])
         self.encode_720p_version(output_files["final_karaoke_lossless_mp4"], output_files["final_karaoke_lossy_720p_mp4"])
 
-        # Prompt user to check final video files before proceeding
-        self.prompt_user_confirmation_or_raise_exception(
-            f"Final video files created:\n"
-            f"- Lossless 4K MP4: {output_files['final_karaoke_lossless_mp4']}\n"
-            f"- Lossless 4K MKV: {output_files['final_karaoke_lossless_mkv']}\n"
-            f"- Lossy 4K MP4: {output_files['final_karaoke_lossy_mp4']}\n"
-            f"- Lossy 720p MP4: {output_files['final_karaoke_lossy_720p_mp4']}\n"
-            f"Please check them! Proceed?",
-            "Refusing to proceed without user confirmation they're happy with the Final videos.",
-            allow_empty=True,
-        )
+        # Skip user confirmation in non-interactive mode for Modal deployment
+        if not self.non_interactive:
+            # Prompt user to check final video files before proceeding
+            self.prompt_user_confirmation_or_raise_exception(
+                f"Final video files created:\n"
+                f"- Lossless 4K MP4: {output_files['final_karaoke_lossless_mp4']}\n"
+                f"- Lossless 4K MKV: {output_files['final_karaoke_lossless_mkv']}\n"
+                f"- Lossy 4K MP4: {output_files['final_karaoke_lossy_mp4']}\n"
+                f"- Lossy 720p MP4: {output_files['final_karaoke_lossy_720p_mp4']}\n"
+                f"Please check them! Proceed?",
+                "Refusing to proceed without user confirmation they're happy with the Final videos.",
+                allow_empty=True,
+            )
+        else:
+            self.logger.info("Non-interactive mode: automatically confirming final video files")
 
     def create_cdg_zip_file(self, input_files, output_files, artist, title):
         self.logger.info(f"Creating CDG and MP3 files, then zipping them...")
@@ -935,9 +1054,9 @@ class KaraokeFinalise:
             self.logger.info(f"Copied final files to public share directory")
 
     def sync_public_share_dir_to_rclone_destination(self):
-        self.logger.info(f"Syncing public share directory to rclone destination...")
+        self.logger.info(f"Copying public share directory to rclone destination...")
 
-        # Delete .DS_Store files recursively before syncing
+        # Delete .DS_Store files recursively before copying
         for root, dirs, files in os.walk(self.public_share_dir):
             for file in files:
                 if file == ".DS_Store":
@@ -945,8 +1064,8 @@ class KaraokeFinalise:
                     os.remove(file_path)
                     self.logger.info(f"Deleted .DS_Store file: {file_path}")
 
-        rclone_cmd = f"rclone sync -v '{self.public_share_dir}' '{self.rclone_destination}'"
-        self.execute_command(rclone_cmd, "Syncing with cloud destination")
+        rclone_cmd = f"rclone copy -v {shlex.quote(self.public_share_dir)} {shlex.quote(self.rclone_destination)}"
+        self.execute_command(rclone_cmd, "Copying to cloud destination")
 
     def post_discord_notification(self):
         self.logger.info(f"Posting Discord notification...")
@@ -980,6 +1099,13 @@ class KaraokeFinalise:
         try:
             self.logger.info(f"Running command: {rclone_link_cmd}")
             result = subprocess.run(rclone_link_cmd, shell=True, check=True, capture_output=True, text=True)
+            
+            # Log command output for debugging
+            if result.stdout and result.stdout.strip():
+                self.logger.debug(f"Command STDOUT: {result.stdout.strip()}")
+            if result.stderr and result.stderr.strip():
+                self.logger.debug(f"Command STDERR: {result.stderr.strip()}")
+                
             self.brand_code_dir_sharing_link = result.stdout.strip()
             self.logger.info(f"Got organised folder sharing link: {self.brand_code_dir_sharing_link}")
         except subprocess.CalledProcessError as e:
@@ -988,6 +1114,125 @@ class KaraokeFinalise:
             self.logger.error(f"Command output (stderr): {e.stderr}")
             self.logger.error(f"Full exception: {e}")
 
+    def get_next_brand_code_server_side(self):
+        """
+        Calculate the next sequence number based on existing directories in the remote organised_dir using rclone.
+        Assumes directories are named with the format: BRAND-XXXX Artist - Title
+        """
+        if not self.organised_dir_rclone_root:
+            raise Exception("organised_dir_rclone_root not configured for server-side brand code generation")
+
+        self.logger.info(f"Getting next brand code from remote organized directory: {self.organised_dir_rclone_root}")
+        
+        max_num = 0
+        pattern = re.compile(rf"^{re.escape(self.brand_prefix)}-(\d{{4}})")
+
+        # Use rclone lsf --dirs-only for clean, machine-readable directory listing
+        rclone_list_cmd = f"rclone lsf --dirs-only {shlex.quote(self.organised_dir_rclone_root)}"
+        
+        if self.dry_run:
+            self.logger.info(f"DRY RUN: Would run: {rclone_list_cmd}")
+            return f"{self.brand_prefix}-0001"
+
+        try:
+            self.logger.info(f"Running command: {rclone_list_cmd}")
+            result = subprocess.run(rclone_list_cmd, shell=True, check=True, capture_output=True, text=True)
+            
+            # Log command output for debugging
+            if result.stdout and result.stdout.strip():
+                self.logger.debug(f"Command STDOUT: {result.stdout.strip()}")
+            if result.stderr and result.stderr.strip():
+                self.logger.debug(f"Command STDERR: {result.stderr.strip()}")
+            
+            # Parse the output to find matching directories
+            matching_dirs = []
+            for line_num, line in enumerate(result.stdout.strip().split('\n')):
+                if line.strip():
+                    # Remove trailing slash and whitespace
+                    dir_name = line.strip().rstrip('/')
+                    
+                    # Check if directory matches our brand pattern
+                    match = pattern.match(dir_name)
+                    if match:
+                        num = int(match.group(1))
+                        max_num = max(max_num, num)
+                        matching_dirs.append((dir_name, num))
+
+            self.logger.info(f"Found {len(matching_dirs)} matching directories with pattern {self.brand_prefix}-XXXX")
+
+            next_seq_number = max_num + 1
+            brand_code = f"{self.brand_prefix}-{next_seq_number:04d}"
+            
+            self.logger.info(f"Highest existing number: {max_num}, next sequence number for brand {self.brand_prefix} calculated as: {next_seq_number}")
+            return brand_code
+            
+        except subprocess.CalledProcessError as e:
+            self.logger.error(f"Failed to list remote organized directory. Exit code: {e.returncode}")
+            self.logger.error(f"Command output (stdout): {e.stdout}")
+            self.logger.error(f"Command output (stderr): {e.stderr}")
+            raise Exception(f"Failed to get brand code from remote directory: {e}")
+
+    def upload_files_to_organized_folder_server_side(self, brand_code, artist, title):
+        """
+        Upload all files from current directory to the remote organized folder using rclone.
+        Creates a brand-prefixed directory in the remote organized folder.
+        """
+        if not self.organised_dir_rclone_root:
+            raise Exception("organised_dir_rclone_root not configured for server-side file upload")
+
+        self.new_brand_code_dir = f"{brand_code} - {artist} - {title}"
+        remote_dest = f"{self.organised_dir_rclone_root}/{self.new_brand_code_dir}"
+        
+        self.logger.info(f"Uploading files to remote organized directory: {remote_dest}")
+
+        # Get current directory path to upload
+        current_dir = os.getcwd()
+        
+        # Use rclone copy to upload the entire current directory to the remote destination
+        rclone_upload_cmd = f"rclone copy -v {shlex.quote(current_dir)} {shlex.quote(remote_dest)}"
+        
+        if self.dry_run:
+            self.logger.info(f"DRY RUN: Would upload current directory to: {remote_dest}")
+            self.logger.info(f"DRY RUN: Command: {rclone_upload_cmd}")
+        else:
+            self.execute_command(rclone_upload_cmd, f"Uploading files to organized folder: {remote_dest}")
+
+        # Generate a sharing link for the uploaded folder
+        self.generate_organised_folder_sharing_link_server_side(remote_dest)
+
+    def generate_organised_folder_sharing_link_server_side(self, remote_path):
+        """Generate a sharing link for the remote organized folder using rclone."""
+        self.logger.info(f"Getting sharing link for remote organized folder: {remote_path}")
+
+        rclone_link_cmd = f"rclone link {shlex.quote(remote_path)}"
+
+        if self.dry_run:
+            self.logger.info(f"DRY RUN: Would get sharing link with: {rclone_link_cmd}")
+            self.brand_code_dir_sharing_link = "https://file-sharing-service.com/example"
+            return
+
+        # Add a 10-second delay to allow the remote service to index the folder before generating a link
+        self.logger.info("Waiting 10 seconds before generating link...")
+        time.sleep(10)
+
+        try:
+            self.logger.info(f"Running command: {rclone_link_cmd}")
+            result = subprocess.run(rclone_link_cmd, shell=True, check=True, capture_output=True, text=True)
+            
+            # Log command output for debugging
+            if result.stdout and result.stdout.strip():
+                self.logger.debug(f"Command STDOUT: {result.stdout.strip()}")
+            if result.stderr and result.stderr.strip():
+                self.logger.debug(f"Command STDERR: {result.stderr.strip()}")
+                
+            self.brand_code_dir_sharing_link = result.stdout.strip()
+            self.logger.info(f"Got organized folder sharing link: {self.brand_code_dir_sharing_link}")
+        except subprocess.CalledProcessError as e:
+            self.logger.error(f"Failed to get organized folder sharing link. Exit code: {e.returncode}")
+            self.logger.error(f"Command output (stdout): {e.stdout}")
+            self.logger.error(f"Command output (stderr): {e.stderr}")
+            self.logger.error(f"Full exception: {e}")
+
     def get_existing_brand_code(self):
         """Extract brand code from current directory name"""
         current_dir = os.path.basename(os.getcwd())
@@ -1018,7 +1263,30 @@ class KaraokeFinalise:
             if self.discord_notication_enabled:
                 self.post_discord_notification()
 
-        if self.folder_organisation_enabled:
+        # Handle folder organization - different logic for server-side vs local mode
+        if self.server_side_mode and self.brand_prefix and self.organised_dir_rclone_root:
+            self.logger.info("Executing server-side organization...")
+            
+            # Generate brand code from remote directory listing
+            if self.keep_brand_code:
+                self.brand_code = self.get_existing_brand_code()
+            else:
+                self.brand_code = self.get_next_brand_code_server_side()
+
+            # Upload files to organized folder via rclone
+            self.upload_files_to_organized_folder_server_side(self.brand_code, artist, title)
+
+            # Copy files to public share if enabled
+            if self.public_share_copy_enabled:
+                self.copy_final_files_to_public_share_dirs(self.brand_code, base_name, output_files)
+
+            # Sync public share to cloud destination if enabled
+            if self.public_share_rclone_enabled:
+                self.sync_public_share_dir_to_rclone_destination()
+
+        elif self.folder_organisation_enabled:
+            self.logger.info("Executing local folder organization...")
+            
             if self.keep_brand_code:
                 self.brand_code = self.get_existing_brand_code()
                 self.new_brand_code_dir = os.path.basename(os.getcwd())
@@ -1037,6 +1305,27 @@ class KaraokeFinalise:
                 self.sync_public_share_dir_to_rclone_destination()
 
             self.generate_organised_folder_sharing_link()
+        
+        elif self.public_share_copy_enabled or self.public_share_rclone_enabled:
+            # If only public share features are enabled (no folder organization), we still need a brand code
+            self.logger.info("No folder organization enabled, but public share features require brand code...")
+            if self.brand_prefix:
+                if self.server_side_mode and self.organised_dir_rclone_root:
+                    self.brand_code = self.get_next_brand_code_server_side()
+                elif not self.server_side_mode and self.organised_dir:
+                    self.brand_code = self.get_next_brand_code()
+                else:
+                    # Fallback to timestamp-based brand code if no organized directory configured
+                    import datetime
+                    timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+                    self.brand_code = f"{self.brand_prefix}-{timestamp}"
+                    self.logger.warning(f"No organized directory configured, using timestamp-based brand code: {self.brand_code}")
+
+                if self.public_share_copy_enabled:
+                    self.copy_final_files_to_public_share_dirs(self.brand_code, base_name, output_files)
+
+                if self.public_share_rclone_enabled:
+                    self.sync_public_share_dir_to_rclone_destination()
 
     def authenticate_gmail(self):
         """Authenticate and return a Gmail service object."""
@@ -1051,6 +1340,9 @@ class KaraokeFinalise:
             if creds and creds.expired and creds.refresh_token:
                 creds.refresh(Request())
             else:
+                if self.non_interactive:
+                    raise Exception("Gmail authentication required but running in non-interactive mode. Please pre-authenticate or disable email drafts.")
+                    
                 flow = InstalledAppFlow.from_client_secrets_file(
                     self.youtube_client_secrets_file, ["https://www.googleapis.com/auth/gmail.compose"]
                 )
@@ -1061,6 +1353,11 @@ class KaraokeFinalise:
         return build("gmail", "v1", credentials=creds)
 
     def draft_completion_email(self, artist, title, youtube_url, dropbox_url):
+        # Completely disable email drafts in server-side mode
+        if self.server_side_mode:
+            self.logger.info("Server-side mode: skipping email draft creation")
+            return
+
         if not self.email_template_file:
             self.logger.info("Email template file not provided, skipping email draft creation.")
             return
@@ -1118,6 +1415,206 @@ class KaraokeFinalise:
             self.logger.info("Using built-in aac codec (basic quality)")
             return "aac"
 
+    def detect_nvenc_support(self):
+        """Detect if NVENC hardware encoding is available with comprehensive checks."""
+        try:
+            self.logger.info("🔍 Detecting NVENC hardware acceleration support...")
+            
+            if self.dry_run:
+                self.logger.info("DRY RUN: Assuming NVENC is available")
+                return True
+            
+            import subprocess
+            import os
+            import shutil
+            
+            # Step 1: Check for nvidia-smi (indicates NVIDIA driver presence)
+            try:
+                nvidia_smi_result = subprocess.run(["nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader"], 
+                                                  capture_output=True, text=True, timeout=10)
+                if nvidia_smi_result.returncode == 0:
+                    gpu_info = nvidia_smi_result.stdout.strip()
+                    self.logger.info(f"✓ NVIDIA GPU detected: {gpu_info}")
+                else:
+                    self.logger.warning("⚠️ nvidia-smi not available or no NVIDIA GPU detected")
+                    return False
+            except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.CalledProcessError):
+                self.logger.warning("⚠️ nvidia-smi not available or failed")
+                return False
+            
+            # Step 2: Check for NVENC encoders in FFmpeg
+            try:
+                encoders_cmd = f"{self.ffmpeg_base_command} -hide_banner -encoders 2>/dev/null | grep nvenc"
+                encoders_result = subprocess.run(encoders_cmd, shell=True, capture_output=True, text=True, timeout=10)
+                if encoders_result.returncode == 0 and "nvenc" in encoders_result.stdout:
+                    nvenc_encoders = [line.strip() for line in encoders_result.stdout.split('\n') if 'nvenc' in line]
+                    self.logger.info("✓ Found NVENC encoders in FFmpeg:")
+                    for encoder in nvenc_encoders:
+                        if encoder:
+                            self.logger.info(f"  {encoder}")
+                else:
+                    self.logger.warning("⚠️ No NVENC encoders found in FFmpeg")
+                    return False
+            except Exception as e:
+                self.logger.warning(f"⚠️ Failed to check FFmpeg NVENC encoders: {e}")
+                return False
+            
+            # Step 3: Check for libcuda.so.1 (critical for NVENC)
+            try:
+                libcuda_check = subprocess.run(["ldconfig", "-p"], capture_output=True, text=True, timeout=10)
+                if libcuda_check.returncode == 0 and "libcuda.so.1" in libcuda_check.stdout:
+                    self.logger.info("✅ libcuda.so.1 found in system libraries")
+                else:
+                    self.logger.warning("❌ libcuda.so.1 NOT found in system libraries")
+                    self.logger.warning("💡 This usually indicates the CUDA runtime image is needed instead of devel")
+                    return False
+            except Exception as e:
+                self.logger.warning(f"⚠️ Failed to check for libcuda.so.1: {e}")
+                return False
+            
+            # Step 4: Test h264_nvenc encoder with simple test
+            self.logger.info("🧪 Testing h264_nvenc encoder...")
+            test_cmd = f"{self.ffmpeg_base_command} -hide_banner -loglevel warning -f lavfi -i testsrc=duration=1:size=320x240:rate=1 -c:v h264_nvenc -f null -"
+            self.logger.debug(f"Running test command: {test_cmd}")
+            
+            try:
+                result = subprocess.run(test_cmd, shell=True, capture_output=True, text=True, timeout=30)
+                
+                if result.returncode == 0:
+                    self.logger.info("✅ NVENC hardware encoding available for video generation")
+                    self.logger.info(f"Test command succeeded. Output: {result.stderr if result.stderr else '...'}")
+                    return True
+                else:
+                    self.logger.warning(f"❌ NVENC test failed with exit code {result.returncode}")
+                    if result.stderr:
+                        self.logger.warning(f"Error output: {result.stderr}")
+                        if "Cannot load libcuda.so.1" in result.stderr:
+                            self.logger.warning("💡 Root cause: libcuda.so.1 cannot be loaded by NVENC")
+                            self.logger.warning("💡 Solution: Use nvidia/cuda:*-devel-* image instead of runtime")
+                    return False
+                    
+            except subprocess.TimeoutExpired:
+                self.logger.warning("❌ NVENC test timed out")
+                return False
+                
+        except Exception as e:
+            self.logger.warning(f"❌ Failed to detect NVENC support: {e}, falling back to software encoding")
+            return False
+
+    def configure_hardware_acceleration(self):
+        """Configure hardware acceleration settings based on detected capabilities."""
+        if self.nvenc_available:
+            self.video_encoder = "h264_nvenc"
+            # Use simpler hardware acceleration that works with complex filter chains
+            # Remove -hwaccel_output_format cuda as it causes pixel format conversion issues
+            self.hwaccel_decode_flags = "-hwaccel cuda"
+            self.scale_filter = "scale"  # Use CPU scaling for complex filter chains
+            self.logger.info("Configured for NVIDIA hardware acceleration (simplified for filter compatibility)")
+        else:
+            self.video_encoder = "libx264"
+            self.hwaccel_decode_flags = ""
+            self.scale_filter = "scale"
+            self.logger.info("Configured for software encoding")
+
+    def get_nvenc_quality_settings(self, quality_mode="high"):
+        """Get NVENC settings based on quality requirements."""
+        if quality_mode == "lossless":
+            return "-preset lossless"
+        elif quality_mode == "high":
+            return "-preset p4 -tune hq -cq 18"  # High quality
+        elif quality_mode == "medium":
+            return "-preset p4 -cq 23"  # Balanced quality/speed
+        elif quality_mode == "fast":
+            return "-preset p1 -tune ll"  # Low latency, faster encoding
+        else:
+            return "-preset p4"  # Balanced default
+
+    def execute_command_with_fallback(self, gpu_command, cpu_command, description):
+        """Execute GPU command with automatic fallback to CPU if it fails."""
+        self.logger.info(f"{description}")
+        
+        if self.dry_run:
+            if self.nvenc_available:
+                self.logger.info(f"DRY RUN: Would run GPU-accelerated command: {gpu_command}")
+            else:
+                self.logger.info(f"DRY RUN: Would run CPU command: {cpu_command}")
+            return
+        
+        # Try GPU-accelerated command first if available
+        if self.nvenc_available and gpu_command != cpu_command:
+            self.logger.debug(f"Attempting hardware-accelerated encoding: {gpu_command}")
+            try:
+                result = subprocess.run(gpu_command, shell=True, capture_output=True, text=True, timeout=300)
+                
+                if result.returncode == 0:
+                    self.logger.info(f"✓ Hardware acceleration successful")
+                    return
+                else:
+                    self.logger.warning(f"✗ Hardware acceleration failed (exit code {result.returncode})")
+                    self.logger.warning(f"GPU Command: {gpu_command}")
+                    
+                    # If we didn't get detailed error info and using fatal loglevel, try again with verbose logging
+                    if (not result.stderr or len(result.stderr.strip()) < 10) and "-loglevel fatal" in gpu_command:
+                        self.logger.warning("Empty error output detected, retrying with verbose logging...")
+                        verbose_gpu_command = gpu_command.replace("-loglevel fatal", "-loglevel error")
+                        try:
+                            verbose_result = subprocess.run(verbose_gpu_command, shell=True, capture_output=True, text=True, timeout=300)
+                            self.logger.warning(f"Verbose GPU Command: {verbose_gpu_command}")
+                            if verbose_result.stderr:
+                                self.logger.warning(f"FFmpeg STDERR (verbose): {verbose_result.stderr}")
+                            if verbose_result.stdout:
+                                self.logger.warning(f"FFmpeg STDOUT (verbose): {verbose_result.stdout}")
+                        except Exception as e:
+                            self.logger.warning(f"Verbose retry failed: {e}")
+                    
+                    if result.stderr:
+                        self.logger.warning(f"FFmpeg STDERR: {result.stderr}")
+                    else:
+                        self.logger.warning("FFmpeg STDERR: (empty)")
+                    if result.stdout:
+                        self.logger.warning(f"FFmpeg STDOUT: {result.stdout}")
+                    else:
+                        self.logger.warning("FFmpeg STDOUT: (empty)")
+                    self.logger.info("Falling back to software encoding...")
+                    
+            except subprocess.TimeoutExpired:
+                self.logger.warning("✗ Hardware acceleration timed out, falling back to software encoding")
+            except Exception as e:
+                self.logger.warning(f"✗ Hardware acceleration failed with exception: {e}, falling back to software encoding")
+        
+        # Use CPU command (either as fallback or primary method)
+        self.logger.debug(f"Running software encoding: {cpu_command}")
+        try:
+            result = subprocess.run(cpu_command, shell=True, capture_output=True, text=True, timeout=600)
+            
+            if result.returncode != 0:
+                error_msg = f"Software encoding failed with exit code {result.returncode}"
+                self.logger.error(error_msg)
+                self.logger.error(f"CPU Command: {cpu_command}")
+                if result.stderr:
+                    self.logger.error(f"FFmpeg STDERR: {result.stderr}")
+                else:
+                    self.logger.error("FFmpeg STDERR: (empty)")
+                if result.stdout:
+                    self.logger.error(f"FFmpeg STDOUT: {result.stdout}")
+                else:
+                    self.logger.error("FFmpeg STDOUT: (empty)")
+                raise Exception(f"{error_msg}: {cpu_command}")
+            else:
+                self.logger.info(f"✓ Software encoding successful")
+                
+        except subprocess.TimeoutExpired:
+            error_msg = "Software encoding timed out"
+            self.logger.error(error_msg)
+            raise Exception(f"{error_msg}: {cpu_command}")
+        except Exception as e:
+            if "Software encoding failed" not in str(e):
+                error_msg = f"Software encoding failed with exception: {e}"
+                self.logger.error(error_msg)
+                raise Exception(f"{error_msg}: {cpu_command}")
+            else:
+                raise
+
     def process(self, replace_existing=False):
         if self.dry_run:
             self.logger.warning("Dry run enabled. No actions will be performed.")

{karaoke_gen-0.55.0 → karaoke_gen-0.57.0}/karaoke_gen/karaoke_gen.py

@@ -69,6 +69,8 @@ class KaraokePrep:
         style_params_json=None,
         # Add the new parameter
         skip_separation=False,
+        # YouTube/Online Configuration
+        cookies_str=None,
     ):
         self.log_level = log_level
         self.log_formatter = log_formatter
@@ -124,6 +126,9 @@ class KaraokePrep:
         self.render_bounding_boxes = render_bounding_boxes # Passed to VideoGenerator
         self.style_params_json = style_params_json # Passed to LyricsProcessor
 
+        # YouTube/Online Config
+        self.cookies_str = cookies_str # Passed to metadata extraction and file download
+
         # Load style parameters using the config module
         self.style_params = load_style_params(self.style_params_json, self.logger)
 
@@ -197,7 +202,7 @@ class KaraokePrep:
     # Compatibility methods for tests - these call the new functions in metadata.py
     def extract_info_for_online_media(self, input_url=None, input_artist=None, input_title=None):
         """Compatibility method that calls the function in metadata.py"""
-        self.extracted_info = extract_info_for_online_media(input_url, input_artist, input_title, self.logger)
+        self.extracted_info = extract_info_for_online_media(input_url, input_artist, input_title, self.logger, self.cookies_str)
         return self.extracted_info
 
     def parse_single_track_metadata(self, input_artist, input_title):
@@ -242,7 +247,7 @@ class KaraokePrep:
                 self.logger.warning(f"Input media '{self.input_media}' is not a file and self.url was not set. Attempting to treat as URL.")
                 # This path requires calling extract/parse again, less efficient
                 try:
-                    extracted = extract_info_for_online_media(self.input_media, self.artist, self.title, self.logger)
+                    extracted = extract_info_for_online_media(self.input_media, self.artist, self.title, self.logger, self.cookies_str)
                     if extracted:
                          metadata_result = parse_track_metadata(
                              extracted, self.artist, self.title, self.persistent_artist, self.logger
@@ -345,7 +350,7 @@ class KaraokePrep:
 
                     self.logger.info(f"Downloading input media from {self.url}...")
                     # Delegate to FileHandler
-                    processed_track["input_media"] = self.file_handler.download_video(self.url, output_filename_no_extension)
+                    processed_track["input_media"] = self.file_handler.download_video(self.url, output_filename_no_extension, self.cookies_str)
 
                     self.logger.info("Extracting still image from downloaded media (if input is video)...")
                     # Delegate to FileHandler
@@ -681,7 +686,7 @@ class KaraokePrep:
             self.url = self.input_media
             # Use the imported extract_info_for_online_media function
             self.extracted_info = extract_info_for_online_media(
-                input_url=self.url, input_artist=self.artist, input_title=self.title, logger=self.logger
+                input_url=self.url, input_artist=self.artist, input_title=self.title, logger=self.logger, cookies_str=self.cookies_str
             )
 
             if self.extracted_info and "playlist_count" in self.extracted_info:
@@ -690,4 +695,6 @@ class KaraokePrep:
                 return await self.process_playlist()
             else:
                 self.logger.info(f"Input URL is not a playlist, processing single track")
+                # Parse metadata to extract artist and title before processing
+                self.parse_single_track_metadata(self.artist, self.title)
                 return [await self.prep_single_track()]

{karaoke_gen-0.55.0 → karaoke_gen-0.57.0}/karaoke_gen/lyrics_processor.py

@@ -2,6 +2,7 @@ import os
 import re
 import logging
 import shutil
+import json
 from lyrics_transcriber import LyricsTranscriber, OutputConfig, TranscriberConfig, LyricsConfig
 from lyrics_transcriber.core.controller import LyricsControllerResult
 from dotenv import load_dotenv
@@ -173,6 +174,7 @@ class LyricsProcessor:
             "spotify_cookie": os.getenv("SPOTIFY_COOKIE_SP_DC"),
             "runpod_api_key": os.getenv("RUNPOD_API_KEY"),
             "whisper_runpod_id": os.getenv("WHISPER_RUNPOD_ID"),
+            "rapidapi_key": os.getenv("RAPIDAPI_KEY"),  # Add missing RAPIDAPI_KEY
         }
 
         # Create config objects for LyricsTranscriber
@@ -183,21 +185,51 @@ class LyricsProcessor:
         lyrics_config = LyricsConfig(
             genius_api_token=env_config.get("genius_api_token"),
             spotify_cookie=env_config.get("spotify_cookie"),
+            rapidapi_key=env_config.get("rapidapi_key"),
             lyrics_file=self.lyrics_file,
         )
-
+        
+        # Debug logging for lyrics_config
+        self.logger.info(f"LyricsConfig created with:")
+        self.logger.info(f"  genius_api_token: {env_config.get('genius_api_token')[:3] + '...' if env_config.get('genius_api_token') else 'None'}")
+        self.logger.info(f"  spotify_cookie: {env_config.get('spotify_cookie')[:3] + '...' if env_config.get('spotify_cookie') else 'None'}")
+        self.logger.info(f"  rapidapi_key: {env_config.get('rapidapi_key')[:3] + '...' if env_config.get('rapidapi_key') else 'None'}")
+        self.logger.info(f"  lyrics_file: {self.lyrics_file}")
+
+        # Detect if we're running in a serverless environment (Modal)
+        # Modal sets specific environment variables we can check for
+        is_serverless = (
+            os.getenv("MODAL_TASK_ID") is not None or 
+            os.getenv("MODAL_FUNCTION_NAME") is not None or
+            os.path.exists("/.modal")  # Modal creates this directory in containers
+        )
+        
+        # In serverless environment, disable interactive review even if skip_transcription_review=False
+        # This preserves CLI behavior while fixing serverless hanging
+        enable_review_setting = not self.skip_transcription_review and not is_serverless
+        
+        if is_serverless and not self.skip_transcription_review:
+            self.logger.info("Detected serverless environment - disabling interactive review to prevent hanging")
+        
+        # In serverless environment, disable video generation during Phase 1 to save compute
+        # Video will be generated in Phase 2 after human review
+        serverless_render_video = render_video and not is_serverless
+        
+        if is_serverless and render_video:
+            self.logger.info("Detected serverless environment - deferring video generation until after review")
+        
         output_config = OutputConfig(
             output_styles_json=self.style_params_json,
             output_dir=lyrics_dir,
-            render_video=render_video,
+            render_video=serverless_render_video,  # Disable video in serverless Phase 1
             fetch_lyrics=True,
             run_transcription=not self.skip_transcription,
             run_correction=True,
             generate_plain_text=True,
             generate_lrc=True,
-            generate_cdg=True,
+            generate_cdg=False,  # Also defer CDG generation to Phase 2 
             video_resolution="4k",
-            enable_review=not self.skip_transcription_review,
+            enable_review=enable_review_setting,
             subtitle_offset_ms=self.subtitle_offset_ms,
         )
 
@@ -240,6 +272,19 @@ class LyricsProcessor:
             )
             transcriber_outputs["corrected_lyrics_text_filepath"] = results.corrected_txt
 
+            # Save correction data to JSON file for review interface
+            # Use the expected filename format: "{artist} - {title} (Lyrics Corrections).json"
+            corrections_filename = f"{filename_artist} - {filename_title} (Lyrics Corrections).json"
+            corrections_filepath = os.path.join(lyrics_dir, corrections_filename)
+            
+            # Use the CorrectionResult's to_dict() method to serialize
+            correction_data = results.transcription_corrected.to_dict()
+            
+            with open(corrections_filepath, 'w') as f:
+                json.dump(correction_data, f, indent=2)
+            
+            self.logger.info(f"Saved correction data to {corrections_filepath}")
+
         if transcriber_outputs:
             self.logger.info(f"*** Transcriber Filepath Outputs: ***")
             for key, value in transcriber_outputs.items():

{karaoke_gen-0.55.0 → karaoke_gen-0.57.0}/karaoke_gen/metadata.py

@@ -1,30 +1,80 @@
 import logging
 import yt_dlp.YoutubeDL as ydl
 
-def extract_info_for_online_media(input_url, input_artist, input_title, logger):
+def extract_info_for_online_media(input_url, input_artist, input_title, logger, cookies_str=None):
     """Extracts metadata using yt-dlp, either from a URL or via search."""
     logger.info(f"Extracting info for input_url: {input_url} input_artist: {input_artist} input_title: {input_title}")
-    extracted_info = None
-    if input_url is not None:
-        # If a URL is provided, use it to extract the metadata
-        with ydl({"quiet": True}) as ydl_instance:
-            extracted_info = ydl_instance.extract_info(input_url, download=False)
+    
+    # Set up yt-dlp options with enhanced anti-detection
+    base_opts = {
+        "quiet": True,
+        # Anti-detection options
+        "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "referer": "https://www.youtube.com/",
+        "sleep_interval": 1,
+        "max_sleep_interval": 3,
+        "fragment_retries": 3,
+        "extractor_retries": 3,
+        "retries": 3,
+        # Headers to appear more human
+        "http_headers": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-us,en;q=0.5",
+            "Accept-Encoding": "gzip, deflate",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+        },
+    }
+    
+    # Add cookies if provided
+    if cookies_str:
+        logger.info("Using provided cookies for enhanced YouTube access")
+        # Save cookies to a temporary file
+        import tempfile
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f:
+            f.write(cookies_str)
+            base_opts['cookiefile'] = f.name
     else:
-        # If no URL is provided, use the query to search for the top result
-        ydl_opts = {"quiet": "True", "format": "bestaudio", "noplaylist": "True", "extract_flat": True}
-        with ydl(ydl_opts) as ydl_instance:
-            query = f"{input_artist} {input_title}"
-            search_results = ydl_instance.extract_info(f"ytsearch1:{query}", download=False)
-            if search_results and "entries" in search_results and search_results["entries"]:
-                 extracted_info = search_results["entries"][0]
-            else:
-                # Raise IndexError to match the expected exception in tests
-                raise IndexError(f"No search results found on YouTube for query: {input_artist} {input_title}")
-
-    if not extracted_info:
-         raise Exception(f"Failed to extract info for query: {input_artist} {input_title} or URL: {input_url}")
-
-    return extracted_info
+        logger.info("No cookies provided - attempting standard extraction")
+    
+    extracted_info = None
+    try:
+        if input_url is not None:
+            # If a URL is provided, use it to extract the metadata
+            with ydl(base_opts) as ydl_instance:
+                extracted_info = ydl_instance.extract_info(input_url, download=False)
+        else:
+            # If no URL is provided, use the query to search for the top result
+            search_opts = base_opts.copy()
+            search_opts.update({
+                "format": "bestaudio", 
+                "noplaylist": "True", 
+                "extract_flat": True
+            })
+            
+            with ydl(search_opts) as ydl_instance:
+                query = f"{input_artist} {input_title}"
+                search_results = ydl_instance.extract_info(f"ytsearch1:{query}", download=False)
+                if search_results and "entries" in search_results and search_results["entries"]:
+                     extracted_info = search_results["entries"][0]
+                else:
+                    # Raise IndexError to match the expected exception in tests
+                    raise IndexError(f"No search results found on YouTube for query: {input_artist} {input_title}")
+
+        if not extracted_info:
+             raise Exception(f"Failed to extract info for query: {input_artist} {input_title} or URL: {input_url}")
+
+        return extracted_info
+        
+    finally:
+        # Clean up temporary cookie file if it was created
+        if cookies_str and 'cookiefile' in base_opts:
+            try:
+                import os
+                os.unlink(base_opts['cookiefile'])
+            except:
+                pass
 
 
 def parse_track_metadata(extracted_info, current_artist, current_title, persistent_artist, logger):

{karaoke_gen-0.55.0 → karaoke_gen-0.57.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "karaoke-gen"
-version = "0.55.0"
+version = "0.57.0"
 description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
 authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
 license = "MIT"
@@ -28,7 +28,7 @@ thefuzz = ">=0.22"
 numpy = ">=2"
 audio-separator = { version = ">=0.34.0", extras = ["cpu"] }
 lyrics-converter = ">=0.2.1"
-lyrics-transcriber = ">=0.54"
+lyrics-transcriber = ">=0.61"
 kbputils = "^0.0.16"
 attrs = ">=24.2.0"
 cattrs = ">=24.1.2"
@@ -36,8 +36,10 @@ toml = ">=0.10"
 argparse = ">=1.4.0"
 psutil = "^7.0.0"
 pyperclip = "*"
-pytest-asyncio = "^0.23.5"
+pytest-asyncio = "*"
 ffmpeg-python = "^0.2.0"
+modal = "^1.0.5"
+python-multipart = "^0.0.20"
 
 [tool.poetry.group.dev.dependencies]
 black = ">=23"
@@ -45,7 +47,7 @@ poetry = "*"
 pytest = ">=7.0"
 pytest-cov = ">=4.0"
 pytest-mock = ">=3.10"
-pytest-asyncio = ">=0.21.0"
+pytest-asyncio = "*"
 
 
 [tool.poetry.scripts]