monkeyplug-enhanced 2.3.0__tar.gz → 2.3.1__tar.gz

{monkeyplug_enhanced-2.3.0 → monkeyplug_enhanced-2.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: monkeyplug-enhanced
-Version: 2.3.0
+Version: 2.3.1
 Summary: Enhanced fork of monkeyplug — censors profanity in audio files using speech recognition with Groq API, AI instrumental generation, and batch processing.
 Project-URL: Homepage, https://github.com/ljbred08/monkeyplug
 Project-URL: Issues, https://github.com/ljbred08/monkeyplug/issues
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
 Requires-Python: >=3.10
 Requires-Dist: aiohttp>=3.9.0
-Requires-Dist: duckduckgo-search>=6.0.0
+Requires-Dist: ddgs>=6.0.0
 Requires-Dist: groq>=0.1.0
 Requires-Dist: mmguero==2.0.3
 Requires-Dist: mutagen==1.47.0
@@ -22,7 +22,7 @@ Requires-Dist: requests==2.32.5
 Requires-Dist: shazamio>=0.8.0
 Requires-Dist: sherpa-onnx>=1.10.0
 Requires-Dist: soundfile>=0.12.0
-Requires-Dist: spotify-scraper>=0.1.0
+Requires-Dist: spotifyscraper>=0.1.0
 Requires-Dist: tqdm>=4.65.0
 Description-Content-Type: text/markdown
 

{monkeyplug_enhanced-2.3.0 → monkeyplug_enhanced-2.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "monkeyplug-enhanced"
-version = "2.3.0"
+version = "2.3.1"
 authors = [
   { name="Seth Grover", email="mero.mero.guero@gmail.com" },
   { name="Lincoln Brown", email="link@brown.fm" },
@@ -29,8 +29,8 @@ dependencies = [
     "tqdm>=4.65.0",
     "shazamio>=0.8.0",
     "aiohttp>=3.9.0",
-    "duckduckgo-search>=6.0.0",
-    "spotify-scraper>=0.1.0",
+    "ddgs>=6.0.0",
+    "spotifyscraper>=0.1.0",
 ]
 
 [project.urls]

{monkeyplug_enhanced-2.3.0 → monkeyplug_enhanced-2.3.1}/src/monkeyplug/monkeyplug.py

@@ -123,13 +123,17 @@ UNIFY_ALBUM_PROMPT_DEFAULT = (
     "titles, and current album names, determine the correct unified album name and "
     "assign track numbers to each song. Consider the existing album name guesses and "
     "song titles to infer the real album. Return track numbers in the order the songs "
-    "should appear on the album."
+    "should appear on the album. "
+    "CRITICAL: You MUST return a track entry for EVERY input file - do not skip any files. "
+    "The tracks array must contain exactly the same number of entries as the input."
 )
 
 UNIFY_ALBUM_RENAME_PROMPT_DEFAULT = (
     "You are a music file naming expert. Suggest clean, consistent filenames for each track. "
     "Use format: 'XX - Song Name' where XX is the track number with leading zero if needed. "
     "Keep only essential information, remove extra words like 'feat', 'explicit', etc. "
+    "IMPORTANT: Never use these invalid characters in filenames: < > : \" / \\ | ? * "
+    "Replace punctuation marks like ? with nothing or - (hyphen). "
     "Return the suggested filename WITHOUT the file extension."
 )
 
@@ -139,6 +143,7 @@ UNIFY_ALBUM_SCHEMA = {
         "unified_album": {"type": "string", "description": "The unified album name"},
         "tracks": {
             "type": "array",
+            "description": "MUST contain one entry for EACH input file - no files may be omitted",
             "items": {
                 "type": "object",
                 "properties": {
@@ -419,6 +424,131 @@ def _unify_album_metadata(file_paths, groq_api_key, model, prompt, rename_prompt
             if debug:
                 mmguero.eprint(f"AI unification response: {content}")
 
+            # Validate that we got results for all files
+            returned_tracks = parsed.get('tracks', [])
+            if len(returned_tracks) < len(metadata_list):
+                missing = len(metadata_list) - len(returned_tracks)
+                mmguero.eprint(f"WARNING: AI only returned {len(returned_tracks)} of {len(metadata_list)} track assignments.")
+                mmguero.eprint(f"This is likely due to Groq's output token limit. {missing} files were not processed.")
+                if spotify_tracks:
+                    mmguero.eprint("Try running again without --use-spotify, or process files in smaller batches.")
+                else:
+                    mmguero.eprint("Try processing files in smaller batches (e.g., split into subdirectories).")
+
+            return parsed
+
+        except requests.exceptions.Timeout:
+            if attempt < max_retries - 1:
+                if debug:
+                    mmguero.eprint(f"Request timed out, retrying in {retry_delay}s...")
+                time.sleep(retry_delay)
+                retry_delay *= 2
+            else:
+                raise Exception("Album unification request timed out")
+
+        except requests.exceptions.RequestException as e:
+            if attempt < max_retries - 1:
+                if debug:
+                    mmguero.eprint(f"Request failed: {e}, retrying in {retry_delay}s...")
+                time.sleep(retry_delay)
+                retry_delay *= 2
+            else:
+                raise Exception(f"Album unification request failed: {e}")
+
+    raise Exception("Album unification failed after maximum retries")
+
+
+def _call_groq_api_single_batch(metadata_list, system_prompt, groq_api_key, model, batch_num=1, total_batches=1, debug=False):
+    """Make a single API call to Groq for album unification.
+
+    Args:
+        metadata_list: List of file metadata dicts to send in this batch
+        system_prompt: System prompt for the AI
+        groq_api_key: Groq API key for authentication
+        model: AI model name
+        batch_num: Current batch number (for debug output)
+        total_batches: Total expected batches (for debug output)
+        debug: Enable debug output
+
+    Returns:
+        dict: Parsed JSON response with 'unified_album' and 'tracks'
+
+    Raises:
+        Exception: If API call fails after retries
+    """
+    import requests
+    import time
+
+    # Build input for AI
+    input_text = json.dumps(metadata_list, indent=2, ensure_ascii=False)
+
+    # API call with retry logic (more retries for transient 400 JSON validation errors)
+    max_retries = 5
+    retry_delay = 1
+
+    for attempt in range(max_retries):
+        try:
+            if debug:
+                batch_info = f" (batch {batch_num}" + (f"/{total_batches}" if isinstance(total_batches, int) and total_batches > 1 else "") + ")"
+                mmguero.eprint(f"Calling Groq API{batch_info} (attempt {attempt + 1}/{max_retries})...")
+                mmguero.eprint(f"Sending {len(metadata_list)} files to AI for unification")
+
+            response = requests.post(
+                "https://api.groq.com/openai/v1/chat/completions",
+                headers={
+                    "Authorization": f"Bearer {groq_api_key}",
+                    "Content-Type": "application/json",
+                },
+                json={
+                    "model": model,
+                    "messages": [
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": input_text},
+                    ],
+                    "response_format": {
+                        "type": "json_schema",
+                        "json_schema": {
+                            "name": "album_unification",
+                            "strict": True,
+                            "schema": UNIFY_ALBUM_SCHEMA,
+                        }
+                    }
+                },
+                timeout=120,
+            )
+
+            # Handle rate limiting
+            if response.status_code == 429:
+                if attempt < max_retries - 1:
+                    if debug:
+                        mmguero.eprint(f"Rate limited, retrying in {retry_delay}s...")
+                    time.sleep(retry_delay)
+                    retry_delay *= 2
+                    continue
+                raise Exception("Album unification rate limit exceeded")
+
+            if response.status_code == 401:
+                raise Exception("Invalid Groq API key for album unification")
+
+            # Handle 400 errors (usually JSON validation failures from Groq - transient)
+            if response.status_code == 400:
+                if attempt < max_retries - 1:
+                    if debug:
+                        mmguero.eprint(f"JSON validation error (400), retrying in {retry_delay}s...")
+                    time.sleep(retry_delay)
+                    retry_delay *= 2
+                    continue
+                raise Exception("Album unification failed due to JSON validation errors")
+
+            response.raise_for_status()
+
+            result = response.json()
+            content = result.get("choices", [{}])[0].get("message", {}).get("content", "{}")
+            parsed = json.loads(content)
+
+            if debug:
+                mmguero.eprint(f"AI returned {len(parsed.get('tracks', []))} track assignments")
+
             return parsed
 
         except requests.exceptions.Timeout:
@@ -442,6 +572,145 @@ def _unify_album_metadata(file_paths, groq_api_key, model, prompt, rename_prompt
     raise Exception("Album unification failed after maximum retries")
 
 
+def _unify_album_metadata_with_batching(file_paths, groq_api_key, model, prompt, rename_prompt=None, spotify_tracks=None, batch_size=10, batch_size_spotify=5, debug=False):
+    """Use Groq AI to unify album metadata with automatic batching for large file lists.
+
+    Implements automatic batching to handle Groq's output token limits.
+    If a partial response is received, automatically retries with remaining files.
+
+    Args:
+        file_paths: List of audio file paths
+        groq_api_key: Groq API key for authentication
+        model: AI model name (e.g., "openai/gpt-oss-120b")
+        prompt: System prompt for the AI
+        rename_prompt: Optional prompt for renaming (if provided, adds suggested_name to response)
+        spotify_tracks: Optional list of track names from Spotify for accurate ordering
+        debug: Enable debug output
+
+    Returns:
+        Dict with 'unified_album' (str) and 'tracks' (list of dicts with
+        'filename', 'track_number', 'album_name', optionally 'suggested_name')
+
+    Raises:
+        ValueError: If API key is missing
+        Exception: If API call fails
+    """
+    if not groq_api_key:
+        raise ValueError("Groq API key required for album unification")
+
+    # Read metadata from all files
+    if debug:
+        mmguero.eprint("Reading metadata from files...")
+    metadata_list = _read_metadata_from_files(file_paths, debug=debug)
+
+    # Build system prompt (add rename and Spotify instructions if needed)
+    system_prompt = prompt
+    if rename_prompt:
+        system_prompt = f"{prompt}\n\n{rename_prompt}"
+
+    unified_album = None  # Will be set from first batch and reused
+    all_tracks = []       # Accumulates results across batches
+    processed_files = set()  # Tracks which files we've gotten results for
+
+    # Proactive batching: limit batch size to avoid overwhelming Groq
+    # With Spotify tracks, use smaller batches since the prompt is larger
+    max_batch_size = batch_size_spotify if spotify_tracks else batch_size
+
+    # Start with first batch
+    batch_metadata = metadata_list[:max_batch_size]
+    remaining_metadata = metadata_list[max_batch_size:]
+    batch_num = 0
+
+    while batch_metadata:
+        batch_num += 1
+
+        # Build system prompt for this batch
+        # ALWAYS pass full Spotify list - don't slice it!
+        batch_system_prompt = system_prompt
+        if spotify_tracks:
+            # Add FULL Spotify track listing every time
+            tracks_json = json.dumps(spotify_tracks, ensure_ascii=False)
+            batch_system_prompt = f"{system_prompt}\n\nOfficial track listing from Spotify: {tracks_json}"
+            if debug and batch_num == 1:
+                mmguero.eprint(f"Providing full Spotify track list ({len(spotify_tracks)} tracks) - AI will match by name")
+
+        # Call API with current batch
+        try:
+            parsed = _call_groq_api_single_batch(
+                batch_metadata, batch_system_prompt, groq_api_key, model,
+                batch_num, total_batches="?", debug=debug
+            )
+        except Exception as e:
+            # If this isn't the first batch, we have partial results - fail gracefully
+            if all_tracks:
+                mmguero.eprint(f"Batch {batch_num} failed after {len(all_tracks)} tracks were processed: {e}")
+                mmguero.eprint("Proceeding with partial results...")
+                break
+            raise
+
+        # On first successful call, capture unified_album name
+        if unified_album is None and parsed.get('unified_album'):
+            unified_album = parsed['unified_album']
+
+        # For subsequent batches, override the album name to match first batch
+        if unified_album and parsed.get('unified_album') != unified_album:
+            parsed['unified_album'] = unified_album
+
+        # Add returned tracks to our collection
+        returned_tracks = parsed.get('tracks', [])
+
+        # Guard against empty response to avoid infinite loop
+        if not returned_tracks:
+            mmguero.eprint(f"WARNING: Batch {batch_num} returned no tracks. Stopping to avoid infinite loop.")
+            break
+
+        for track in returned_tracks:
+            filename = track['filename']
+            if filename not in processed_files:
+                all_tracks.append(track)
+                processed_files.add(filename)
+
+        if debug:
+            mmguero.eprint(f"Batch {batch_num} complete: {len(returned_tracks)} tracks returned, {len(all_tracks)} total processed")
+
+        # Determine what's missing from this batch
+        returned_filenames = {t['filename'] for t in returned_tracks}
+        missing_metadata = [
+            m for m in batch_metadata
+            if m['filename'] not in returned_filenames and m['filename'] not in processed_files
+        ]
+
+        if not missing_metadata and not remaining_metadata:
+            # All files processed!
+            if debug:
+                mmguero.eprint(f"All {len(all_tracks)} files processed successfully!")
+            break
+
+        # Prepare next batch: combine missing files + next chunk from remaining
+        next_batch = missing_metadata if missing_metadata else []
+        if remaining_metadata:
+            take = min(max_batch_size - len(next_batch), len(remaining_metadata))
+            next_batch.extend(remaining_metadata[:take])
+            remaining_metadata = remaining_metadata[take:]
+
+        if not next_batch:
+            break
+
+        if debug:
+            if missing_metadata:
+                new_files = len(next_batch) - len(missing_metadata)
+                mmguero.eprint(f"Partial response: {len(missing_metadata)} files from this batch need retry. Next batch: {len(missing_metadata)} retries + {new_files} new files = {len(next_batch)} total")
+            else:
+                mmguero.eprint(f"Starting batch {batch_num + 1} with {len(next_batch)} files...")
+
+        batch_metadata = next_batch
+
+    return {
+        'unified_album': unified_album or '',
+        'tracks': all_tracks
+    }
+
+
 ###################################################################################################
 # Apply unified metadata to audio files
 def _apply_unified_metadata(file_paths, unified_result, debug=False):
@@ -523,6 +792,35 @@ def _apply_unified_metadata(file_paths, unified_result, debug=False):
 
 ###################################################################################################
 # Apply smart renaming to audio files
+def _sanitize_filename(filename, debug=False):
+    r"""Sanitize filename for Windows compatibility.
+
+    Removes/replaces characters that are invalid on Windows:
+    < > : " / \ | ? *
+
+    Args:
+        filename: The filename to sanitize (without extension)
+        debug: Enable debug output
+
+    Returns:
+        str: Sanitized filename
+    """
+    # Windows invalid characters: < > : " / \ | ? *
+    invalid_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*']
+    sanitized = filename
+
+    for char in invalid_chars:
+        sanitized = sanitized.replace(char, '')
+
+    # Also handle leading/trailing spaces and dots
+    sanitized = sanitized.strip('. ')
+
+    if debug and sanitized != filename:
+        mmguero.eprint(f"Sanitized filename: '{filename}' → '{sanitized}'")
+
+    return sanitized
+
+
 def _apply_renames(file_paths, unified_result, rename_prompt, debug=False):
     """Rename files based on AI-suggested names.
 
@@ -568,8 +866,9 @@ def _apply_renames(file_paths, unified_result, rename_prompt, debug=False):
         dirname = os.path.dirname(filepath)
         ext = os.path.splitext(filepath)[1]
 
-        # Build new filename
-        new_name = f"{suggested_name}{ext}"
+        # Build new filename (sanitize for Windows compatibility)
+        sanitized_name = _sanitize_filename(suggested_name, debug=debug)
+        new_name = f"{sanitized_name}{ext}"
         new_path = os.path.join(dirname, new_name)
 
         # Skip if same name
@@ -578,10 +877,11 @@ def _apply_renames(file_paths, unified_result, rename_prompt, debug=False):
                 mmguero.eprint(f"Skipping rename (same name): {basename}")
             continue
 
-        # Check if target already exists
+        # Check if target already exists - remove it so we can rename/overwrite
         if os.path.exists(new_path):
-            mmguero.eprint(f"Cannot rename {basename} to {new_name}: target already exists")
-            continue
+            if debug:
+                mmguero.eprint(f"Removing existing file {new_name} to allow rename")
+            os.remove(new_path)
 
         try:
             shutil.move(filepath, new_path)
@@ -610,7 +910,7 @@ def _search_spotify_album(album_name, debug=False):
         from ddgs import DDGS
     except ImportError:
         if debug:
-            mmguero.eprint("duckduckgo-search not installed, skipping Spotify search")
+            mmguero.eprint("ddgs not installed, skipping Spotify search")
         return None
 
     query = f"site:spotify.com {album_name} album"
@@ -645,7 +945,7 @@ def _get_spotify_album_info(spotify_url, debug=False):
         from spotify_scraper import SpotifyClient
     except ImportError:
         if debug:
-            mmguero.eprint("spotify-scraper not installed, skipping Spotify info")
+            mmguero.eprint("spotifyscraper not installed, skipping Spotify info")
         return None
 
     try:
@@ -790,6 +1090,8 @@ def _run_album_unification(input_path, output_path, config, rename_prompt=None,
 
     model = config.get("unify_album_model", "openai/gpt-oss-120b")
     prompt = config.get("unify_album_prompt", UNIFY_ALBUM_PROMPT_DEFAULT)
+    batch_size = config.get("unify_album_batch_size", 10)
+    batch_size_spotify = config.get("unify_album_batch_size_with_spotify", 5)
 
     # Determine files to process
     audio_extensions = ['.mp3', '.mp4', '.m4a', '.wav', '.flac', '.ogg', '.aac', '.wma']
@@ -833,8 +1135,9 @@ def _run_album_unification(input_path, output_path, config, rename_prompt=None,
     mmguero.eprint(f"Found {len(file_paths)} audio files for album unification")
 
     # Call AI to unify album metadata (first pass - gets unified album name)
-    unified_result = _unify_album_metadata(
-        file_paths, groq_api_key, model, prompt, rename_prompt=rename_prompt, debug=debug
+    unified_result = _unify_album_metadata_with_batching(
+        file_paths, groq_api_key, model, prompt, rename_prompt=rename_prompt,
+        batch_size=batch_size, batch_size_spotify=batch_size_spotify, debug=debug
     )
 
     unified_album = unified_result.get('unified_album', '')
@@ -863,10 +1166,11 @@ def _run_album_unification(input_path, output_path, config, rename_prompt=None,
 
             # Second AI pass with Spotify track listing for accurate ordering
             mmguero.eprint("Refining track order with Spotify data...")
-            unified_result = _unify_album_metadata(
+            unified_result = _unify_album_metadata_with_batching(
                 file_paths, groq_api_key, model, prompt,
                 rename_prompt=rename_prompt,
                 spotify_tracks=spotify_info.get('tracks', []),
+                batch_size=batch_size, batch_size_spotify=batch_size_spotify,
                 debug=debug
             )
         else:
@@ -2180,6 +2484,65 @@ class Plugger(object):
 
             self.CreateCleanMuteList()
 
+            # Fast-path: if no profanity detected and same format, just copy
+            no_profanity = len(self.naughtyWordList) == 0
+            same_format = (
+                self.inputFileParts[1].lower().lstrip('.') == self.outputAudioFileFormat
+                if hasattr(self, 'inputFileParts')
+                else False
+            )
+
+            if no_profanity and same_format:
+                # Direct copy - no processing needed
+                if self.debug:
+                    mmguero.eprint("No profanity detected and same format - using direct copy")
+
+                # Update progress bar description to reflect copy operation
+                if progress:
+                    progress.set_description("Copying")
+
+                copy_start = time.time()
+                shutil.copyfile(self.inputFileSpec, self.outputFileSpec)
+                copy_time = time.time() - copy_start
+
+                # Still embed metadata and tag as processed
+                if self.debug:
+                    mmguero.eprint("Embedding Shazam metadata into copied file...")
+                self._embed_metadata(self.outputFileSpec)
+                SetMonkeyplugTag(self.outputFileSpec, debug=self.debug)
+
+                # Complete progress
+                if progress:
+                    if smooth_ticker:
+                        progress.n = progress.total
+                        progress.refresh()
+                    else:
+                        progress.update(1)
+                    progress.close()
+
+                # Record timing separately for copy (don't skew encode estimates)
+                # Copy is much faster than encode, so we track it separately
+                if step_timings is not None and file_duration > 0:
+                    step_timings['copy'] = (copy_time, file_duration)
+                if timing_log is not None and file_duration > 0:
+                    for op, (wall_secs, audio_secs) in step_timings.items():
+                        update_timing_measurement(timing_log, op, wall_secs, audio_secs)
+                    save_timing_log(timing_log)
+
+                # Clean up progress references
+                if hasattr(self, '_progress'):
+                    delattr(self, '_progress')
+                for attr in ('_smooth_ticker', '_smooth_cumulative', '_smooth_extract_est',
+                              '_smooth_transcribe_est', '_will_transcribe',
+                              '_step_timings', '_timing_log', '_timing_file_duration'):
+                    if hasattr(self, attr):
+                        delattr(self, attr)
+
+                # Print profanity detection summary
+                self._print_words_summary()
+
+                return self.outputFileSpec
+
             # Update progress after CreateCleanMuteList (step-based mode only)
             if progress and not smooth_ticker:
                 did_extraction = (
@@ -3323,8 +3686,42 @@ DEFAULT_CONFIG = {
     "ai_detect_prompt": AI_DETECT_PROMPT_DEFAULT,
     "unify_album_model": "openai/gpt-oss-120b",
     "unify_album_prompt": UNIFY_ALBUM_PROMPT_DEFAULT,
+    "unify_album_batch_size": 10,
+    "unify_album_batch_size_with_spotify": 5,
 }
 
+# Validation rules for config values with defined options
+CONFIG_VALIDATION = {
+    "show_words": {"choices": ["full", "clean", "none"], "default": "clean"},
+    "detect_mode": {"choices": ["list", "ai", "both"], "default": "list"},
+}
+
+
+def validate_config_settings(config, debug=False):
+    """
+    Validate config settings and fix invalid values.
+
+    For each config key that has defined choices, validates the value.
+    If invalid, prints a warning and uses the default value.
+
+    Args:
+        config: dict - Config settings to validate
+        debug: bool - Enable debug output
+
+    Returns:
+        dict: Validated config with invalid values replaced by defaults
+    """
+    validated = dict(config)
+    for key, rules in CONFIG_VALIDATION.items():
+        if key in validated:
+            value = validated[key]
+            choices = rules["choices"]
+            if value not in choices:
+                default = rules["default"]
+                mmguero.eprint(f"WARNING: CONFIG \"{key}\" SET TO INVALID VALUE \"{value}\". USING DEFAULT \"{default}\".")
+                validated[key] = default
+    return validated
+
 
 def load_config_settings(debug=False):
     """
@@ -3354,7 +3751,8 @@ def load_config_settings(debug=False):
                 if debug:
                     mmguero.eprint(f"Loaded config from: {config_path}")
 
-                return config
+                # Validate and fix any invalid config values
+                return validate_config_settings(config, debug=debug)
             except (json.JSONDecodeError, IOError) as e:
                 if debug:
                     mmguero.eprint(f"Warning: Failed to load config from {config_path}: {e}")
@@ -3372,6 +3770,7 @@ def load_config_settings(debug=False):
         if debug:
             mmguero.eprint(f"Warning: Could not create default config: {e}")
 
+    # Return a copy of DEFAULT_CONFIG (already validated)
     return dict(DEFAULT_CONFIG)