content-core 1.5.0__tar.gz → 1.7.0__tar.gz

content_core-1.7.0/.github/workflows/create-tag.yml

@@ -0,0 +1,56 @@
+name: Create Tag
+
+on:
+  workflow_dispatch:
+    inputs:
+      confirm:
+        description: 'Type "yes" to confirm tag creation'
+        required: true
+        default: 'no'
+
+jobs:
+  create-tag:
+    runs-on: ubuntu-latest
+    if: github.event.inputs.confirm == 'yes'
+    permissions:
+      contents: write
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Extract version from pyproject.toml
+      id: version
+      run: |
+        VERSION=$(grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
+        echo "version=$VERSION" >> $GITHUB_OUTPUT
+        echo "tag=v$VERSION" >> $GITHUB_OUTPUT
+        echo "📦 Version found: $VERSION"
+
+    - name: Check if tag already exists
+      id: check
+      run: |
+        if git rev-parse "v${{ steps.version.outputs.version }}" >/dev/null 2>&1; then
+          echo "exists=true" >> $GITHUB_OUTPUT
+          echo "❌ Tag v${{ steps.version.outputs.version }} already exists!"
+          exit 1
+        else
+          echo "exists=false" >> $GITHUB_OUTPUT
+          echo "✅ Tag v${{ steps.version.outputs.version }} does not exist yet"
+        fi
+
+    - name: Create and push tag
+      if: steps.check.outputs.exists == 'false'
+      run: |
+        git config user.name "github-actions[bot]"
+        git config user.email "github-actions[bot]@users.noreply.github.com"
+        git tag "v${{ steps.version.outputs.version }}"
+        git push origin "v${{ steps.version.outputs.version }}"
+        echo "🏷️ Created and pushed tag: v${{ steps.version.outputs.version }}"
+
+    - name: Tag created successfully
+      if: steps.check.outputs.exists == 'false'
+      run: |
+        echo "✨ Tag v${{ steps.version.outputs.version }} has been created!"
+        echo "🚀 This will trigger the publish workflow automatically."

{content_core-1.5.0 → content_core-1.7.0}/CHANGELOG.md

@@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Pure Python file type detection via the new `FileDetector` class
 - Comprehensive file signature detection for 25+ file formats
 - Smart detection for ZIP-based formats (DOCX, XLSX, PPTX, EPUB)
+- Custom audio model configuration - override speech-to-text provider and model at runtime
+  - Pass `audio_provider` and `audio_model` parameters through `extract_content()` API
+  - Supports any provider/model combination available through Esperanto library
+  - Maintains full backward compatibility - existing code works unchanged
+  - Includes validation with helpful warnings and error messages
 
 ### Changed
 - File type detection now uses pure Python implementation instead of libmagic

{content_core-1.5.0 → content_core-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 1.5.0
+Version: 1.7.0
 Summary: Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE
@@ -263,6 +263,14 @@ cleaned_text = await cc.clean("...messy text with [brackets] and extra spaces...
 
 # Summarize content with optional context
 summary = await cc.summarize_content("long article text", context="explain to a child")
+
+# Extract audio with custom speech-to-text model
+from content_core.common import ProcessSourceInput
+result = await cc.extract(ProcessSourceInput(
+    file_path="interview.mp3",
+    audio_provider="openai",
+    audio_model="whisper-1"
+))
 ```
 
 ## Documentation

{content_core-1.5.0 → content_core-1.7.0}/README.md

@@ -228,6 +228,14 @@ cleaned_text = await cc.clean("...messy text with [brackets] and extra spaces...
 
 # Summarize content with optional context
 summary = await cc.summarize_content("long article text", context="explain to a child")
+
+# Extract audio with custom speech-to-text model
+from content_core.common import ProcessSourceInput
+result = await cc.extract(ProcessSourceInput(
+    file_path="interview.mp3",
+    audio_provider="openai",
+    audio_model="whisper-1"
+))
 ```
 
 ## Documentation

{content_core-1.5.0 → content_core-1.7.0}/docs/usage.md

@@ -362,6 +362,203 @@ result = await cc.extract({"file_path": "conference_talk.mp4"})
 print(result.content)  # Full transcript
 ```
 
+## Custom Audio Model Configuration
+
+Content Core allows you to override the default speech-to-text model at runtime, enabling you to choose different AI providers and models based on your specific needs (language support, cost, accuracy, etc.).
+
+### Overview
+
+By default, audio and video files are transcribed using the model configured in `models_config.yaml` (typically OpenAI Whisper-1). You can override this on a per-call basis by specifying both `audio_provider` and `audio_model` parameters.
+
+**Key Features:**
+- ✅ **Runtime flexibility**: Choose different models for different use cases
+- ✅ **Backward compatible**: Existing code works unchanged
+- ✅ **Multiple providers**: Support for any provider supported by Esperanto
+- ✅ **Automatic fallback**: Graceful handling of invalid configurations
+
+### Basic Usage
+
+```python
+from content_core.common import ProcessSourceInput
+import content_core as cc
+
+# Use custom audio model for transcription
+result = await cc.extract(ProcessSourceInput(
+    file_path="interview.mp3",
+    audio_provider="openai",
+    audio_model="whisper-1"
+))
+
+print(result.content)  # Transcribed text using specified model
+```
+
+### Supported Providers
+
+Content Core uses the Esperanto library for AI model abstraction, which supports multiple providers:
+
+- **OpenAI**: `provider="openai"`, models: `whisper-1`
+- **Google**: `provider="google"`, models: `chirp` (if available)
+- **Other providers**: Any provider supported by Esperanto
+
+Check the [Esperanto documentation](https://github.com/yourusername/esperanto) for the full list of supported providers and models.
+
+### Use Cases
+
+**Multilingual Transcription:**
+```python
+from content_core.common import ProcessSourceInput
+import content_core as cc
+
+# Use a model optimized for a specific language
+result = await cc.extract(ProcessSourceInput(
+    file_path="spanish_interview.mp3",
+    audio_provider="openai",
+    audio_model="whisper-1"  # Whisper supports 99 languages
+))
+```
+
+**Cost Optimization:**
+```python
+from content_core.common import ProcessSourceInput
+import content_core as cc
+
+# Use different models based on quality requirements
+# For high-value content, use premium model
+premium_result = await cc.extract(ProcessSourceInput(
+    file_path="important_meeting.mp3",
+    audio_provider="openai",
+    audio_model="whisper-1"
+))
+
+# For casual content, use default or cost-effective model
+casual_result = await cc.extract(ProcessSourceInput(
+    file_path="casual_recording.mp3"
+    # No custom params = uses default configured model
+))
+```
+
+**Video Transcription with Custom Model:**
+```python
+from content_core.common import ProcessSourceInput
+import content_core as cc
+
+# Custom model works for video files too (audio is extracted automatically)
+result = await cc.extract(ProcessSourceInput(
+    file_path="conference_presentation.mp4",
+    audio_provider="openai",
+    audio_model="whisper-1"
+))
+```
+
+### Parameter Requirements
+
+Both `audio_provider` and `audio_model` must be specified together:
+
+```python
+# ✅ CORRECT: Both parameters provided
+result = await cc.extract(ProcessSourceInput(
+    file_path="audio.mp3",
+    audio_provider="openai",
+    audio_model="whisper-1"
+))
+
+# ✅ CORRECT: Neither parameter (uses default)
+result = await cc.extract(ProcessSourceInput(
+    file_path="audio.mp3"
+))
+
+# ⚠️ WARNING: Only one parameter (logs warning, uses default)
+result = await cc.extract(ProcessSourceInput(
+    file_path="audio.mp3",
+    audio_provider="openai"  # Missing audio_model
+))
+# Logs: "audio_provider provided without audio_model. Both must be specified together. Falling back to default model."
+```
+
+### Error Handling
+
+Content Core gracefully handles invalid model configurations:
+
+**Invalid Provider:**
+```python
+result = await cc.extract(ProcessSourceInput(
+    file_path="audio.mp3",
+    audio_provider="invalid_provider",
+    audio_model="whisper-1"
+))
+# Logs error and falls back to default model
+# Transcription continues successfully
+```
+
+**Invalid Model Name:**
+```python
+result = await cc.extract(ProcessSourceInput(
+    file_path="audio.mp3",
+    audio_provider="openai",
+    audio_model="nonexistent-model"
+))
+# Logs error and falls back to default model
+# Transcription continues successfully
+```
+
+**Error Message Example:**
+```
+ERROR: Failed to create custom audio model 'invalid_provider/whisper-1': Unsupported provider.
+Check that the provider and model are supported by Esperanto. Falling back to default model.
+```
+
+### Concurrency Control
+
+Custom audio models respect the same concurrency limits as the default model (configured via `CCORE_AUDIO_CONCURRENCY` or `set_audio_concurrency()`). This ensures consistent API rate limit handling regardless of which model you use.
+
+```python
+from content_core.config import set_audio_concurrency
+from content_core.common import ProcessSourceInput
+import content_core as cc
+
+# Set concurrency for all transcriptions (default and custom models)
+set_audio_concurrency(5)
+
+# Both use the same concurrency limit
+default_result = await cc.extract(ProcessSourceInput(file_path="audio1.mp3"))
+custom_result = await cc.extract(ProcessSourceInput(
+    file_path="audio2.mp3",
+    audio_provider="openai",
+    audio_model="whisper-1"
+))
+```
+
+### Backward Compatibility
+
+All existing code continues to work without any changes:
+
+```python
+import content_core as cc
+
+# Old code (no custom params) - still works perfectly
+result = await cc.extract("audio.mp3")
+result = await cc.extract({"file_path": "audio.mp3"})
+
+# New capability (optional custom params)
+from content_core.common import ProcessSourceInput
+result = await cc.extract(ProcessSourceInput(
+    file_path="audio.mp3",
+    audio_provider="openai",
+    audio_model="whisper-1"
+))
+```
+
+### Troubleshooting
+
+**Issue**: "Both audio_provider and audio_model must be specified together"
+- **Solution**: Provide both parameters or neither. Don't specify just one.
+
+**Issue**: "Failed to create custom audio model"
+- **Solution**: Verify the provider and model are supported by Esperanto. Check your API keys are configured correctly.
+
+**Issue**: Custom model seems to be ignored
+- **Solution**: Ensure you're using `ProcessSourceInput` class (not plain dict) when passing custom parameters.
+
 ## File Type Detection
 
 Content Core uses a pure Python implementation for file type detection, eliminating the need for system dependencies like libmagic. This ensures consistent behavior across all platforms (Windows, macOS, Linux).

{content_core-1.5.0 → content_core-1.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "content-core"
-version = "1.5.0"
+version = "1.7.0"
 description = "Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server"
 readme = "README.md"
 homepage = "https://github.com/lfnovo/content-core"

{content_core-1.5.0 → content_core-1.7.0}/src/content_core/__init__.py

@@ -214,5 +214,3 @@ def csum():
 
 if __name__ == "__main__":
     ccore()
-if __name__ == "__main__":
-    ccore()

{content_core-1.5.0 → content_core-1.7.0}/src/content_core/common/state.py

@@ -27,6 +27,14 @@ class ProcessSourceState(BaseModel):
         default=None,
         description="Override Docling output format: 'markdown', 'html', or 'json'",
     )
+    audio_provider: Optional[str] = Field(
+        default=None,
+        description="Override speech-to-text provider (e.g., 'openai', 'google')",
+    )
+    audio_model: Optional[str] = Field(
+        default=None,
+        description="Override speech-to-text model name (e.g., 'whisper-1', 'chirp')",
+    )
 
 
 class ProcessSourceInput(BaseModel):
@@ -36,6 +44,8 @@ class ProcessSourceInput(BaseModel):
     document_engine: Optional[str] = None
     url_engine: Optional[str] = None
     output_format: Optional[str] = None
+    audio_provider: Optional[str] = None
+    audio_model: Optional[str] = None
 
 
 class ProcessSourceOutput(BaseModel):

{content_core-1.5.0 → content_core-1.7.0}/src/content_core/content/identification/file_detector.py

@@ -13,10 +13,17 @@ from content_core.logging import logger
 
 class FileDetector:
     """Pure Python file type detection using magic bytes and content analysis."""
-    
-    # Configuration constants
+
+    # Configuration constants for binary/text detection
     SIGNATURE_READ_SIZE = 512  # Bytes to read for binary signature detection
     TEXT_READ_SIZE = 1024      # Bytes to read for text content analysis
+
+    # Configuration constants for CSV detection
+    CSV_MAX_FIELD_LENGTH = 100  # Maximum average field length for CSV (longer suggests prose)
+    CSV_MAX_VARIANCE = 500      # Maximum variance in field lengths (higher suggests natural text)
+    CSV_MIN_SCORE = 2           # Minimum score required to classify as CSV
+    CSV_MIN_FIELDS = 2          # Minimum number of fields required for CSV
+    CSV_MAX_HEADER_FIELD_LENGTH = 50  # Maximum length for individual header fields
     
     def __init__(self):
         """Initialize the FileDetector with signature mappings."""
@@ -364,18 +371,102 @@ class FileDetector:
     
     
     def _looks_like_csv(self, content: str) -> bool:
-        """Check if content looks like CSV format."""
-        lines = content.split('\n', 5)[:5]  # Check first 5 lines
-        if len(lines) < 2:
+        """
+        Check if content looks like CSV format with improved heuristics.
+
+        Uses a multi-stage approach with performance optimization:
+        1. Basic structural checks (cheap)
+        2. Field length analysis (cheap, early exit)
+        3. Pattern matching (moderate cost)
+        4. Variance analysis (expensive, only if needed)
+        """
+        lines = content.split('\n', 10)[:10]  # Check first 10 lines for better accuracy
+        non_empty_lines = [line for line in lines if line.strip()]
+
+        # Stage 1: Basic structural checks
+        if len(non_empty_lines) < 2:
             return False
-        
+
         # Count commas in each line
-        comma_counts = [line.count(',') for line in lines if line.strip()]
-        if not comma_counts:
+        comma_counts = [line.count(',') for line in non_empty_lines]
+
+        # Must have at least one comma per line
+        if not all(count > 0 for count in comma_counts):
             return False
-        
-        # CSV should have consistent comma counts
-        return len(set(comma_counts)) == 1 and comma_counts[0] > 0
+
+        # CSV should have consistent comma counts across lines
+        if len(set(comma_counts)) != 1:
+            return False
+
+        num_fields = comma_counts[0] + 1  # Number of fields = commas + 1
+
+        # Must have minimum number of fields to be CSV
+        if num_fields < self.CSV_MIN_FIELDS:
+            return False
+
+        # Stage 2: Field length analysis (PERFORMANCE OPTIMIZATION: early exit)
+        first_line = non_empty_lines[0]
+        fields = first_line.split(',')
+
+        # CSV fields should be relatively short (not long sentences)
+        # Average field length should be reasonable (not paragraphs)
+        # Early exit avoids expensive variance calculations for obvious prose
+        avg_field_length = sum(len(f.strip()) for f in fields) / len(fields)
+        if avg_field_length > self.CSV_MAX_FIELD_LENGTH:
+            return False  # Too long to be typical CSV fields - exit early
+
+        # Stage 3: Pattern matching
+        # Check for CSV-like patterns:
+        # 1. Fields that look like headers (short, alphanumeric)
+        # 2. Quoted fields (common in CSV)
+        # 3. Numeric fields
+        has_quoted_fields = any('"' in line or "'" in line for line in non_empty_lines[:3])
+
+        first_line_fields = [f.strip() for f in fields]
+        # Check if first line looks like a header (short, no sentence-ending punctuation)
+        looks_like_header = all(
+            len(f) < self.CSV_MAX_HEADER_FIELD_LENGTH and not f.endswith('.') and not f.endswith('!')
+            for f in first_line_fields
+        )
+
+        # Stage 4: Variance analysis (EXPENSIVE - only if we have enough data)
+        # Check if subsequent lines have similar field structure
+        # Real CSV tends to have consistent field lengths
+        if len(non_empty_lines) >= 3:
+            field_lengths_per_line = []
+            for line in non_empty_lines[:5]:
+                line_fields = line.split(',')
+                field_lengths = [len(f.strip()) for f in line_fields]
+                field_lengths_per_line.append(field_lengths)
+
+            # Calculate variance in field positions
+            # CSV data should have relatively consistent field lengths at each position
+            # Natural text with commas will have much more variance
+            position_variances = []
+            for i in range(num_fields):
+                lengths_at_position = [fl[i] if i < len(fl) else 0 for fl in field_lengths_per_line]
+                if lengths_at_position:
+                    avg = sum(lengths_at_position) / len(lengths_at_position)
+                    variance = sum((x - avg) ** 2 for x in lengths_at_position) / len(lengths_at_position)
+                    position_variances.append(variance)
+
+            # High variance suggests natural text, not structured CSV
+            if position_variances:
+                avg_variance = sum(position_variances) / len(position_variances)
+                if avg_variance > self.CSV_MAX_VARIANCE:
+                    return False  # Very high variance = likely prose
+
+        # Scoring: Require at least some CSV-like characteristics
+        csv_score = 0
+        if looks_like_header:
+            csv_score += 1
+        if has_quoted_fields:
+            csv_score += 1
+        if num_fields >= 3:  # Multiple fields is more CSV-like
+            csv_score += 1
+
+        # Need minimum score to confidently classify as CSV
+        return csv_score >= self.CSV_MIN_SCORE
     
     
     def _is_text_file(self, content: str) -> bool:

{content_core-1.5.0 → content_core-1.7.0}/src/content_core/processors/audio.py

@@ -45,7 +45,7 @@ async def split_audio(input_file, segment_length_minutes=15, output_prefix=None)
             end_time = min((i + 1) * segment_length_s, audio.duration)
 
             # Extract segment
-            output_filename = f"{output_prefix}_{str(i+1).zfill(3)}.mp3"
+            output_filename = f"{output_prefix}_{str(i + 1).zfill(3)}.mp3"
             output_path = os.path.join(output_dir, output_filename)
 
             # Export segment
@@ -53,7 +53,9 @@ async def split_audio(input_file, segment_length_minutes=15, output_prefix=None)
 
             output_files.append(output_path)
 
-            logger.debug(f"Exported segment {i+1}/{total_segments}: {output_filename}")
+            logger.debug(
+                f"Exported segment {i + 1}/{total_segments}: {output_filename}"
+            )
 
         return output_files
 
@@ -172,7 +174,7 @@ async def extract_audio_data(data: ProcessSourceState):
                     end_time = min((i + 1) * segment_length_s, audio.duration)
 
                     # Extract segment
-                    output_filename = f"{output_prefix}_{str(i+1).zfill(3)}.mp3"
+                    output_filename = f"{output_prefix}_{str(i + 1).zfill(3)}.mp3"
                     output_path = os.path.join(output_dir, output_filename)
 
                     extract_audio(input_audio_path, output_path, start_time, end_time)
@@ -188,12 +190,45 @@ async def extract_audio_data(data: ProcessSourceState):
 
             # Transcribe audio files in parallel with concurrency limit
             from content_core.models import ModelFactory
+            from esperanto import AIFactory
+
+            # Determine which model to use based on state parameters
+            if data.audio_provider and data.audio_model:
+                # Custom model provided - create new instance
+                try:
+                    logger.info(
+                        f"Using custom audio model: {data.audio_provider}/{data.audio_model}"
+                    )
+                    speech_to_text_model = AIFactory.create_speech_to_text(
+                        data.audio_provider, data.audio_model
+                    )
+                except Exception as e:
+                    logger.error(
+                        f"Failed to create custom audio model '{data.audio_provider}/{data.audio_model}': {e}. "
+                        f"Check that the provider and model are supported by Esperanto. "
+                        f"Falling back to default model."
+                    )
+                    speech_to_text_model = ModelFactory.get_model("speech_to_text")
+            elif data.audio_provider or data.audio_model:
+                # Only one parameter provided - log warning and use default
+                missing = "audio_model" if data.audio_provider else "audio_provider"
+                provided = "audio_provider" if data.audio_provider else "audio_model"
+                logger.warning(
+                    f"{provided} provided without {missing}. "
+                    f"Both audio_provider and audio_model must be specified together. "
+                    f"Falling back to default model."
+                )
+                speech_to_text_model = ModelFactory.get_model("speech_to_text")
+            else:
+                # No custom parameters - use default (backward compatible)
+                speech_to_text_model = ModelFactory.get_model("speech_to_text")
 
-            speech_to_text_model = ModelFactory.get_model("speech_to_text")
             concurrency = get_audio_concurrency()
             semaphore = asyncio.Semaphore(concurrency)
 
-            logger.debug(f"Transcribing {len(output_files)} audio segments with concurrency limit of {concurrency}")
+            logger.debug(
+                f"Transcribing {len(output_files)} audio segments with concurrency limit of {concurrency}"
+            )
 
             # Create tasks for parallel transcription
             transcription_tasks = [

{content_core-1.5.0 → content_core-1.7.0}/src/content_core/templated_message.py

@@ -2,7 +2,6 @@ from typing import Dict, Optional, Union
 
 from ai_prompter import Prompter
 from esperanto import LanguageModel
-from esperanto.common_types import Message
 from pydantic import BaseModel, Field
 
 from content_core.models import ModelFactory

{content_core-1.5.0 → content_core-1.7.0}/tests/integration/conftest.py

@@ -1,7 +1,6 @@
 """Pytest configuration for integration tests."""
 import asyncio
 import gc
-import warnings
 
 import pytest