content-core 1.5.0__tar.gz → 1.6.0__tar.gz

content_core-1.6.0/.github/workflows/create-tag.yml

@@ -0,0 +1,56 @@
+name: Create Tag
+
+on:
+  workflow_dispatch:
+    inputs:
+      confirm:
+        description: 'Type "yes" to confirm tag creation'
+        required: true
+        default: 'no'
+
+jobs:
+  create-tag:
+    runs-on: ubuntu-latest
+    if: github.event.inputs.confirm == 'yes'
+    permissions:
+      contents: write
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Extract version from pyproject.toml
+      id: version
+      run: |
+        VERSION=$(grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
+        echo "version=$VERSION" >> $GITHUB_OUTPUT
+        echo "tag=v$VERSION" >> $GITHUB_OUTPUT
+        echo "📦 Version found: $VERSION"
+
+    - name: Check if tag already exists
+      id: check
+      run: |
+        if git rev-parse "v${{ steps.version.outputs.version }}" >/dev/null 2>&1; then
+          echo "exists=true" >> $GITHUB_OUTPUT
+          echo "❌ Tag v${{ steps.version.outputs.version }} already exists!"
+          exit 1
+        else
+          echo "exists=false" >> $GITHUB_OUTPUT
+          echo "✅ Tag v${{ steps.version.outputs.version }} does not exist yet"
+        fi
+
+    - name: Create and push tag
+      if: steps.check.outputs.exists == 'false'
+      run: |
+        git config user.name "github-actions[bot]"
+        git config user.email "github-actions[bot]@users.noreply.github.com"
+        git tag "v${{ steps.version.outputs.version }}"
+        git push origin "v${{ steps.version.outputs.version }}"
+        echo "🏷️ Created and pushed tag: v${{ steps.version.outputs.version }}"
+
+    - name: Tag created successfully
+      if: steps.check.outputs.exists == 'false'
+      run: |
+        echo "✨ Tag v${{ steps.version.outputs.version }} has been created!"
+        echo "🚀 This will trigger the publish workflow automatically."

{content_core-1.5.0 → content_core-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 1.5.0
+Version: 1.6.0
 Summary: Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE

{content_core-1.5.0 → content_core-1.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "content-core"
-version = "1.5.0"
+version = "1.6.0"
 description = "Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server"
 readme = "README.md"
 homepage = "https://github.com/lfnovo/content-core"

{content_core-1.5.0 → content_core-1.6.0}/src/content_core/__init__.py

@@ -214,5 +214,3 @@ def csum():
 
 if __name__ == "__main__":
     ccore()
-if __name__ == "__main__":
-    ccore()

{content_core-1.5.0 → content_core-1.6.0}/src/content_core/content/identification/file_detector.py

@@ -13,10 +13,17 @@ from content_core.logging import logger
 
 class FileDetector:
     """Pure Python file type detection using magic bytes and content analysis."""
-    
-    # Configuration constants
+
+    # Configuration constants for binary/text detection
     SIGNATURE_READ_SIZE = 512  # Bytes to read for binary signature detection
     TEXT_READ_SIZE = 1024      # Bytes to read for text content analysis
+
+    # Configuration constants for CSV detection
+    CSV_MAX_FIELD_LENGTH = 100  # Maximum average field length for CSV (longer suggests prose)
+    CSV_MAX_VARIANCE = 500      # Maximum variance in field lengths (higher suggests natural text)
+    CSV_MIN_SCORE = 2           # Minimum score required to classify as CSV
+    CSV_MIN_FIELDS = 2          # Minimum number of fields required for CSV
+    CSV_MAX_HEADER_FIELD_LENGTH = 50  # Maximum length for individual header fields
     
     def __init__(self):
         """Initialize the FileDetector with signature mappings."""
@@ -364,18 +371,102 @@ class FileDetector:
     
     
     def _looks_like_csv(self, content: str) -> bool:
-        """Check if content looks like CSV format."""
-        lines = content.split('\n', 5)[:5]  # Check first 5 lines
-        if len(lines) < 2:
+        """
+        Check if content looks like CSV format with improved heuristics.
+
+        Uses a multi-stage approach with performance optimization:
+        1. Basic structural checks (cheap)
+        2. Field length analysis (cheap, early exit)
+        3. Pattern matching (moderate cost)
+        4. Variance analysis (expensive, only if needed)
+        """
+        lines = content.split('\n', 10)[:10]  # Check first 10 lines for better accuracy
+        non_empty_lines = [line for line in lines if line.strip()]
+
+        # Stage 1: Basic structural checks
+        if len(non_empty_lines) < 2:
             return False
-        
+
         # Count commas in each line
-        comma_counts = [line.count(',') for line in lines if line.strip()]
-        if not comma_counts:
+        comma_counts = [line.count(',') for line in non_empty_lines]
+
+        # Must have at least one comma per line
+        if not all(count > 0 for count in comma_counts):
             return False
-        
-        # CSV should have consistent comma counts
-        return len(set(comma_counts)) == 1 and comma_counts[0] > 0
+
+        # CSV should have consistent comma counts across lines
+        if len(set(comma_counts)) != 1:
+            return False
+
+        num_fields = comma_counts[0] + 1  # Number of fields = commas + 1
+
+        # Must have minimum number of fields to be CSV
+        if num_fields < self.CSV_MIN_FIELDS:
+            return False
+
+        # Stage 2: Field length analysis (PERFORMANCE OPTIMIZATION: early exit)
+        first_line = non_empty_lines[0]
+        fields = first_line.split(',')
+
+        # CSV fields should be relatively short (not long sentences)
+        # Average field length should be reasonable (not paragraphs)
+        # Early exit avoids expensive variance calculations for obvious prose
+        avg_field_length = sum(len(f.strip()) for f in fields) / len(fields)
+        if avg_field_length > self.CSV_MAX_FIELD_LENGTH:
+            return False  # Too long to be typical CSV fields - exit early
+
+        # Stage 3: Pattern matching
+        # Check for CSV-like patterns:
+        # 1. Fields that look like headers (short, alphanumeric)
+        # 2. Quoted fields (common in CSV)
+        # 3. Numeric fields
+        has_quoted_fields = any('"' in line or "'" in line for line in non_empty_lines[:3])
+
+        first_line_fields = [f.strip() for f in fields]
+        # Check if first line looks like a header (short, no sentence-ending punctuation)
+        looks_like_header = all(
+            len(f) < self.CSV_MAX_HEADER_FIELD_LENGTH and not f.endswith('.') and not f.endswith('!')
+            for f in first_line_fields
+        )
+
+        # Stage 4: Variance analysis (EXPENSIVE - only if we have enough data)
+        # Check if subsequent lines have similar field structure
+        # Real CSV tends to have consistent field lengths
+        if len(non_empty_lines) >= 3:
+            field_lengths_per_line = []
+            for line in non_empty_lines[:5]:
+                line_fields = line.split(',')
+                field_lengths = [len(f.strip()) for f in line_fields]
+                field_lengths_per_line.append(field_lengths)
+
+            # Calculate variance in field positions
+            # CSV data should have relatively consistent field lengths at each position
+            # Natural text with commas will have much more variance
+            position_variances = []
+            for i in range(num_fields):
+                lengths_at_position = [fl[i] if i < len(fl) else 0 for fl in field_lengths_per_line]
+                if lengths_at_position:
+                    avg = sum(lengths_at_position) / len(lengths_at_position)
+                    variance = sum((x - avg) ** 2 for x in lengths_at_position) / len(lengths_at_position)
+                    position_variances.append(variance)
+
+            # High variance suggests natural text, not structured CSV
+            if position_variances:
+                avg_variance = sum(position_variances) / len(position_variances)
+                if avg_variance > self.CSV_MAX_VARIANCE:
+                    return False  # Very high variance = likely prose
+
+        # Scoring: Require at least some CSV-like characteristics
+        csv_score = 0
+        if looks_like_header:
+            csv_score += 1
+        if has_quoted_fields:
+            csv_score += 1
+        if num_fields >= 3:  # Multiple fields is more CSV-like
+            csv_score += 1
+
+        # Need minimum score to confidently classify as CSV
+        return csv_score >= self.CSV_MIN_SCORE
     
     
     def _is_text_file(self, content: str) -> bool:

{content_core-1.5.0 → content_core-1.6.0}/src/content_core/processors/audio.py

@@ -45,7 +45,7 @@ async def split_audio(input_file, segment_length_minutes=15, output_prefix=None)
             end_time = min((i + 1) * segment_length_s, audio.duration)
 
             # Extract segment
-            output_filename = f"{output_prefix}_{str(i+1).zfill(3)}.mp3"
+            output_filename = f"{output_prefix}_{str(i + 1).zfill(3)}.mp3"
             output_path = os.path.join(output_dir, output_filename)
 
             # Export segment
@@ -53,7 +53,9 @@ async def split_audio(input_file, segment_length_minutes=15, output_prefix=None)
 
             output_files.append(output_path)
 
-            logger.debug(f"Exported segment {i+1}/{total_segments}: {output_filename}")
+            logger.debug(
+                f"Exported segment {i + 1}/{total_segments}: {output_filename}"
+            )
 
         return output_files
 
@@ -172,7 +174,7 @@ async def extract_audio_data(data: ProcessSourceState):
                     end_time = min((i + 1) * segment_length_s, audio.duration)
 
                     # Extract segment
-                    output_filename = f"{output_prefix}_{str(i+1).zfill(3)}.mp3"
+                    output_filename = f"{output_prefix}_{str(i + 1).zfill(3)}.mp3"
                     output_path = os.path.join(output_dir, output_filename)
 
                     extract_audio(input_audio_path, output_path, start_time, end_time)
@@ -193,7 +195,9 @@ async def extract_audio_data(data: ProcessSourceState):
             concurrency = get_audio_concurrency()
             semaphore = asyncio.Semaphore(concurrency)
 
-            logger.debug(f"Transcribing {len(output_files)} audio segments with concurrency limit of {concurrency}")
+            logger.debug(
+                f"Transcribing {len(output_files)} audio segments with concurrency limit of {concurrency}"
+            )
 
             # Create tasks for parallel transcription
             transcription_tasks = [