content-core 1.10.0__py3-none-any.whl

content_core/notebooks/urls.ipynb

@@ -0,0 +1,154 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "873a872b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from content_core.content.extraction import extract_content\n",
+    "\n",
+    "async def process_url(url):\n",
+    "    print(\"Processing: \", url)\n",
+    "    print(\"Simple: -------\")\n",
+    "    result = await extract_content(dict(url=url, engine=\"simple\"))\n",
+    "    print(result.title[:100])\n",
+    "    print(result.content[:100])\n",
+    "    print(\"Jina: -------\")\n",
+    "    result = await extract_content(dict(url=url, engine=\"jina\"))\n",
+    "    print(result.title[:100])\n",
+    "    print(result.content[:100])\n",
+    "    print(\"Firecrawl: -------\")\n",
+    "    result = await extract_content(dict(url=url, engine=\"firecrawl\"))\n",
+    "    print(result.title[:100])\n",
+    "    print(result.content[:100])\n",
+    "    print(\"=============================\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "263dc3af",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing:  https://www.supernovalabs.com.br/\n",
+      "Simple: -------\n",
+      "Readability failed: No content extracted by readability\n",
+      "Supernova Labs | AI Consulting\n",
+      "Supernova Labs | AI Consulting\n",
+      "Jina: -------\n",
+      "Supernova Labs | Elite AI Consulting to help you build the Future\n",
+      "URL Source: https://www.supernovalabs.com.br/\n",
+      "\n",
+      "Markdown Content:\n",
+      "Supernova Labs\n",
+      "\n",
+      "[About](https://www\n",
+      "Firecrawl: -------\n",
+      "Supernova Labs | AI Consulting\n",
+      "# Unleash Your AI Edge. Fast.\n",
+      "\n",
+      "We turn your data, tech and capabilities into impact with lean AI sol\n",
+      "=============================\n",
+      "None\n",
+      "Processing:  https://building.nubank.com/fine-tuning-transaction-user-models/\n",
+      "Simple: -------\n",
+      "Fine-Tuning Transaction User Models - Building Nubank\n",
+      "Fine-Tuning Transaction User Models Learn how we combine transaction embeddings with tabular data us\n",
+      "Jina: -------\n",
+      "Fine-Tuning Transaction User Models - Building Nubank\n",
+      "URL Source: https://building.nubank.com/fine-tuning-transaction-user-models/\n",
+      "\n",
+      "Published Time: 2025-0\n",
+      "Firecrawl: -------\n",
+      "Fine-Tuning Transaction User Models - Building Nubank\n",
+      "# Fine-Tuning Transaction User Models\n",
+      "\n",
+      "Learn how we combine transaction embeddings with tabular data\n",
+      "=============================\n",
+      "None\n",
+      "Processing:  https://medium.com/writing-for-profit-with-ai/you-can-make-money-with-ai-without-quitting-your-job-5296bbcb703b\n",
+      "Simple: -------\n",
+      "You Can Make Money With AI Without Quitting Your Job | by Nipuna Maduranga | LearnAIforproft.com | M\n",
+      "Most people think they need to quit their job to build a new life. I thought that too. You scroll th\n",
+      "Jina: -------\n",
+      "You Can Make Money With AI Without Quitting Your Job\n",
+      "URL Source: https://medium.com/writing-for-profit-with-ai/you-can-make-money-with-ai-without-quittin\n",
+      "Firecrawl: -------\n",
+      "You Can Make Money With AI Without Quitting Your Job | by Nipuna Maduranga | LearnAIforproft.com | M\n",
+      "[Sitemap](https://medium.com/sitemap/sitemap.xml)\n",
+      "\n",
+      "[Open in app](https://rsci.app.link/?%24canonical\n",
+      "=============================\n",
+      "None\n",
+      "Processing:  https://github.com/mirkonasato/pyodconverter\n",
+      "Simple: -------\n",
+      "GitHub - mirkonasato/pyodconverter: Python script to automate document conversions using LibreOffice\n",
+      "This repository was archived by the owner on Dec 1, 2023. It is now read-only. mirkonasato/pyodconve\n",
+      "Jina: -------\n",
+      "GitHub - mirkonasato/pyodconverter: Python script to automate document conversions using LibreOffice\n",
+      "URL Source: https://github.com/mirkonasato/pyodconverter\n",
+      "\n",
+      "Markdown Content:\n",
+      "GitHub - mirkonasato/pyo\n",
+      "Firecrawl: -------\n",
+      "GitHub - mirkonasato/pyodconverter: Python script to automate document conversions using LibreOffice\n",
+      "[Skip to content](https://github.com/mirkonasato/pyodconverter#start-of-content)\n",
+      "\n",
+      "You signed in with\n",
+      "=============================\n",
+      "None\n",
+      "Processing:  https://www.amazon.com.br/Ultra-aprendizado-habilidades-valiosas-competi%C3%A7%C3%A3o-carreira/dp/6555110058/ref=asc_df_6555110058?tag=googleshopp00-20&hvadid=709857900630&hvpos=&hvnetw=g&hvrand=17798174883330212364&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9195894&hvtargid=pla-1148630207439&psc=1&language=pt_BR\n",
+      "Simple: -------\n",
+      "Error processing URL https://www.amazon.com.br/Ultra-aprendizado-habilidades-valiosas-competi%C3%A7%C3%A3o-carreira/dp/6555110058/ref=asc_df_6555110058?tag=googleshopp00-20&hvadid=709857900630&hvpos=&hvnetw=g&hvrand=17798174883330212364&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9195894&hvtargid=pla-1148630207439&psc=1&language=pt_BR: HTTP error: 500\n",
+      "Error\n",
+      "Failed to extract content: HTTP error: 500\n",
+      "Jina: -------\n",
+      "Ultra-aprendizado: domine habilidades valiosas, seja mais esperto que a competição e dê um impulso n\n",
+      "URL Source: https://www.amazon.com.br/Ultra-aprendizado-habilidades-valiosas-competi%C3%A7%C3%A3o-ca\n",
+      "Firecrawl: -------\n",
+      "Amazon.com.br\n",
+      "#### Digite os caracteres que você vê abaixo\n",
+      "\n",
+      "Desculpe pelo inconveniente. Para continuar realizando\n",
+      "=============================\n",
+      "None\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "urls= [\"https://www.supernovalabs.com.br/\", \"https://building.nubank.com/fine-tuning-transaction-user-models/\", \"https://medium.com/writing-for-profit-with-ai/you-can-make-money-with-ai-without-quitting-your-job-5296bbcb703b\", \"https://github.com/mirkonasato/pyodconverter\",  \"https://www.amazon.com.br/Ultra-aprendizado-habilidades-valiosas-competi%C3%A7%C3%A3o-carreira/dp/6555110058/ref=asc_df_6555110058?tag=googleshopp00-20&hvadid=709857900630&hvpos=&hvnetw=g&hvrand=17798174883330212364&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9195894&hvtargid=pla-1148630207439&psc=1&language=pt_BR\"]\n",
+    "for url in urls:\n",
+    "    result = await process_url(url=url)\n",
+    "    print(result)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

content_core/processors/audio.py

@@ -0,0 +1,272 @@
+import asyncio
+import math
+import os
+import tempfile
+import traceback
+from functools import partial
+
+from moviepy import AudioFileClip
+
+from content_core.common import ProcessSourceState
+from content_core.common.retry import retry_audio_transcription
+from content_core.config import get_audio_concurrency, get_proxy
+from content_core.logging import logger
+
+
+async def split_audio(input_file, segment_length_minutes=15, output_prefix=None):
+    """
+    Split an audio file into segments asynchronously.
+    """
+
+    def _split(input_file, segment_length_minutes, output_prefix):
+        # Convert input file to absolute path
+        input_file_abs = os.path.abspath(input_file)
+        output_dir = os.path.dirname(input_file_abs)
+        os.makedirs(output_dir, exist_ok=True)
+
+        # Set up output prefix
+        if output_prefix is None:
+            output_prefix = os.path.splitext(os.path.basename(input_file_abs))[0]
+
+        # Load the audio file
+        audio = AudioFileClip(input_file_abs)
+
+        # Calculate segment length in seconds
+        segment_length_s = segment_length_minutes * 60
+
+        # Calculate number of segments
+        total_segments = math.ceil(audio.duration / segment_length_s)
+        logger.debug(f"Splitting file: {input_file_abs} into {total_segments} segments")
+
+        output_files = []
+
+        # Split the audio into segments
+        for i in range(total_segments):
+            start_time = i * segment_length_s
+            end_time = min((i + 1) * segment_length_s, audio.duration)
+
+            # Extract segment
+            output_filename = f"{output_prefix}_{str(i + 1).zfill(3)}.mp3"
+            output_path = os.path.join(output_dir, output_filename)
+
+            # Export segment
+            extract_audio(input_file_abs, output_path, start_time, end_time)
+
+            output_files.append(output_path)
+
+            logger.debug(
+                f"Exported segment {i + 1}/{total_segments}: {output_filename}"
+            )
+
+        return output_files
+
+    # Run CPU-bound audio processing in thread pool
+    return await asyncio.get_event_loop().run_in_executor(
+        None, partial(_split, input_file, segment_length_minutes, output_prefix)
+    )
+
+
+def extract_audio(
+    input_file: str, output_file: str, start_time: float = None, end_time: float = None
+) -> None:
+    """
+    Extract audio from a video or audio file and save it as an MP3 file.
+    If start_time and end_time are provided, only that segment of audio is extracted.
+
+    Args:
+        input_file (str): Path to the input video or audio file.
+        output_file (str): Path where the output MP3 file will be saved.
+        start_time (float, optional): Start time of the audio segment in seconds. Defaults to None.
+        end_time (float, optional): End time of the audio segment in seconds. Defaults to None.
+    """
+    try:
+        # Load the file as an AudioFileClip
+        audio_clip = AudioFileClip(input_file)
+
+        # If start_time and/or end_time are provided, trim the audio using subclipped
+        if start_time is not None and end_time is not None:
+            audio_clip = audio_clip.subclipped(start_time, end_time)
+        elif start_time is not None:
+            audio_clip = audio_clip.subclipped(start_time)
+        elif end_time is not None:
+            audio_clip = audio_clip.subclipped(0, end_time)
+
+        # Export the audio as MP3
+        audio_clip.write_audiofile(output_file, codec="mp3")
+        audio_clip.close()
+    except Exception as e:
+        logger.error(f"Error extracting audio: {str(e)}")
+        raise
+
+
+@retry_audio_transcription()
+async def _transcribe_segment(audio_file, model):
+    """Internal function to transcribe a single segment - wrapped with retry logic."""
+    return (await model.atranscribe(audio_file)).text
+
+
+async def transcribe_audio_segment(audio_file, model, semaphore):
+    """
+    Transcribe a single audio segment asynchronously with concurrency control and retry logic.
+
+    This function uses a semaphore to limit the number of concurrent transcriptions,
+    preventing API rate limits while allowing parallel processing for improved performance.
+    Includes retry logic for transient API failures.
+
+    Args:
+        audio_file (str): Path to the audio file segment to transcribe
+        model: Speech-to-text model instance with atranscribe() method
+        semaphore (asyncio.Semaphore): Semaphore to control concurrency
+
+    Returns:
+        str: Transcribed text from the audio segment
+
+    Note:
+        Multiple instances of this function can run concurrently, but the semaphore
+        ensures that no more than N transcriptions happen simultaneously, where N
+        is configured via get_audio_concurrency() (default: 3, range: 1-10).
+    """
+    async with semaphore:
+        return await _transcribe_segment(audio_file, model)
+
+
+async def extract_audio_data(data: ProcessSourceState):
+    """
+    Extract and transcribe audio from a file with automatic segmentation and parallel processing.
+
+    This function handles the complete audio processing pipeline:
+    1. Splits long audio files (>10 minutes) into segments
+    2. Transcribes segments in parallel using configurable concurrency
+    3. Joins transcriptions in correct order
+
+    For files longer than 10 minutes, segments are processed concurrently with a
+    configurable concurrency limit to balance performance and API rate limits.
+
+    Args:
+        data (ProcessSourceState): State object containing file_path to audio/video file
+
+    Returns:
+        dict: Dictionary containing:
+            - metadata: Information about processed segments count
+            - content: Complete transcribed text
+
+    Configuration:
+        Concurrency is controlled via:
+        - Environment variable: CCORE_AUDIO_CONCURRENCY (1-10, default: 3)
+        - YAML config: extraction.audio.concurrency
+
+    Raises:
+        Exception: If audio extraction or transcription fails
+    """
+    input_audio_path = data.file_path
+    audio = None
+
+    try:
+        # Use TemporaryDirectory context manager for automatic cleanup
+        with tempfile.TemporaryDirectory() as temp_dir:
+            output_prefix = os.path.splitext(os.path.basename(input_audio_path))[0]
+            output_dir = temp_dir
+
+            # Split audio into segments if longer than 10 minutes
+            audio = AudioFileClip(input_audio_path)
+            duration_s = audio.duration
+            segment_length_s = 10 * 60  # 10 minutes in seconds
+            output_files = []
+
+            if duration_s > segment_length_s:
+                logger.info(
+                    f"Audio is longer than 10 minutes ({duration_s}s), splitting into {math.ceil(duration_s / segment_length_s)} segments"
+                )
+                for i in range(math.ceil(duration_s / segment_length_s)):
+                    start_time = i * segment_length_s
+                    end_time = min((i + 1) * segment_length_s, audio.duration)
+
+                    # Extract segment
+                    output_filename = f"{output_prefix}_{str(i + 1).zfill(3)}.mp3"
+                    output_path = os.path.join(output_dir, output_filename)
+
+                    extract_audio(input_audio_path, output_path, start_time, end_time)
+
+                    output_files.append(output_path)
+            else:
+                output_files = [input_audio_path]
+
+            # Close audio clip after determining segments
+            if audio:
+                audio.close()
+                audio = None
+
+            # Transcribe audio files in parallel with concurrency limit
+            from content_core.config import CONFIG
+            from content_core.models import ModelFactory
+            from esperanto import AIFactory
+
+            # Determine which model to use based on state parameters
+            if data.audio_provider and data.audio_model:
+                # Custom model provided - create new instance
+                try:
+                    logger.info(
+                        f"Using custom audio model: {data.audio_provider}/{data.audio_model}"
+                    )
+                    # Get timeout from config (same as default model) or use fallback
+                    timeout = CONFIG.get('speech_to_text', {}).get('timeout', 3600)
+                    stt_config = {'timeout': timeout} if timeout else {}
+                    # Add proxy to config if configured
+                    current_proxy = get_proxy(data.proxy)
+                    if current_proxy:
+                        stt_config['proxy'] = current_proxy
+                    speech_to_text_model = AIFactory.create_speech_to_text(
+                        data.audio_provider, data.audio_model, stt_config
+                    )
+                except Exception as e:
+                    logger.error(
+                        f"Failed to create custom audio model '{data.audio_provider}/{data.audio_model}': {e}. "
+                        f"Check that the provider and model are supported by Esperanto. "
+                        f"Falling back to default model."
+                    )
+                    speech_to_text_model = ModelFactory.get_model("speech_to_text")
+            elif data.audio_provider or data.audio_model:
+                # Only one parameter provided - log warning and use default
+                missing = "audio_model" if data.audio_provider else "audio_provider"
+                provided = "audio_provider" if data.audio_provider else "audio_model"
+                logger.warning(
+                    f"{provided} provided without {missing}. "
+                    f"Both audio_provider and audio_model must be specified together. "
+                    f"Falling back to default model."
+                )
+                speech_to_text_model = ModelFactory.get_model("speech_to_text")
+            else:
+                # No custom parameters - use default (backward compatible)
+                speech_to_text_model = ModelFactory.get_model("speech_to_text")
+
+            concurrency = get_audio_concurrency()
+            semaphore = asyncio.Semaphore(concurrency)
+
+            logger.debug(
+                f"Transcribing {len(output_files)} audio segments with concurrency limit of {concurrency}"
+            )
+
+            # Create tasks for parallel transcription
+            transcription_tasks = [
+                transcribe_audio_segment(audio_file, speech_to_text_model, semaphore)
+                for audio_file in output_files
+            ]
+
+            # Execute all transcriptions concurrently (limited by semaphore)
+            transcriptions = await asyncio.gather(*transcription_tasks)
+
+            return {
+                "metadata": {"segments_count": len(output_files)},
+                "content": " ".join(transcriptions),
+            }
+    except Exception as e:
+        logger.error(f"Error processing audio: {str(e)}")
+        logger.error(traceback.format_exc())
+        raise
+    finally:
+        # Ensure audio clip is closed even if an error occurs
+        if audio:
+            try:
+                audio.close()
+            except Exception:
+                pass

content_core/processors/docling.py

@@ -0,0 +1,79 @@
+"""
+Docling-based document extraction processor.
+"""
+
+from content_core.common.state import ProcessSourceState
+from content_core.config import CONFIG
+
+DOCLING_AVAILABLE = False
+try:
+    from docling.document_converter import DocumentConverter
+    DOCLING_AVAILABLE = True
+except ImportError:
+
+    class DocumentConverter:
+        """Stub when docling is not installed."""
+
+        def __init__(self):
+            raise ImportError(
+                "Docling not installed. Install with: pip install content-core[docling] "
+                "or use CCORE_DOCUMENT_ENGINE=simple to skip docling."
+            )
+
+        def convert(self, source: str):
+            raise ImportError(
+                "Docling not installed. Install with: pip install content-core[docling] "
+                "or use CCORE_DOCUMENT_ENGINE=simple to skip docling."
+            )
+
+# Supported MIME types for Docling extraction
+DOCLING_SUPPORTED = {
+    "application/pdf",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    "text/markdown",
+    # "text/plain", #docling currently not supporting txt
+    "text/x-markdown",
+    "text/csv",
+    "text/html",
+    "image/png",
+    "image/jpeg",
+    "image/tiff",
+    "image/bmp",
+}
+
+
+async def extract_with_docling(state: ProcessSourceState) -> ProcessSourceState:
+    """
+    Use Docling to parse files, URLs, or content into the desired format.
+    """
+    # Initialize Docling converter
+    converter = DocumentConverter()
+
+    # Determine source: file path, URL, or direct content
+    source = state.file_path or state.url or state.content
+    if not source:
+        raise ValueError("No input provided for Docling extraction.")
+
+    # Convert document
+    result = converter.convert(source)
+    doc = result.document
+
+    # Determine output format (per execution override, metadata, then config)
+    cfg_fmt = (
+        CONFIG.get("extraction", {}).get("docling", {}).get("output_format", "markdown")
+    )
+    fmt = state.output_format or state.metadata.get("docling_format") or cfg_fmt
+    # Record the format used
+    state.metadata["docling_format"] = fmt
+    if fmt == "html":
+        output = doc.export_to_html()
+    elif fmt == "json":
+        output = doc.export_to_json()
+    else:
+        output = doc.export_to_markdown()
+
+    # Update state
+    state.content = output
+    return state