content-core 0.7.2__py3-none-any.whl → 0.8.1__py3-none-any.whl

content_core/common/state.py

@@ -2,6 +2,9 @@ from typing import Optional
 
 from pydantic import BaseModel, Field
 
+from content_core.common.types import Engine
+from content_core.common.types import Engine
+
 
 class ProcessSourceState(BaseModel):
     file_path: Optional[str] = ""
@@ -13,8 +16,9 @@ class ProcessSourceState(BaseModel):
     identified_provider: Optional[str] = ""
     metadata: Optional[dict] = Field(default_factory=lambda: {})
     content: Optional[str] = ""
-    engine: Optional[str] = Field(
-        default=None, description="Override extraction engine: 'legacy' or 'docling'"
+    engine: Optional[Engine] = Field(
+        default=None,
+        description="Override extraction engine: 'auto', 'simple', 'legacy', 'firecrawl', 'jina', or 'docling'",
     )
     output_format: Optional[str] = Field(
         default=None,

content_core/common/types.py

@@ -0,0 +1,21 @@
+from typing import Literal
+import warnings
+
+Engine = Literal[
+    "auto",
+    "simple",
+    "legacy",
+    "firecrawl",
+    "jina",
+    "docling",
+]
+
+DEPRECATED_ENGINES = {"legacy": "simple"}
+
+def warn_if_deprecated_engine(engine: str):
+    if engine in DEPRECATED_ENGINES:
+        warnings.warn(
+            f"Engine '{engine}' is deprecated and will be removed in a future release. Use '{DEPRECATED_ENGINES[engine]}' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )

content_core/content/extraction/graph.py

@@ -2,6 +2,7 @@ import os
 import tempfile
 from typing import Any, Dict, Optional
 from urllib.parse import urlparse
+from content_core.common.types import warn_if_deprecated_engine
 
 import aiohttp
 import magic
@@ -114,14 +115,28 @@ async def download_remote_file(state: ProcessSourceState) -> Dict[str, Any]:
     return {"file_path": tmp, "identified_type": mime}
 
 
+
 async def file_type_router_docling(state: ProcessSourceState) -> str:
     """
-    Route to Docling if enabled and supported; otherwise use legacy file type edge.
+    Route to Docling if enabled and supported; otherwise use simple file type edge.
+    Supports 'auto', 'docling', 'simple', and 'legacy' (deprecated, alias for simple).
+    'auto' tries simple first, then falls back to docling if simple fails.
     """
-    # allow per-execution override of engine via state.engine
-    engine = state.engine or CONFIG.get("extraction", {}).get("engine", "legacy")
+    engine = state.engine or CONFIG.get("extraction", {}).get("engine", "auto")
+    warn_if_deprecated_engine(engine)
+    if engine == "auto":
+        # Try docling first; if it fails or is not supported, fallback to simple
+        if state.identified_type in DOCLING_SUPPORTED:
+            try:
+                return "extract_docling"
+            except Exception as e:
+                logger.warning(f"Docling extraction failed in 'auto' mode, falling back to simple: {e}")
+        # Fallback to simple
+        return await file_type_edge(state)
+
     if engine == "docling" and state.identified_type in DOCLING_SUPPORTED:
         return "extract_docling"
+    # For 'simple' and 'legacy', use the default file type edge
     return await file_type_edge(state)
 
 

content_core/processors/audio.py

@@ -1,9 +1,10 @@
 import asyncio
+import math
 import os
 import tempfile
-import math
 import traceback
 from functools import partial
+
 from moviepy import AudioFileClip
 
 from content_core.common import ProcessSourceState
@@ -64,7 +65,9 @@ async def split_audio(input_file, segment_length_minutes=15, output_prefix=None)
     )
 
 
-def extract_audio(input_file: str, output_file: str, start_time: float = None, end_time: float = None) -> None:
+def extract_audio(
+    input_file: str, output_file: str, start_time: float = None, end_time: float = None
+) -> None:
     """
     Extract audio from a video or audio file and save it as an MP3 file.
     If start_time and end_time are provided, only that segment of audio is extracted.
@@ -78,17 +81,17 @@ def extract_audio(input_file: str, output_file: str, start_time: float = None, e
     try:
         # Load the file as an AudioFileClip
         audio_clip = AudioFileClip(input_file)
-        
-        # If start_time and end_time are provided, trim the audio
+
+        # If start_time and/or end_time are provided, trim the audio using subclipped
         if start_time is not None and end_time is not None:
-            audio_clip = audio_clip.cutout(0, start_time).cutout(end_time - start_time, audio_clip.duration)
+            audio_clip = audio_clip.subclipped(start_time, end_time)
         elif start_time is not None:
-            audio_clip = audio_clip.cutout(0, start_time)
+            audio_clip = audio_clip.subclipped(start_time)
         elif end_time is not None:
-            audio_clip = audio_clip.cutout(end_time, audio_clip.duration)
+            audio_clip = audio_clip.subclipped(0, end_time)
 
         # Export the audio as MP3
-        audio_clip.write_audiofile(output_file, codec='mp3')
+        audio_clip.write_audiofile(output_file, codec="mp3")
         audio_clip.close()
     except Exception as e:
         logger.error(f"Error extracting audio: {str(e)}")
@@ -117,7 +120,9 @@ async def extract_audio_data(data: ProcessSourceState):
         output_files = []
 
         if duration_s > segment_length_s:
-            logger.info(f"Audio is longer than 10 minutes ({duration_s}s), splitting into {math.ceil(duration_s / segment_length_s)} segments")
+            logger.info(
+                f"Audio is longer than 10 minutes ({duration_s}s), splitting into {math.ceil(duration_s / segment_length_s)} segments"
+            )
             for i in range(math.ceil(duration_s / segment_length_s)):
                 start_time = i * segment_length_s
                 end_time = min((i + 1) * segment_length_s, audio.duration)
@@ -134,15 +139,18 @@ async def extract_audio_data(data: ProcessSourceState):
 
         # Transcribe audio files
         from content_core.models import ModelFactory
+
         speech_to_text_model = ModelFactory.get_model("speech_to_text")
         transcriptions = []
         for audio_file in output_files:
-            transcription = await transcribe_audio_segment(audio_file, speech_to_text_model)
+            transcription = await transcribe_audio_segment(
+                audio_file, speech_to_text_model
+            )
             transcriptions.append(transcription)
 
         return {
             "metadata": {"audio_files": output_files},
-            "content": " ".join(transcriptions)
+            "content": " ".join(transcriptions),
         }
     except Exception as e:
         logger.error(f"Error processing audio: {str(e)}")

content_core/processors/url.py

@@ -1,20 +1,21 @@
-import re
-from urllib.parse import urlparse
+import os
 from io import BytesIO
+from urllib.parse import urlparse
 
 import aiohttp
 import docx
-from bs4 import BeautifulSoup, Comment
+from bs4 import BeautifulSoup
+from readability import Document
 
 from content_core.common import ProcessSourceState
+from content_core.common.types import warn_if_deprecated_engine
 from content_core.logging import logger
 from content_core.processors.pdf import SUPPORTED_FITZ_TYPES
 
-# future: better extraction methods
-# https://github.com/buriy/python-readability
-# also try readability: from readability import Document
+DOCX_MIME_TYPE = (
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+)
 
-DOCX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
 
 async def _extract_docx_content(docx_bytes: bytes, url: str):
     """
@@ -25,21 +26,25 @@ async def _extract_docx_content(docx_bytes: bytes, url: str):
         doc = docx.Document(BytesIO(docx_bytes))
         content_parts = [p.text for p in doc.paragraphs if p.text]
         full_content = "\n\n".join(content_parts)
-        
+
         # Try to get a title from document properties or first heading
         title = doc.core_properties.title
         if not title and doc.paragraphs:
             # Look for a potential title in the first few paragraphs (e.g., if styled as heading)
-            for p in doc.paragraphs[:5]: # Check first 5 paragraphs
-                if p.style.name.startswith('Heading'):
+            for p in doc.paragraphs[:5]:  # Check first 5 paragraphs
+                if p.style.name.startswith("Heading"):
                     title = p.text
                     break
-            if not title: # Fallback to first line if no heading found
-                 title = doc.paragraphs[0].text.strip() if doc.paragraphs[0].text.strip() else None
+            if not title:  # Fallback to first line if no heading found
+                title = (
+                    doc.paragraphs[0].text.strip()
+                    if doc.paragraphs[0].text.strip()
+                    else None
+                )
 
         # If no title found, use filename from URL
         if not title:
-            title = urlparse(url).path.split('/')[-1]
+            title = urlparse(url).path.split("/")[-1]
 
         logger.info(f"Successfully extracted content from DOCX: {url}, Title: {title}")
         return {
@@ -58,6 +63,7 @@ async def _extract_docx_content(docx_bytes: bytes, url: str):
             "url": url,
         }
 
+
 async def url_provider(state: ProcessSourceState):
     """
     Identify the provider
@@ -71,7 +77,9 @@ async def url_provider(state: ProcessSourceState):
             # remote URL: check content-type to catch PDFs
             try:
                 async with aiohttp.ClientSession() as session:
-                    async with session.head(url, timeout=10, allow_redirects=True) as resp:
+                    async with session.head(
+                        url, timeout=10, allow_redirects=True
+                    ) as resp:
                         mime = resp.headers.get("content-type", "").split(";", 1)[0]
             except Exception as e:
                 logger.debug(f"HEAD check failed for {url}: {e}")
@@ -83,142 +91,82 @@ async def url_provider(state: ProcessSourceState):
     return return_dict
 
 
-async def extract_url_bs4(url: str):
-    """
-    Get the title and content of a URL using bs4
+async def extract_url_bs4(url: str) -> dict:
     """
-    try:
-        headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-        }
+    Get the title and content of a URL using readability with a fallback to BeautifulSoup.
 
-        # If URL is actually HTML content
-        if url.startswith("<!DOCTYPE html>") or url.startswith("<html"):
-            html_content = url
-        else:
-            async with aiohttp.ClientSession() as session:
-                async with session.get(url, headers=headers, timeout=10) as response:
-                    response.raise_for_status()
-                    # Check content type for DOCX
-                    if response.content_type == DOCX_MIME_TYPE:
-                        logger.debug(f"Detected DOCX content type for {url}")
-                        docx_bytes = await response.read()
-                        return await _extract_docx_content(docx_bytes, url)
-                    
-                    # If not DOCX, proceed as HTML
-                    html_content = await response.text()
-
-        soup = BeautifulSoup(html_content, "html.parser")
-
-        # Remove unwanted elements
-        for element in soup.find_all(
-            ["script", "style", "nav", "footer", "iframe", "noscript", "ad"]
-        ):
-            element.decompose()
-
-        # Remove comments
-        for comment in soup.find_all(string=lambda text: isinstance(text, Comment)):
-            comment.extract()
-
-        # Get title
-        title = None
-        title_tags = [
-            soup.find("meta", property="og:title"),
-            soup.find("meta", property="twitter:title"),
-            soup.find("title"),
-            soup.find("h1"),
-        ]
-
-        for tag in title_tags:
-            if tag:
-                if tag.string:
-                    title = tag.string
-                elif tag.get("content"):
-                    title = tag.get("content")
-                break
-
-        # Clean up title
-        if title:
-            title = " ".join(title.split())
-            title = re.sub(r"\s*\|.*$", "", title)
-            title = re.sub(r"\s*-.*$", "", title)
-
-        # Get content
-        content = []
-
-        # Look for main article content
-        main_content = None
-        content_tags = [
-            soup.find("article"),
-            soup.find("main"),
-            soup.find(class_=re.compile(r"article|post|content|entry|document")),
-            soup.find(id=re.compile(r"article|post|content|entry|main")),
-        ]
-
-        for tag in content_tags:
-            if tag:
-                main_content = tag
-                break
-
-        if not main_content:
-            main_content = soup
-
-        # Process content
-        for element in main_content.find_all(
-            ["p", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "div"]
-        ):
-            # Handle code blocks
-            if element.name == "pre" or "highlight" in element.get("class", []):
-                code_text = element.get_text().strip()
-                if code_text:
-                    content.append("\n```\n" + code_text + "\n```\n")
-                continue
-
-            # Handle regular text
-            text = element.get_text().strip()
-            if text:
-                # Skip if text matches common patterns for navigation/footer
-                if re.search(
-                    r"copyright|all rights reserved|privacy policy|terms of use",
-                    text.lower(),
-                ):
-                    continue
-
-                content.append(text)
-
-        # Join content with proper spacing
-        final_content = "\n\n".join(content)
-
-        # Clean up content
-        final_content = re.sub(
-            r"\n\s*\n\s*\n", "\n\n", final_content
-        )  # Remove extra newlines
-        final_content = re.sub(r" +", " ", final_content)  # Normalize whitespace
-        final_content = final_content.strip()
+    Args:
+        url (str): The URL of the webpage to extract content from.
 
-        return {
-            "title": title,
-            "content": final_content,
-            "domain": urlparse(url).netloc
-            if not url.startswith("<!DOCTYPE html>")
-            else None,
-            "url": url if not url.startswith("<!DOCTYPE html>") else None,
-        }
+    Returns:
+        dict: A dictionary containing the 'title' and 'content' of the webpage.
+    """
+    async with aiohttp.ClientSession() as session:
+        try:
+            # Fetch the webpage content
+            async with session.get(url, timeout=10) as response:
+                if response.status != 200:
+                    raise Exception(f"HTTP error: {response.status}")
+                html = await response.text()
+
+            # Try extracting with readability
+            try:
+                doc = Document(html)
+                title = doc.title() or "No title found"
+                # Extract content as plain text by parsing the cleaned HTML
+                soup = BeautifulSoup(doc.summary(), "lxml")
+                content = soup.get_text(separator=" ", strip=True)
+                if not content.strip():
+                    raise ValueError("No content extracted by readability")
+            except Exception as e:
+                print(f"Readability failed: {e}")
+                # Fallback to BeautifulSoup
+                soup = BeautifulSoup(html, "lxml")
+                # Extract title
+                title_tag = (
+                    soup.find("title")
+                    or soup.find("h1")
+                    or soup.find("meta", property="og:title")
+                )
+                title = (
+                    title_tag.get_text(strip=True) if title_tag else "No title found"
+                )
+                # Extract content from common content tags
+                content_tags = soup.select(
+                    'article, .content, .post, main, [role="main"], div[class*="content"], div[class*="article"]'
+                )
+                content = (
+                    " ".join(
+                        tag.get_text(separator=" ", strip=True) for tag in content_tags
+                    )
+                    if content_tags
+                    else soup.get_text(separator=" ", strip=True)
+                )
+                content = content.strip() or "No content found"
 
-    except aiohttp.ClientError as e:
-        logger.error(f"Failed to fetch URL {url}: {e}")
-        return None
-    except Exception as e:
-        logger.error(f"Failed to process content: {e}")
-        return None
+            return {
+                "title": title,
+                "content": content,
+            }
+
+        except Exception as e:
+            print(f"Error processing URL {url}: {e}")
+            return {
+                "title": "Error",
+                "content": f"Failed to extract content: {str(e)}",
+            }
 
 
 async def extract_url_jina(url: str):
     """
-    Get the content of a URL using Jina
+    Get the content of a URL using Jina. Uses Bearer token if JINA_API_KEY is set.
     """
+    headers = {}
+    api_key = os.environ.get("JINA_API_KEY")
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
     async with aiohttp.ClientSession() as session:
-        async with session.get(f"https://r.jina.ai/{url}") as response:
+        async with session.get(f"https://r.jina.ai/{url}", headers=headers) as response:
             text = await response.text()
             if text.startswith("Title:") and "\n" in text:
                 title_end = text.index("\n")
@@ -235,17 +183,65 @@ async def extract_url_jina(url: str):
                 return {"content": text}
 
 
+async def extract_url_firecrawl(url: str):
+    """
+    Get the content of a URL using Firecrawl.
+    Returns {"title": ..., "content": ...} or None on failure.
+    """
+    try:
+        from firecrawl import AsyncFirecrawlApp
+
+        app = AsyncFirecrawlApp(api_key=os.environ.get("FIRECRAWL_API_KEY"))
+        scrape_result = await app.scrape_url(url, formats=["markdown", "html"])
+        return {
+            "title": scrape_result.metadata["title"] or scrape_result.title,
+            "content": scrape_result.markdown,
+        }
+
+    except Exception as e:
+        logger.error(f"Firecrawl extraction error for URL: {url}: {e}")
+        return None
+
+
 async def extract_url(state: ProcessSourceState):
+    """
+    Extract content from a URL using the engine specified in the state.
+    Supported engines: 'auto', 'simple', 'legacy' (deprecated), 'firecrawl', 'jina'.
+    """
     assert state.url, "No URL provided"
     url = state.url
+    engine = state.engine or "auto"
+    warn_if_deprecated_engine(engine)
     try:
-        result = await extract_url_bs4(url)
-        if not result or not result.get("content"):
-            logger.debug(
-                f"BS4 extraction failed for url {url}, falling back to Jina extractor"
-            )
-            result = await extract_url_jina(url)
-        return result
+        if engine == "auto":
+            if os.environ.get("FIRECRAWL_API_KEY"):
+                logger.debug(
+                    "Engine 'auto' selected: using Firecrawl (FIRECRAWL_API_KEY detected)"
+                )
+                return await extract_url_firecrawl(url)
+            else:
+                try:
+                    logger.debug("Trying to use Jina to extract URL")
+                    return await extract_url_jina(url)
+                except Exception as e:
+                    logger.error(f"Jina extraction error for URL: {url}: {e}")
+                    logger.debug("Falling back to BeautifulSoup")
+                    return await extract_url_bs4(url)
+        elif engine == "simple" or engine == "legacy":
+            # 'legacy' is deprecated alias for 'simple'
+            return await extract_url_bs4(url)
+        elif engine == "firecrawl":
+            return await extract_url_firecrawl(url)
+        elif engine == "jina":
+            return await extract_url_jina(url)
+        elif engine == "docling":
+            from content_core.processors.docling import extract_with_docling
+
+            state.url = url
+            result_state = await extract_with_docling(state)
+            return {"title": None, "content": result_state.content}
+        else:
+            raise ValueError(f"Unknown engine: {engine}")
     except Exception as e:
         logger.error(f"URL extraction failed for URL: {url}")
         logger.exception(e)

{content_core-0.7.2.dist-info → content_core-0.8.1.dist-info}/METADATA

@@ -1,15 +1,18 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 0.7.2
+Version: 0.8.1
 Summary: Extract what matters from any media source
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: ai-prompter>=0.2.3
 Requires-Dist: aiohttp>=3.11
+Requires-Dist: asciidoc>=10.2.1
 Requires-Dist: bs4>=0.0.2
 Requires-Dist: dicttoxml>=1.7.16
+Requires-Dist: docling>=2.34.0
 Requires-Dist: esperanto[openai]>=1.2.0
+Requires-Dist: firecrawl-py>=2.7.0
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: langdetect>=1.0.9
 Requires-Dist: langgraph>=0.3.29
@@ -17,18 +20,15 @@ Requires-Dist: loguru>=0.7.3
 Requires-Dist: moviepy>=2.1.2
 Requires-Dist: openpyxl>=3.1.5
 Requires-Dist: pandas>=2.2.3
+Requires-Dist: pillow>=10.4.0
 Requires-Dist: pymupdf>=1.25.5
 Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: python-magic>=0.4.27
 Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: readability-lxml>=0.8.4.1
 Requires-Dist: validators>=0.34.0
 Requires-Dist: youtube-transcript-api>=1.0.3
-Provides-Extra: docling
-Requires-Dist: asciidoc; extra == 'docling'
-Requires-Dist: docling; extra == 'docling'
-Requires-Dist: pandas; extra == 'docling'
-Requires-Dist: pillow; extra == 'docling'
 Description-Content-Type: text/markdown
 
 # Content Core
@@ -39,6 +39,8 @@ Description-Content-Type: text/markdown
 
 ## Overview
 
+> **Note:** As of v0.8, the default extraction engine is `'auto'`. Content Core will automatically select the best extraction method based on your environment and available API keys, with a smart fallback order for both URLs and files. For files/documents, `'auto'` now tries Docling first, then falls back to simple extraction. You can override the engine if needed, but `'auto'` is recommended for most users.
+
 The primary goal of Content Core is to simplify the process of ingesting content from diverse origins. Whether you have raw text, a URL pointing to an article, or a local file like a video or markdown document, Content Core aims to extract the meaningful content for further use.
 
 ## Key Features
@@ -48,6 +50,10 @@ The primary goal of Content Core is to simplify the process of ingesting content
     *   Web URLs (using robust extraction methods).
     *   Local files (including automatic transcription for video/audio files and parsing for text-based formats).
 *   **Intelligent Processing:** Applies appropriate extraction techniques based on the source type. See the [Processors Documentation](./docs/processors.md) for detailed information on how different content types are handled.
+*   **Smart Engine Selection:** By default, Content Core uses the `'auto'` engine, which:
+    * For URLs: Uses Firecrawl if `FIRECRAWL_API_KEY` is set, else tries Jina. Jina might fail because of rate limits, which can be fixed by adding `JINA_API_KEY`. If Jina failes, BeautifulSoup is used as a fallback.
+    * For files: Tries Docling extraction first (for robust document parsing), then falls back to simple extraction if needed.
+    * You can override this by specifying an engine, but `'auto'` is recommended for most users.
 *   **Content Cleaning (Optional):** Likely integrates with LLMs (via `prompter.py` and Jinja templates) to refine and clean the extracted content.
 *   **Asynchronous:** Built with `asyncio` for efficient I/O operations.
 
@@ -60,8 +66,6 @@ Install Content Core using `pip`:
 ```bash
 # Install the package (without Docling)
 pip install content-core
-# Install with Docling support
-pip install content-core[docling]
 ```
 
 Alternatively, if you’re developing locally:
@@ -218,15 +222,15 @@ async def main():
     text_data = await extract_content({"content": "This is my sample text content."})
     print(text_data)
 
-    # Extract from a URL
+    # Extract from a URL (uses 'auto' engine by default)
     url_data = await extract_content({"url": "https://www.example.com"})
     print(url_data)
 
-    # Extract from a local video file (gets transcript)
+    # Extract from a local video file (gets transcript, engine='auto' by default)
     video_data = await extract_content({"file_path": "path/to/your/video.mp4"})
     print(video_data)
 
-    # Extract from a local markdown file
+    # Extract from a local markdown file (engine='auto' by default)
     md_data = await extract_content({"file_path": "path/to/your/document.md"})
     print(md_data)
 
@@ -248,15 +252,11 @@ if __name__ == "__main__":
 
 Content Core supports an optional Docling-based extraction engine for rich document formats (PDF, DOCX, PPTX, XLSX, Markdown, AsciiDoc, HTML, CSV, Images).
 
-### Installation
-
-```bash
-# Install with Docling support
-pip install content-core[docling]
-```
 
 ### Enabling Docling
 
+Docling is not the default engine when parsing documents. If you don't want to use it, you need to set engine to "simple". 
+
 #### Via configuration file
 
 In your `cc_config.yaml` or custom config, set:

{content_core-0.7.2.dist-info → content_core-0.8.1.dist-info}/RECORD

@@ -8,30 +8,31 @@ content_core/py.typed,sha256=pLuU3XTTeVpXo4UomOjcvAIQqOrzIotlWlJ3KFo2lxQ,154
 content_core/templated_message.py,sha256=KbI2rcvgGM5oRIcsG68zAZfgNsC97fR16D61683ZSnY,1617
 content_core/common/__init__.py,sha256=SjDp-0QRjX9PMubyTjv77_GrUqm6eC4gBuXr593JVK4,525
 content_core/common/exceptions.py,sha256=NpYedVbckIq4kP2wek7bicMVgGGn0fkhCvid5cIxfy4,1304
-content_core/common/state.py,sha256=2yhb87ZXZIgxAzVH6KKz9_glUzI8rSJ4Iq_0s4sKb7U,1284
+content_core/common/state.py,sha256=pO8Oq71KxznlZ4K5qUVfyLrNsZWd2yMO9bXKmrTIXQo,1427
+content_core/common/types.py,sha256=FpIzYadBvafGI4e1EuwGjjiPuawL1HitxsQOciNjTZo,497
 content_core/common/utils.py,sha256=0o4jovPEw_6wu7EcPPbDNZskbhhfLUBJBvRmp0Yc4R4,1182
 content_core/content/__init__.py,sha256=ymocLXXwWnnhQFHCB3jXanNvJ2m27TVs1yO8EhCrefU,171
 content_core/content/cleanup/__init__.py,sha256=wymD24WLDDdsZrv-5WhparSiHBK9SJCcqBHmokuZqk4,121
 content_core/content/cleanup/core.py,sha256=AXUGUWxGob8si5uKRnDrreOcHV_gbGJr4YnRsNm2GX0,531
 content_core/content/extraction/__init__.py,sha256=TaYw6CAcG62GZfsJxeZ6VJDLP85BU2a7_G271v6WWPk,446
-content_core/content/extraction/graph.py,sha256=IKu-bV3YG2MigHnYixYYhtrQ-4qgGpETerXBEFn73zU,6304
+content_core/content/extraction/graph.py,sha256=51B_j_hi7SsKh7kKNLFsMmxyR2HVS-mOYfKvDFyuYfw,7001
 content_core/content/summary/__init__.py,sha256=ReKCZWKfDtqlInKeh87Y1DEfiNzVWabGybEz3hS2FrI,114
 content_core/content/summary/core.py,sha256=LejUbPxnRD0sbO6MupiIb-IHLxEUGU5beBZwmIiBncc,542
 content_core/notebooks/run.ipynb,sha256=WPBNcQUNXR5MldNMghVcU4vE4ibrVmlANa80baQn8TA,371078
-content_core/processors/audio.py,sha256=KnwxK85X9qRyVziMhFd103kfHkE8qGB1D4yW5lYO90E,5701
+content_core/processors/audio.py,sha256=Mie20g_2Akhw6BHBVo3sHMpDRYUkqBI72lEDakscx3s,5729
 content_core/processors/docling.py,sha256=wQ8ThAcyrCy-c95QtgplQ9UZtjCZTddLD9y1_CrRtSQ,2111
 content_core/processors/office.py,sha256=DXkfmjqUhmhP6rJaO5Z5Y9sv-iK0zaPZ3waynFIPtsk,12153
 content_core/processors/pdf.py,sha256=9jf-eROAqw6yQwdlbsxPXsaJXY26hVG7nSTPH9n4afY,5301
 content_core/processors/text.py,sha256=kKHA60-NYjLmCTYUnk8TdJxQQ0Shkg-K61Ezqaelz7k,1158
-content_core/processors/url.py,sha256=vmkBVfJ1xpZQzlhRdkO64V1J9xdTBr6nrXY4M74QzEo,9094
+content_core/processors/url.py,sha256=yt-uuzS4N-RAOJ8vo5x-b4bgnrFeTV-3SDIatRTRI3g,9462
 content_core/processors/video.py,sha256=3WnZwTswvTLm8PtQhKwoqJ2BH6YZi62dMUjALwJiebo,5196
 content_core/processors/youtube.py,sha256=nM286Km7FLN0r1f-n-dRkqs6mSXxCo4YOhTeGzj7Suo,5798
 content_core/tools/__init__.py,sha256=DuJmd7fE-NpDvLP8IW1XY5MUkAQcdks52rn2jk4N8jQ,231
 content_core/tools/cleanup.py,sha256=5IdKedsFyRQMdYzgFSKtsfyxJldbroXQXHesHICNENI,523
 content_core/tools/extract.py,sha256=-r2_jsuMMXyXxGVqWhh1ilNPo_UMYAbw3Pkp1FzPy5g,577
 content_core/tools/summarize.py,sha256=DPfeglLWB08q8SvHrsKpOKZ35XjduUDs2J02ISwjdj0,596
-content_core-0.7.2.dist-info/METADATA,sha256=oWJhvImNzPAcGuAnkEKIqWozWW383pszrszyYaSLB-s,10471
-content_core-0.7.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-content_core-0.7.2.dist-info/entry_points.txt,sha256=9fGQUk6bxBVXj9PRwfWVPn54ClSEJV7J-KBLXtjOhQw,99
-content_core-0.7.2.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
-content_core-0.7.2.dist-info/RECORD,,
+content_core-0.8.1.dist-info/METADATA,sha256=ZIW6gtawFeFo2uQqWkFH2ctSYIUq5PBrke4gyHQQAWU,11439
+content_core-0.8.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+content_core-0.8.1.dist-info/entry_points.txt,sha256=9fGQUk6bxBVXj9PRwfWVPn54ClSEJV7J-KBLXtjOhQw,99
+content_core-0.8.1.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
+content_core-0.8.1.dist-info/RECORD,,