content-core 1.10.0__py3-none-any.whl

content_core/processors/youtube.py

@@ -0,0 +1,262 @@
+import re
+import ssl
+
+import aiohttp
+from bs4 import BeautifulSoup
+from youtube_transcript_api import YouTubeTranscriptApi  # type: ignore
+from youtube_transcript_api.formatters import TextFormatter  # type: ignore
+
+from content_core.common import ProcessSourceState
+from content_core.common.exceptions import NoTranscriptFound
+from content_core.common.retry import retry_youtube
+from content_core.config import CONFIG, get_proxy, _redact_proxy_url
+from content_core.logging import logger
+
+ssl._create_default_https_context = ssl._create_unverified_context
+
+
+@retry_youtube()
+async def _fetch_video_title(video_id, proxy: str | None = None):
+    """Internal function that fetches video title - wrapped with retry logic."""
+    url = f"https://www.youtube.com/watch?v={video_id}"
+    resolved_proxy = get_proxy(proxy)
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, proxy=resolved_proxy) as response:
+            html = await response.text()
+
+    # BeautifulSoup doesn't support async operations
+    soup = BeautifulSoup(html, "html.parser")
+
+    # YouTube stores title in a meta tag
+    title = soup.find("meta", property="og:title")["content"]
+    return title
+
+
+async def get_video_title(video_id, proxy: str | None = None):
+    """Get video title from YouTube, with retry logic for transient failures."""
+    try:
+        return await _fetch_video_title(video_id, proxy)
+    except Exception as e:
+        logger.error(f"Failed to get video title after retries: {e}")
+        return None
+
+
+async def _extract_youtube_id(url):
+    """
+    Extract the YouTube video ID from a given URL using regular expressions.
+
+    Args:
+    url (str): The YouTube URL from which to extract the video ID.
+
+    Returns:
+    str: The extracted YouTube video ID or None if no valid ID is found.
+    """
+    # Define a regular expression pattern to capture the YouTube video ID
+    youtube_regex = (
+        r"(?:https?://)?"  # Optional scheme
+        r"(?:www\.)?"  # Optional www.
+        r"(?:"
+        r"youtu\.be/"  # Shortened URL
+        r"|youtube\.com"  # Main URL
+        r"(?:"  # Group start
+        r"/embed/"  # Embed URL
+        r"|/v/"  # Older video URL
+        r"|/watch\?v="  # Standard watch URL
+        r"|/watch\?.+&v="  # Other watch URL
+        r")"  # Group end
+        r")"  # End main group
+        r"([\w-]{11})"  # 11 characters (YouTube video ID)
+    )
+
+    # Search the URL for the pattern
+    match = re.search(youtube_regex, url)
+
+    # Return the video ID if a match is found
+    return match.group(1) if match else None
+
+
+@retry_youtube()
+async def _fetch_best_transcript(
+    video_id, preferred_langs=["en", "es", "pt"], proxy: str | None = None
+):
+    """Internal function that fetches transcript - wrapped with retry logic."""
+    resolved_proxy = get_proxy(proxy)
+    proxies = None
+    if resolved_proxy:
+        proxies = {"http": resolved_proxy, "https": resolved_proxy}
+        logger.debug(f"YouTubeTranscriptApi using proxy: {_redact_proxy_url(resolved_proxy)}")
+
+    transcript_list = YouTubeTranscriptApi.list_transcripts(video_id, proxies=proxies)
+
+    # First try: Manual transcripts in preferred languages
+    manual_transcripts = []
+    try:
+        for transcript in transcript_list:
+            if not transcript.is_generated and not transcript.is_translatable:
+                manual_transcripts.append(transcript)
+
+        if manual_transcripts:
+            # Sort based on preferred language order
+            for lang in preferred_langs:
+                for transcript in manual_transcripts:
+                    if transcript.language_code == lang:
+                        return transcript.fetch()
+            # If no preferred language found, return first manual transcript
+            return manual_transcripts[0].fetch()
+    except NoTranscriptFound:
+        pass
+
+    # Second try: Auto-generated transcripts in preferred languages
+    generated_transcripts = []
+    try:
+        for transcript in transcript_list:
+            if transcript.is_generated and not transcript.is_translatable:
+                generated_transcripts.append(transcript)
+
+        if generated_transcripts:
+            # Sort based on preferred language order
+            for lang in preferred_langs:
+                for transcript in generated_transcripts:
+                    if transcript.language_code == lang:
+                        return transcript.fetch()
+            # If no preferred language found, return first generated transcript
+            return generated_transcripts[0].fetch()
+    except NoTranscriptFound:
+        pass
+
+    # Last try: Translated transcripts in preferred languages
+    translated_transcripts = []
+    try:
+        for transcript in transcript_list:
+            if transcript.is_translatable:
+                translated_transcripts.append(transcript)
+
+        if translated_transcripts:
+            # Sort based on preferred language order
+            for lang in preferred_langs:
+                for transcript in translated_transcripts:
+                    if transcript.language_code == lang:
+                        return transcript.fetch()
+            # If no preferred language found, return translation to first preferred language
+            translation = translated_transcripts[0].translate(preferred_langs[0])
+            return translation.fetch()
+    except NoTranscriptFound:
+        pass
+
+    raise NoTranscriptFound("No suitable transcript found for this video")
+
+
+async def get_best_transcript(
+    video_id, preferred_langs=["en", "es", "pt"], proxy: str | None = None
+):
+    """Get best available transcript with retry logic for transient failures."""
+    try:
+        return await _fetch_best_transcript(video_id, preferred_langs, proxy)
+    except Exception as e:
+        logger.error(
+            f"Failed to get transcript for video {video_id} after retries: {e}"
+        )
+        return None
+
+
+@retry_youtube()
+def _fetch_transcript_pytubefix(
+    url, languages=["en", "es", "pt"], proxy: str | None = None
+):
+    """Internal function that fetches transcript via pytubefix - wrapped with retry logic."""
+    from pytubefix import YouTube
+
+    resolved_proxy = get_proxy(proxy)
+    proxies = None
+    if resolved_proxy:
+        proxies = {"http": resolved_proxy, "https": resolved_proxy}
+        logger.debug(f"pytubefix using proxy: {_redact_proxy_url(resolved_proxy)}")
+
+    yt = YouTube(url, proxies=proxies)
+    logger.debug(f"Captions: {yt.captions}")
+
+    # Try to get captions in the preferred languages
+    if yt.captions:
+        for lang in languages:
+            if lang in yt.captions:
+                caption = yt.captions[lang]
+                break
+            elif f"a.{lang}" in yt.captions:
+                caption = yt.captions[f"a.{lang}"]
+                break
+        else:  # No preferred language found, use the first available
+            caption_key = list(yt.captions.keys())[0]
+            caption = yt.captions[caption_key.code]
+
+        srt_captions = caption.generate_srt_captions()
+        txt_captions = caption.generate_txt_captions()
+        return txt_captions, srt_captions
+
+    return None, None
+
+
+def extract_transcript_pytubefix(
+    url, languages=["en", "es", "pt"], proxy: str | None = None
+):
+    """Extract transcript via pytubefix with retry logic for transient failures."""
+    try:
+        return _fetch_transcript_pytubefix(url, languages, proxy)
+    except Exception as e:
+        logger.error(f"Failed to extract transcript via pytubefix after retries: {e}")
+        return None, None
+
+
+async def extract_youtube_transcript(state: ProcessSourceState):
+    """
+    Parse the text file and print its content.
+
+    Proxy configuration is passed through state.proxy and resolved using get_proxy().
+    """
+
+    assert state.url, "No URL provided"
+    logger.debug(f"Extracting transcript from URL: {state.url}")
+    languages = CONFIG.get("youtube_transcripts", {}).get(
+        "preferred_languages", ["en", "es", "pt"]
+    )
+    proxy = state.proxy
+
+    # quick fix since transcripts api is not working for now
+    engine = "pytubefix"
+    video_id = await _extract_youtube_id(state.url)
+
+    try:
+        title = await get_video_title(video_id, proxy)
+    except Exception as e:
+        logger.critical(f"Failed to get video title for video_id: {video_id}")
+        logger.exception(e)
+        title = ""
+
+    if engine == "pytubefix":
+        formatted_content, transcript_raw = extract_transcript_pytubefix(
+            state.url, languages, proxy
+        )
+    if engine == "transcripts-api":
+        transcript = await get_best_transcript(video_id, languages, proxy)
+
+        logger.debug(f"Found transcript: {transcript}")
+        formatter = TextFormatter()
+
+        try:
+            formatted_content = formatter.format_transcript(transcript)
+        except Exception as e:
+            logger.critical(f"Failed to format transcript for video_id: {video_id}")
+            logger.exception(e)
+            formatted_content = ""
+
+        try:
+            transcript_raw = transcript.to_raw_data()
+        except Exception as e:
+            logger.critical(f"Failed to get raw transcript for video_id: {video_id}")
+            logger.exception(e)
+            transcript_raw = ""
+
+    return {
+        "content": formatted_content,
+        "title": title,
+        "metadata": {"video_id": video_id, "transcript": transcript_raw},
+    }

content_core/py.typed

@@ -0,0 +1,2 @@
+# This file indicates that the content_core package supports type checking.
+# See PEP 561 for more information: https://www.python.org/dev/peps/pep-0561/

content_core/templated_message.py

@@ -0,0 +1,70 @@
+from typing import Dict, Optional, Union
+
+from ai_prompter import Prompter
+from esperanto import LanguageModel
+from pydantic import BaseModel, Field
+
+from content_core.common.retry import retry_llm
+from content_core.logging import logger
+from content_core.models import ModelFactory
+
+
+class TemplatedMessageInput(BaseModel):
+    system_prompt_template: Optional[str] = None
+    system_prompt_text: Optional[str] = None
+    user_prompt_template: Optional[str] = None
+    user_prompt_text: Optional[str] = None
+    data: Optional[Union[Dict, BaseModel]] = Field(default_factory=lambda: {})
+    config: Dict = Field(
+        description="The config for the LLM",
+        default={
+            "temperature": 0,
+            "top_p": 1,
+            "max_tokens": 600,
+        },
+    )
+
+
+@retry_llm()
+async def _execute_llm_call(model: LanguageModel, msgs: list) -> str:
+    """Internal function to execute LLM call - wrapped with retry logic."""
+    result = await model.achat_complete(msgs)
+    return result.content
+
+
+async def templated_message(
+    input: TemplatedMessageInput, model: Optional[LanguageModel] = None
+) -> Optional[str]:
+    """
+    Execute a templated LLM message with retry logic for transient failures.
+
+    Args:
+        input: TemplatedMessageInput with prompt templates and data
+        model: Optional LanguageModel instance (defaults to factory model)
+
+    Returns:
+        Optional[str]: LLM response content, or None if all retries exhausted
+    """
+    if not model:
+        model = ModelFactory.get_model("default_model")
+
+    msgs = []
+    if input.system_prompt_template or input.system_prompt_text:
+        system_prompt = Prompter(
+            prompt_template=input.system_prompt_template,
+            template_text=input.system_prompt_text,
+        ).render(data=input.data)
+        msgs.append({"role": "system", "content": system_prompt})
+
+    if input.user_prompt_template or input.user_prompt_text:
+        user_prompt = Prompter(
+            prompt_template=input.user_prompt_template,
+            template_text=input.user_prompt_text,
+        ).render(data=input.data)
+        msgs.append({"role": "user", "content": user_prompt})
+
+    try:
+        return await _execute_llm_call(model, msgs)
+    except Exception as e:
+        logger.error(f"LLM call failed after retries: {e}")
+        return None

content_core/tools/__init__.py

@@ -0,0 +1,9 @@
+from .cleanup import cleanup_content_tool
+from .extract import extract_content_tool
+from .summarize import summarize_content_tool
+
+__all__ = [
+    "cleanup_content_tool",
+    "extract_content_tool",
+    "summarize_content_tool",
+]

content_core/tools/cleanup.py

@@ -0,0 +1,15 @@
+from langchain_core.tools import tool
+
+from content_core.content_cleanup import cleanup_content
+from content_core.common import process_input_content
+
+
+@tool
+async def cleanup_content_tool(content: str) -> str:
+    """
+    Clean content. Rewrite paragraphs. Fix grammar and spelling.
+    Accepts direct text, URLs, or file paths. If a URL or file path is provided,
+    the content will be extracted first before cleaning.
+    """
+    content = await process_input_content(content)
+    return await cleanup_content(content)

content_core/tools/extract.py

@@ -0,0 +1,21 @@
+from typing import Dict
+
+from langchain_core.tools import tool
+
+from content_core.extraction import extract_content
+
+
+@tool
+async def extract_content_tool(file_path_or_url: str) -> Dict:
+    """
+    Extract title, content and metadata from URLs and Links.
+
+    Args:
+        file_path_or_url: URL or file path to extract content from.
+
+    Returns:
+        Dict: Extracted content and metadata.
+    """
+    if file_path_or_url.startswith("http"):
+        return await extract_content({"url": file_path_or_url})
+    return await extract_content({"file_path": file_path_or_url})

content_core/tools/summarize.py

@@ -0,0 +1,17 @@
+from typing import Optional
+
+from langchain_core.tools import tool
+
+from content_core.content_summary import summarize
+from content_core.common import process_input_content
+
+
+@tool
+async def summarize_content_tool(content: str, context: Optional[str] = None) -> str:
+    """
+    Summarize content according to instructions provided via context.
+    Accepts direct text, URLs, or file paths. If a URL or file path is provided,
+    the content will be extracted first before summarizing.
+    """
+    content = await process_input_content(content)
+    return await summarize(content, context or "")