content-core 1.0.2__py3-none-any.whl → 1.0.4__py3-none-any.whl

content_core/content/summary/core.py

@@ -8,7 +8,7 @@ async def summarize(content: str, context: str) -> str:
     templated_message_fn = partial(templated_message, model=ModelFactory.get_model('summary_model'))
     response = await templated_message_fn(
         TemplatedMessageInput(
-            user_prompt_template="content/summarize",
+            user_prompt_template="prompts/content/summarize",
             data={"content": content, "context": context},
         )
     )

content_core/processors/youtube.py

@@ -1,16 +1,14 @@
-import asyncio
 import re
 import ssl
 
 import aiohttp
 from bs4 import BeautifulSoup
-from youtube_transcript_api import YouTubeTranscriptApi  # type: ignore
-from youtube_transcript_api.formatters import TextFormatter  # type: ignore
-
 from content_core.common import ProcessSourceState
 from content_core.common.exceptions import NoTranscriptFound
 from content_core.config import CONFIG
 from content_core.logging import logger
+from youtube_transcript_api import YouTubeTranscriptApi  # type: ignore
+from youtube_transcript_api.formatters import TextFormatter  # type: ignore
 
 ssl._create_default_https_context = ssl._create_unverified_context
 
@@ -69,86 +67,103 @@ async def _extract_youtube_id(url):
 
 
 async def get_best_transcript(video_id, preferred_langs=["en", "es", "pt"]):
-    max_attempts = 5
-    for attempt in range(max_attempts):
+    try:
+        transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+
+        # First try: Manual transcripts in preferred languages
+        manual_transcripts = []
         try:
-            transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
-
-            # First try: Manual transcripts in preferred languages
-            manual_transcripts = []
-            try:
-                for transcript in transcript_list:
-                    if not transcript.is_generated and not transcript.is_translatable:
-                        manual_transcripts.append(transcript)
-
-                if manual_transcripts:
-                    # Sort based on preferred language order
-                    for lang in preferred_langs:
-                        for transcript in manual_transcripts:
-                            if transcript.language_code == lang:
-                                return transcript.fetch()
-                    # If no preferred language found, return first manual transcript
-                    return manual_transcripts[0].fetch()
-            except NoTranscriptFound:
-                pass
-
-            # Second try: Auto-generated transcripts in preferred languages
-            generated_transcripts = []
-            try:
-                for transcript in transcript_list:
-                    if transcript.is_generated and not transcript.is_translatable:
-                        generated_transcripts.append(transcript)
-
-                if generated_transcripts:
-                    # Sort based on preferred language order
-                    for lang in preferred_langs:
-                        for transcript in generated_transcripts:
-                            if transcript.language_code == lang:
-                                return transcript.fetch()
-                    # If no preferred language found, return first generated transcript
-                    return generated_transcripts[0].fetch()
-            except NoTranscriptFound:
-                pass
-
-            # Last try: Translated transcripts in preferred languages
-            translated_transcripts = []
-            try:
-                for transcript in transcript_list:
-                    if transcript.is_translatable:
-                        translated_transcripts.append(transcript)
-
-                if translated_transcripts:
-                    # Sort based on preferred language order
-                    for lang in preferred_langs:
-                        for transcript in translated_transcripts:
-                            if transcript.language_code == lang:
-                                return transcript.fetch()
-                    # If no preferred language found, return translation to first preferred language
-                    translation = translated_transcripts[0].translate(
-                        preferred_langs[0]
-                    )
-                    return translation.fetch()
-            except NoTranscriptFound:
-                pass
-
-            raise Exception("No suitable transcript found")
+            for transcript in transcript_list:
+                if not transcript.is_generated and not transcript.is_translatable:
+                    manual_transcripts.append(transcript)
+
+            if manual_transcripts:
+                # Sort based on preferred language order
+                for lang in preferred_langs:
+                    for transcript in manual_transcripts:
+                        if transcript.language_code == lang:
+                            return transcript.fetch()
+                # If no preferred language found, return first manual transcript
+                return manual_transcripts[0].fetch()
+        except NoTranscriptFound:
+            pass
+
+        # Second try: Auto-generated transcripts in preferred languages
+        generated_transcripts = []
+        try:
+            for transcript in transcript_list:
+                if transcript.is_generated and not transcript.is_translatable:
+                    generated_transcripts.append(transcript)
+
+            if generated_transcripts:
+                # Sort based on preferred language order
+                for lang in preferred_langs:
+                    for transcript in generated_transcripts:
+                        if transcript.language_code == lang:
+                            return transcript.fetch()
+                # If no preferred language found, return first generated transcript
+                return generated_transcripts[0].fetch()
+        except NoTranscriptFound:
+            pass
+
+        # Last try: Translated transcripts in preferred languages
+        translated_transcripts = []
+        try:
+            for transcript in transcript_list:
+                if transcript.is_translatable:
+                    translated_transcripts.append(transcript)
+
+            if translated_transcripts:
+                # Sort based on preferred language order
+                for lang in preferred_langs:
+                    for transcript in translated_transcripts:
+                        if transcript.language_code == lang:
+                            return transcript.fetch()
+                # If no preferred language found, return translation to first preferred language
+                translation = translated_transcripts[0].translate(preferred_langs[0])
+                return translation.fetch()
+        except NoTranscriptFound:
+            pass
+
+        raise Exception("No suitable transcript found")
 
+    except Exception as e:
+        logger.error(f"Failed to get transcript for video {video_id}: {e}")
+        return None
+
+
+def extract_transcript_pytubefix(url, languages=["en", "es", "pt"]):
+    from pytubefix import YouTube
+
+    yt = YouTube(url)
+    logger.debug(f"Captions: {yt.captions}")
+
+    # Try to get captions in the preferred languages
+    if yt.captions:
+        for lang in languages:
+            if lang in yt.captions:
+                caption = yt.captions[lang]
+                break
+            elif f"a.{lang}" in yt.captions:
+                caption = yt.captions[f"a.{lang}"]
+                break
+        else:  # No preferred language found, use the first available
+            caption_key = list(yt.captions.keys())[0]
+            caption = yt.captions[caption_key.code]
+        try:
+            srt_captions = caption.generate_srt_captions()
+            txt_captions = caption.generate_txt_captions()
+            return txt_captions, srt_captions
+        except KeyError as e:
+            logger.error(f"KeyError while generating captions for {caption}: {e}")
+            return None, None
         except Exception as e:
-            if e.__class__.__name__ == "ParserError":
-                logger.warning(
-                    f"ParserError on attempt {attempt+1}/5 for video {video_id}. Retrying..."
-                )
-                if attempt == max_attempts - 1:
-                    logger.error(
-                        f"Failed to get transcript for video {video_id} after {max_attempts} attempts due to repeated ParserError."
-                    )
-                    return None
-                await asyncio.sleep(2)
-                continue
-            else:
-                logger.error(f"Failed to get transcript for video {video_id}: {e}")
-                return None
-    return None
+            logger.error(
+                f"Unexpected error while generating captions for {caption}: {e}"
+            )
+            return None, None
+
+    return None, None
 
 
 async def extract_youtube_transcript(state: ProcessSourceState):
@@ -162,11 +177,10 @@ async def extract_youtube_transcript(state: ProcessSourceState):
         "preferred_languages", ["en", "es", "pt"]
     )
 
+    # quick fix since transcripts api is not working for now
+    engine = "pytubefix"
     video_id = await _extract_youtube_id(state.url)
-    transcript = await get_best_transcript(video_id, languages)
 
-    logger.debug(f"Found transcript: {transcript}")
-    formatter = TextFormatter()
     try:
         title = await get_video_title(video_id)
     except Exception as e:
@@ -174,19 +188,29 @@ async def extract_youtube_transcript(state: ProcessSourceState):
         logger.exception(e)
         title = ""
 
-    try:
-        formatted_content = formatter.format_transcript(transcript)
-    except Exception as e:
-        logger.critical(f"Failed to format transcript for video_id: {video_id}")
-        logger.exception(e)
-        formatted_content = ""
+    if engine == "pytubefix":
+        formatted_content, transcript_raw = extract_transcript_pytubefix(
+            state.url, languages
+        )
+    if engine == "transcripts-api":
+        transcript = await get_best_transcript(video_id, languages)
 
-    try:
-        transcript_raw = transcript.to_raw_data()
-    except Exception as e:
-        logger.critical(f"Failed to get raw transcript for video_id: {video_id}")
-        logger.exception(e)
-        transcript_raw = ""
+        logger.debug(f"Found transcript: {transcript}")
+        formatter = TextFormatter()
+
+        try:
+            formatted_content = formatter.format_transcript(transcript)
+        except Exception as e:
+            logger.critical(f"Failed to format transcript for video_id: {video_id}")
+            logger.exception(e)
+            formatted_content = ""
+
+        try:
+            transcript_raw = transcript.to_raw_data()
+        except Exception as e:
+            logger.critical(f"Failed to get raw transcript for video_id: {video_id}")
+            logger.exception(e)
+            transcript_raw = ""
 
     return {
         "content": formatted_content,

{content_core-1.0.2.dist-info → content_core-1.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 1.0.2
+Version: 1.0.4
 Summary: Extract what matters from any media source
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE
@@ -27,6 +27,7 @@ Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: python-magic-bin==0.4.14; sys_platform == 'win32'
 Requires-Dist: python-magic>=0.4.27
 Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: pytubefix>=9.1.1
 Requires-Dist: readability-lxml>=0.8.4.1
 Requires-Dist: validators>=0.34.0
 Requires-Dist: youtube-transcript-api>=1.0.3

{content_core-1.0.2.dist-info → content_core-1.0.4.dist-info}/RECORD

@@ -18,7 +18,7 @@ content_core/content/extraction/__init__.py,sha256=TaYw6CAcG62GZfsJxeZ6VJDLP85BU
 content_core/content/extraction/graph.py,sha256=Nn2iaQc6YJ4Qt8WKTolwUQUNNqUlwpV8YnijESGvnD0,7605
 content_core/content/identification/__init__.py,sha256=x4n8JIjDwmPvAopEEEcmZjlozg-zGbMq_s9VYdBjzYU,169
 content_core/content/summary/__init__.py,sha256=ReKCZWKfDtqlInKeh87Y1DEfiNzVWabGybEz3hS2FrI,114
-content_core/content/summary/core.py,sha256=LejUbPxnRD0sbO6MupiIb-IHLxEUGU5beBZwmIiBncc,542
+content_core/content/summary/core.py,sha256=kEabpETljzUb-yf0NcVWTOuCtayESo74gGBVDX7YTFs,550
 content_core/notebooks/run.ipynb,sha256=WPBNcQUNXR5MldNMghVcU4vE4ibrVmlANa80baQn8TA,371078
 content_core/processors/audio.py,sha256=Mie20g_2Akhw6BHBVo3sHMpDRYUkqBI72lEDakscx3s,5729
 content_core/processors/docling.py,sha256=dkXehsQdfyWXfrK1K_6Pye50ABM7DxMk6TMguabM9Pc,2151
@@ -27,13 +27,13 @@ content_core/processors/pdf.py,sha256=9jf-eROAqw6yQwdlbsxPXsaJXY26hVG7nSTPH9n4af
 content_core/processors/text.py,sha256=kKHA60-NYjLmCTYUnk8TdJxQQ0Shkg-K61Ezqaelz7k,1158
 content_core/processors/url.py,sha256=6WT8Sw2VHiKyhgWXi_jZjKjwnT_QPSPcH4P99RKbjgU,7521
 content_core/processors/video.py,sha256=3WnZwTswvTLm8PtQhKwoqJ2BH6YZi62dMUjALwJiebo,5196
-content_core/processors/youtube.py,sha256=pLyNy6ebz80T0e-XnBnj36o22Gm4s_jfjkGCNtKz0so,7254
+content_core/processors/youtube.py,sha256=MOeZboVfM9_C87L5mnUVvsbQeKoznwJoYn1wP1_hA_U,7869
 content_core/tools/__init__.py,sha256=DuJmd7fE-NpDvLP8IW1XY5MUkAQcdks52rn2jk4N8jQ,231
 content_core/tools/cleanup.py,sha256=5IdKedsFyRQMdYzgFSKtsfyxJldbroXQXHesHICNENI,523
 content_core/tools/extract.py,sha256=-r2_jsuMMXyXxGVqWhh1ilNPo_UMYAbw3Pkp1FzPy5g,577
 content_core/tools/summarize.py,sha256=DPfeglLWB08q8SvHrsKpOKZ35XjduUDs2J02ISwjdj0,596
-content_core-1.0.2.dist-info/METADATA,sha256=Pq6mo1Atip1YKA3mB6q-eMrbkPgkodV1K_TVGtOsNiA,11876
-content_core-1.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-content_core-1.0.2.dist-info/entry_points.txt,sha256=9fGQUk6bxBVXj9PRwfWVPn54ClSEJV7J-KBLXtjOhQw,99
-content_core-1.0.2.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
-content_core-1.0.2.dist-info/RECORD,,
+content_core-1.0.4.dist-info/METADATA,sha256=SdXexgOV0tc4ArCYWjxrZog4esHJxW0zh8pdnZFqLi8,11908
+content_core-1.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+content_core-1.0.4.dist-info/entry_points.txt,sha256=9fGQUk6bxBVXj9PRwfWVPn54ClSEJV7J-KBLXtjOhQw,99
+content_core-1.0.4.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
+content_core-1.0.4.dist-info/RECORD,,