content-core 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

content_core/processors/youtube.py

@@ -1,4 +1,3 @@
-import asyncio
 import re
 import ssl
 
@@ -69,86 +68,95 @@ async def _extract_youtube_id(url):
 
 
 async def get_best_transcript(video_id, preferred_langs=["en", "es", "pt"]):
-    max_attempts = 5
-    for attempt in range(max_attempts):
+    try:
+        transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+
+        # First try: Manual transcripts in preferred languages
+        manual_transcripts = []
+        try:
+            for transcript in transcript_list:
+                if not transcript.is_generated and not transcript.is_translatable:
+                    manual_transcripts.append(transcript)
+
+            if manual_transcripts:
+                # Sort based on preferred language order
+                for lang in preferred_langs:
+                    for transcript in manual_transcripts:
+                        if transcript.language_code == lang:
+                            return transcript.fetch()
+                # If no preferred language found, return first manual transcript
+                return manual_transcripts[0].fetch()
+        except NoTranscriptFound:
+            pass
+
+        # Second try: Auto-generated transcripts in preferred languages
+        generated_transcripts = []
+        try:
+            for transcript in transcript_list:
+                if transcript.is_generated and not transcript.is_translatable:
+                    generated_transcripts.append(transcript)
+
+            if generated_transcripts:
+                # Sort based on preferred language order
+                for lang in preferred_langs:
+                    for transcript in generated_transcripts:
+                        if transcript.language_code == lang:
+                            return transcript.fetch()
+                # If no preferred language found, return first generated transcript
+                return generated_transcripts[0].fetch()
+        except NoTranscriptFound:
+            pass
+
+        # Last try: Translated transcripts in preferred languages
+        translated_transcripts = []
         try:
-            transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
-
-            # First try: Manual transcripts in preferred languages
-            manual_transcripts = []
-            try:
-                for transcript in transcript_list:
-                    if not transcript.is_generated and not transcript.is_translatable:
-                        manual_transcripts.append(transcript)
-
-                if manual_transcripts:
-                    # Sort based on preferred language order
-                    for lang in preferred_langs:
-                        for transcript in manual_transcripts:
-                            if transcript.language_code == lang:
-                                return transcript.fetch()
-                    # If no preferred language found, return first manual transcript
-                    return manual_transcripts[0].fetch()
-            except NoTranscriptFound:
-                pass
-
-            # Second try: Auto-generated transcripts in preferred languages
-            generated_transcripts = []
-            try:
-                for transcript in transcript_list:
-                    if transcript.is_generated and not transcript.is_translatable:
-                        generated_transcripts.append(transcript)
-
-                if generated_transcripts:
-                    # Sort based on preferred language order
-                    for lang in preferred_langs:
-                        for transcript in generated_transcripts:
-                            if transcript.language_code == lang:
-                                return transcript.fetch()
-                    # If no preferred language found, return first generated transcript
-                    return generated_transcripts[0].fetch()
-            except NoTranscriptFound:
-                pass
-
-            # Last try: Translated transcripts in preferred languages
-            translated_transcripts = []
-            try:
-                for transcript in transcript_list:
-                    if transcript.is_translatable:
-                        translated_transcripts.append(transcript)
-
-                if translated_transcripts:
-                    # Sort based on preferred language order
-                    for lang in preferred_langs:
-                        for transcript in translated_transcripts:
-                            if transcript.language_code == lang:
-                                return transcript.fetch()
-                    # If no preferred language found, return translation to first preferred language
-                    translation = translated_transcripts[0].translate(
-                        preferred_langs[0]
-                    )
-                    return translation.fetch()
-            except NoTranscriptFound:
-                pass
-
-            raise Exception("No suitable transcript found")
+            for transcript in transcript_list:
+                if transcript.is_translatable:
+                    translated_transcripts.append(transcript)
+
+            if translated_transcripts:
+                # Sort based on preferred language order
+                for lang in preferred_langs:
+                    for transcript in translated_transcripts:
+                        if transcript.language_code == lang:
+                            return transcript.fetch()
+                # If no preferred language found, return translation to first preferred language
+                translation = translated_transcripts[0].translate(preferred_langs[0])
+                return translation.fetch()
+        except NoTranscriptFound:
+            pass
+
+        raise Exception("No suitable transcript found")
 
-        except Exception as e:
-            if e.__class__.__name__ == "ParserError":
-                logger.warning(
-                    f"ParserError on attempt {attempt+1}/5 for video {video_id}. Retrying..."
-                )
-                if attempt == max_attempts - 1:
-                    logger.error(
-                        f"Failed to get transcript for video {video_id} after {max_attempts} attempts due to repeated ParserError."
-                    )
-                    return None
-                await asyncio.sleep(2)
-                continue
-            else:
-                logger.error(f"Failed to get transcript for video {video_id}: {e}")
-                return None
-    return None
+    except Exception as e:
+        logger.error(f"Failed to get transcript for video {video_id}: {e}")
+        return None
+
+
+def extract_transcript_pytubefix(url, languages=["en", "es", "pt"]):
+    from pytubefix import YouTube
+
+    yt = YouTube(url)
+    print(yt.captions)
+
+    # Try to get captions in the preferred languages
+    if yt.captions:
+        for lang in languages:
+            if lang in yt.captions:
+                caption = yt.captions[lang]
+                break
+            elif f"a.{lang}" in yt.captions:
+                caption = yt.captions[f"a.{lang}"]
+                break
+        else:  # No preferred language found, use the first available
+            caption_key = next(iter(yt.captions))
+            caption = yt.captions[caption_key]
+
+        srt_captions = caption.generate_srt_captions()
+        txt_captions = caption.generate_txt_captions()
+        return txt_captions, srt_captions
+
+    return None, None
 
 
 async def extract_youtube_transcript(state: ProcessSourceState):
@@ -162,11 +170,10 @@ async def extract_youtube_transcript(state: ProcessSourceState):
         "preferred_languages", ["en", "es", "pt"]
     )
 
+    # quick fix since transcripts api is not working for now
+    engine = "pytubefix"
     video_id = await _extract_youtube_id(state.url)
-    transcript = await get_best_transcript(video_id, languages)
 
-    logger.debug(f"Found transcript: {transcript}")
-    formatter = TextFormatter()
     try:
         title = await get_video_title(video_id)
     except Exception as e:
@@ -174,19 +181,29 @@ async def extract_youtube_transcript(state: ProcessSourceState):
         logger.exception(e)
         title = ""
 
-    try:
-        formatted_content = formatter.format_transcript(transcript)
-    except Exception as e:
-        logger.critical(f"Failed to format transcript for video_id: {video_id}")
-        logger.exception(e)
-        formatted_content = ""
+    if engine == "pytubefix":
+        formatted_content, transcript_raw = extract_transcript_pytubefix(
+            state.url, languages
+        )
+    if engine == "transcripts-api":
+        transcript = await get_best_transcript(video_id, languages)
 
-    try:
-        transcript_raw = transcript.to_raw_data()
-    except Exception as e:
-        logger.critical(f"Failed to get raw transcript for video_id: {video_id}")
-        logger.exception(e)
-        transcript_raw = ""
+        logger.debug(f"Found transcript: {transcript}")
+        formatter = TextFormatter()
+
+        try:
+            formatted_content = formatter.format_transcript(transcript)
+        except Exception as e:
+            logger.critical(f"Failed to format transcript for video_id: {video_id}")
+            logger.exception(e)
+            formatted_content = ""
+
+        try:
+            transcript_raw = transcript.to_raw_data()
+        except Exception as e:
+            logger.critical(f"Failed to get raw transcript for video_id: {video_id}")
+            logger.exception(e)
+            transcript_raw = ""
 
     return {
         "content": formatted_content,

{content_core-1.0.1.dist-info → content_core-1.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 1.0.1
+Version: 1.0.3
 Summary: Extract what matters from any media source
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE
@@ -11,7 +11,7 @@ Requires-Dist: asciidoc>=10.2.1
 Requires-Dist: bs4>=0.0.2
 Requires-Dist: dicttoxml>=1.7.16
 Requires-Dist: docling>=2.34.0
-Requires-Dist: esperanto[openai]>=1.2.0
+Requires-Dist: esperanto>=1.2.0
 Requires-Dist: firecrawl-py>=2.7.0
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: langdetect>=1.0.9
@@ -24,8 +24,10 @@ Requires-Dist: pillow>=10.4.0
 Requires-Dist: pymupdf>=1.25.5
 Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: python-magic-bin==0.4.14; sys_platform == 'win32'
 Requires-Dist: python-magic>=0.4.27
 Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: pytubefix>=9.1.1
 Requires-Dist: readability-lxml>=0.8.4.1
 Requires-Dist: validators>=0.34.0
 Requires-Dist: youtube-transcript-api>=1.0.3

{content_core-1.0.1.dist-info → content_core-1.0.3.dist-info}/RECORD

@@ -27,13 +27,13 @@ content_core/processors/pdf.py,sha256=9jf-eROAqw6yQwdlbsxPXsaJXY26hVG7nSTPH9n4af
 content_core/processors/text.py,sha256=kKHA60-NYjLmCTYUnk8TdJxQQ0Shkg-K61Ezqaelz7k,1158
 content_core/processors/url.py,sha256=6WT8Sw2VHiKyhgWXi_jZjKjwnT_QPSPcH4P99RKbjgU,7521
 content_core/processors/video.py,sha256=3WnZwTswvTLm8PtQhKwoqJ2BH6YZi62dMUjALwJiebo,5196
-content_core/processors/youtube.py,sha256=pLyNy6ebz80T0e-XnBnj36o22Gm4s_jfjkGCNtKz0so,7254
+content_core/processors/youtube.py,sha256=y-1KH_1uYyO6l6QpjBtzP1KSBjclYeGlj6zDrloAcPc,7490
 content_core/tools/__init__.py,sha256=DuJmd7fE-NpDvLP8IW1XY5MUkAQcdks52rn2jk4N8jQ,231
 content_core/tools/cleanup.py,sha256=5IdKedsFyRQMdYzgFSKtsfyxJldbroXQXHesHICNENI,523
 content_core/tools/extract.py,sha256=-r2_jsuMMXyXxGVqWhh1ilNPo_UMYAbw3Pkp1FzPy5g,577
 content_core/tools/summarize.py,sha256=DPfeglLWB08q8SvHrsKpOKZ35XjduUDs2J02ISwjdj0,596
-content_core-1.0.1.dist-info/METADATA,sha256=ANYJN5e0FUF1rozZd1dpbncKwHiUJlLlu_DsF43DDAE,11819
-content_core-1.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-content_core-1.0.1.dist-info/entry_points.txt,sha256=9fGQUk6bxBVXj9PRwfWVPn54ClSEJV7J-KBLXtjOhQw,99
-content_core-1.0.1.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
-content_core-1.0.1.dist-info/RECORD,,
+content_core-1.0.3.dist-info/METADATA,sha256=R6woFztrB88fGVSOVMkrBFAgqwPAPrIw5MClIUKq1Xo,11908
+content_core-1.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+content_core-1.0.3.dist-info/entry_points.txt,sha256=9fGQUk6bxBVXj9PRwfWVPn54ClSEJV7J-KBLXtjOhQw,99
+content_core-1.0.3.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
+content_core-1.0.3.dist-info/RECORD,,