content-core 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

content_core/processors/youtube.py

@@ -1,3 +1,4 @@
+import asyncio
 import re
 import ssl
 
@@ -68,69 +69,86 @@ async def _extract_youtube_id(url):
 
 
 async def get_best_transcript(video_id, preferred_langs=["en", "es", "pt"]):
-    try:
-        transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
-
-        # First try: Manual transcripts in preferred languages
-        manual_transcripts = []
-        try:
-            for transcript in transcript_list:
-                if not transcript.is_generated and not transcript.is_translatable:
-                    manual_transcripts.append(transcript)
-
-            if manual_transcripts:
-                # Sort based on preferred language order
-                for lang in preferred_langs:
-                    for transcript in manual_transcripts:
-                        if transcript.language_code == lang:
-                            return transcript.fetch()
-                # If no preferred language found, return first manual transcript
-                return manual_transcripts[0].fetch()
-        except NoTranscriptFound:
-            pass
-
-        # Second try: Auto-generated transcripts in preferred languages
-        generated_transcripts = []
-        try:
-            for transcript in transcript_list:
-                if transcript.is_generated and not transcript.is_translatable:
-                    generated_transcripts.append(transcript)
-
-            if generated_transcripts:
-                # Sort based on preferred language order
-                for lang in preferred_langs:
-                    for transcript in generated_transcripts:
-                        if transcript.language_code == lang:
-                            return transcript.fetch()
-                # If no preferred language found, return first generated transcript
-                return generated_transcripts[0].fetch()
-        except NoTranscriptFound:
-            pass
-
-        # Last try: Translated transcripts in preferred languages
-        translated_transcripts = []
+    max_attempts = 5
+    for attempt in range(max_attempts):
         try:
-            for transcript in transcript_list:
-                if transcript.is_translatable:
-                    translated_transcripts.append(transcript)
-
-            if translated_transcripts:
-                # Sort based on preferred language order
-                for lang in preferred_langs:
-                    for transcript in translated_transcripts:
-                        if transcript.language_code == lang:
-                            return transcript.fetch()
-                # If no preferred language found, return translation to first preferred language
-                translation = translated_transcripts[0].translate(preferred_langs[0])
-                return translation.fetch()
-        except NoTranscriptFound:
-            pass
-
-        raise Exception("No suitable transcript found")
-
-    except Exception as e:
-        logger.error(f"Failed to get transcript for video {video_id}: {e}")
-        return None
+            transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+
+            # First try: Manual transcripts in preferred languages
+            manual_transcripts = []
+            try:
+                for transcript in transcript_list:
+                    if not transcript.is_generated and not transcript.is_translatable:
+                        manual_transcripts.append(transcript)
+
+                if manual_transcripts:
+                    # Sort based on preferred language order
+                    for lang in preferred_langs:
+                        for transcript in manual_transcripts:
+                            if transcript.language_code == lang:
+                                return transcript.fetch()
+                    # If no preferred language found, return first manual transcript
+                    return manual_transcripts[0].fetch()
+            except NoTranscriptFound:
+                pass
+
+            # Second try: Auto-generated transcripts in preferred languages
+            generated_transcripts = []
+            try:
+                for transcript in transcript_list:
+                    if transcript.is_generated and not transcript.is_translatable:
+                        generated_transcripts.append(transcript)
+
+                if generated_transcripts:
+                    # Sort based on preferred language order
+                    for lang in preferred_langs:
+                        for transcript in generated_transcripts:
+                            if transcript.language_code == lang:
+                                return transcript.fetch()
+                    # If no preferred language found, return first generated transcript
+                    return generated_transcripts[0].fetch()
+            except NoTranscriptFound:
+                pass
+
+            # Last try: Translated transcripts in preferred languages
+            translated_transcripts = []
+            try:
+                for transcript in transcript_list:
+                    if transcript.is_translatable:
+                        translated_transcripts.append(transcript)
+
+                if translated_transcripts:
+                    # Sort based on preferred language order
+                    for lang in preferred_langs:
+                        for transcript in translated_transcripts:
+                            if transcript.language_code == lang:
+                                return transcript.fetch()
+                    # If no preferred language found, return translation to first preferred language
+                    translation = translated_transcripts[0].translate(
+                        preferred_langs[0]
+                    )
+                    return translation.fetch()
+            except NoTranscriptFound:
+                pass
+
+            raise Exception("No suitable transcript found")
+
+        except Exception as e:
+            if e.__class__.__name__ == "ParserError":
+                logger.warning(
+                    f"ParserError on attempt {attempt+1}/5 for video {video_id}. Retrying..."
+                )
+                if attempt == max_attempts - 1:
+                    logger.error(
+                        f"Failed to get transcript for video {video_id} after {max_attempts} attempts due to repeated ParserError."
+                    )
+                    return None
+                await asyncio.sleep(2)
+                continue
+            else:
+                logger.error(f"Failed to get transcript for video {video_id}: {e}")
+                return None
+    return None
 
 
 async def extract_youtube_transcript(state: ProcessSourceState):

{content_core-1.0.0.dist-info → content_core-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 1.0.0
+Version: 1.0.1
 Summary: Extract what matters from any media source
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE

{content_core-1.0.0.dist-info → content_core-1.0.1.dist-info}/RECORD

@@ -27,13 +27,13 @@ content_core/processors/pdf.py,sha256=9jf-eROAqw6yQwdlbsxPXsaJXY26hVG7nSTPH9n4af
 content_core/processors/text.py,sha256=kKHA60-NYjLmCTYUnk8TdJxQQ0Shkg-K61Ezqaelz7k,1158
 content_core/processors/url.py,sha256=6WT8Sw2VHiKyhgWXi_jZjKjwnT_QPSPcH4P99RKbjgU,7521
 content_core/processors/video.py,sha256=3WnZwTswvTLm8PtQhKwoqJ2BH6YZi62dMUjALwJiebo,5196
-content_core/processors/youtube.py,sha256=g_A-rv5bzq2GIuwqMH70YAnDK-4BZqpgQP0IQ3j9zXE,6340
+content_core/processors/youtube.py,sha256=pLyNy6ebz80T0e-XnBnj36o22Gm4s_jfjkGCNtKz0so,7254
 content_core/tools/__init__.py,sha256=DuJmd7fE-NpDvLP8IW1XY5MUkAQcdks52rn2jk4N8jQ,231
 content_core/tools/cleanup.py,sha256=5IdKedsFyRQMdYzgFSKtsfyxJldbroXQXHesHICNENI,523
 content_core/tools/extract.py,sha256=-r2_jsuMMXyXxGVqWhh1ilNPo_UMYAbw3Pkp1FzPy5g,577
 content_core/tools/summarize.py,sha256=DPfeglLWB08q8SvHrsKpOKZ35XjduUDs2J02ISwjdj0,596
-content_core-1.0.0.dist-info/METADATA,sha256=0TBaT17WQQ3u3YKX4dZYGXLkvnnyFwuxe1Z5uHQr9rQ,11819
-content_core-1.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-content_core-1.0.0.dist-info/entry_points.txt,sha256=9fGQUk6bxBVXj9PRwfWVPn54ClSEJV7J-KBLXtjOhQw,99
-content_core-1.0.0.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
-content_core-1.0.0.dist-info/RECORD,,
+content_core-1.0.1.dist-info/METADATA,sha256=ANYJN5e0FUF1rozZd1dpbncKwHiUJlLlu_DsF43DDAE,11819
+content_core-1.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+content_core-1.0.1.dist-info/entry_points.txt,sha256=9fGQUk6bxBVXj9PRwfWVPn54ClSEJV7J-KBLXtjOhQw,99
+content_core-1.0.1.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
+content_core-1.0.1.dist-info/RECORD,,