content-core 1.0.0__tar.gz → 1.0.2__tar.gz

{content_core-1.0.0 → content_core-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 1.0.0
+Version: 1.0.2
 Summary: Extract what matters from any media source
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE
@@ -11,7 +11,7 @@ Requires-Dist: asciidoc>=10.2.1
 Requires-Dist: bs4>=0.0.2
 Requires-Dist: dicttoxml>=1.7.16
 Requires-Dist: docling>=2.34.0
-Requires-Dist: esperanto[openai]>=1.2.0
+Requires-Dist: esperanto>=1.2.0
 Requires-Dist: firecrawl-py>=2.7.0
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: langdetect>=1.0.9
@@ -24,6 +24,7 @@ Requires-Dist: pillow>=10.4.0
 Requires-Dist: pymupdf>=1.25.5
 Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: python-magic-bin==0.4.14; sys_platform == 'win32'
 Requires-Dist: python-magic>=0.4.27
 Requires-Dist: python-pptx>=1.0.2
 Requires-Dist: readability-lxml>=0.8.4.1

{content_core-1.0.0 → content_core-1.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "content-core"
-version = "1.0.0"
+version = "1.0.2"
 description = "Extract what matters from any media source"
 readme = "README.md"
 homepage = "https://github.com/lfnovo/content-core"
@@ -11,7 +11,7 @@ requires-python = ">=3.10"
 dependencies = [
     "aiohttp>=3.11",
     "bs4>=0.0.2",
-    "esperanto[openai]>=1.2.0",
+    "esperanto>=1.2.0",
     "jinja2>=3.1.6",
     "langdetect>=1.0.9",
     "loguru>=0.7.3",
@@ -33,6 +33,8 @@ dependencies = [
     "docling>=2.34.0",
     "pillow>=10.4.0",
     "asciidoc>=10.2.1",
+    "python-magic-bin==0.4.14; sys_platform == 'win32'",
+
 ]
 
 [project.scripts]

{content_core-1.0.0 → content_core-1.0.2}/src/content_core/processors/youtube.py

@@ -1,3 +1,4 @@
+import asyncio
 import re
 import ssl
 
@@ -68,69 +69,86 @@ async def _extract_youtube_id(url):
 
 
 async def get_best_transcript(video_id, preferred_langs=["en", "es", "pt"]):
-    try:
-        transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
-
-        # First try: Manual transcripts in preferred languages
-        manual_transcripts = []
-        try:
-            for transcript in transcript_list:
-                if not transcript.is_generated and not transcript.is_translatable:
-                    manual_transcripts.append(transcript)
-
-            if manual_transcripts:
-                # Sort based on preferred language order
-                for lang in preferred_langs:
-                    for transcript in manual_transcripts:
-                        if transcript.language_code == lang:
-                            return transcript.fetch()
-                # If no preferred language found, return first manual transcript
-                return manual_transcripts[0].fetch()
-        except NoTranscriptFound:
-            pass
-
-        # Second try: Auto-generated transcripts in preferred languages
-        generated_transcripts = []
-        try:
-            for transcript in transcript_list:
-                if transcript.is_generated and not transcript.is_translatable:
-                    generated_transcripts.append(transcript)
-
-            if generated_transcripts:
-                # Sort based on preferred language order
-                for lang in preferred_langs:
-                    for transcript in generated_transcripts:
-                        if transcript.language_code == lang:
-                            return transcript.fetch()
-                # If no preferred language found, return first generated transcript
-                return generated_transcripts[0].fetch()
-        except NoTranscriptFound:
-            pass
-
-        # Last try: Translated transcripts in preferred languages
-        translated_transcripts = []
+    max_attempts = 5
+    for attempt in range(max_attempts):
         try:
-            for transcript in transcript_list:
-                if transcript.is_translatable:
-                    translated_transcripts.append(transcript)
-
-            if translated_transcripts:
-                # Sort based on preferred language order
-                for lang in preferred_langs:
-                    for transcript in translated_transcripts:
-                        if transcript.language_code == lang:
-                            return transcript.fetch()
-                # If no preferred language found, return translation to first preferred language
-                translation = translated_transcripts[0].translate(preferred_langs[0])
-                return translation.fetch()
-        except NoTranscriptFound:
-            pass
-
-        raise Exception("No suitable transcript found")
-
-    except Exception as e:
-        logger.error(f"Failed to get transcript for video {video_id}: {e}")
-        return None
+            transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+
+            # First try: Manual transcripts in preferred languages
+            manual_transcripts = []
+            try:
+                for transcript in transcript_list:
+                    if not transcript.is_generated and not transcript.is_translatable:
+                        manual_transcripts.append(transcript)
+
+                if manual_transcripts:
+                    # Sort based on preferred language order
+                    for lang in preferred_langs:
+                        for transcript in manual_transcripts:
+                            if transcript.language_code == lang:
+                                return transcript.fetch()
+                    # If no preferred language found, return first manual transcript
+                    return manual_transcripts[0].fetch()
+            except NoTranscriptFound:
+                pass
+
+            # Second try: Auto-generated transcripts in preferred languages
+            generated_transcripts = []
+            try:
+                for transcript in transcript_list:
+                    if transcript.is_generated and not transcript.is_translatable:
+                        generated_transcripts.append(transcript)
+
+                if generated_transcripts:
+                    # Sort based on preferred language order
+                    for lang in preferred_langs:
+                        for transcript in generated_transcripts:
+                            if transcript.language_code == lang:
+                                return transcript.fetch()
+                    # If no preferred language found, return first generated transcript
+                    return generated_transcripts[0].fetch()
+            except NoTranscriptFound:
+                pass
+
+            # Last try: Translated transcripts in preferred languages
+            translated_transcripts = []
+            try:
+                for transcript in transcript_list:
+                    if transcript.is_translatable:
+                        translated_transcripts.append(transcript)
+
+                if translated_transcripts:
+                    # Sort based on preferred language order
+                    for lang in preferred_langs:
+                        for transcript in translated_transcripts:
+                            if transcript.language_code == lang:
+                                return transcript.fetch()
+                    # If no preferred language found, return translation to first preferred language
+                    translation = translated_transcripts[0].translate(
+                        preferred_langs[0]
+                    )
+                    return translation.fetch()
+            except NoTranscriptFound:
+                pass
+
+            raise Exception("No suitable transcript found")
+
+        except Exception as e:
+            if e.__class__.__name__ == "ParserError":
+                logger.warning(
+                    f"ParserError on attempt {attempt+1}/5 for video {video_id}. Retrying..."
+                )
+                if attempt == max_attempts - 1:
+                    logger.error(
+                        f"Failed to get transcript for video {video_id} after {max_attempts} attempts due to repeated ParserError."
+                    )
+                    return None
+                await asyncio.sleep(2)
+                continue
+            else:
+                logger.error(f"Failed to get transcript for video {video_id}: {e}")
+                return None
+    return None
 
 
 async def extract_youtube_transcript(state: ProcessSourceState):

{content_core-1.0.0 → content_core-1.0.2}/uv.lock

@@ -410,7 +410,7 @@ wheels = [
 
 [[package]]
 name = "content-core"
-version = "1.0"
+version = "1.0.2"
 source = { editable = "." }
 dependencies = [
     { name = "ai-prompter" },
@@ -419,7 +419,7 @@ dependencies = [
     { name = "bs4" },
     { name = "dicttoxml" },
     { name = "docling" },
-    { name = "esperanto", extra = ["openai"] },
+    { name = "esperanto" },
     { name = "firecrawl-py" },
     { name = "jinja2" },
     { name = "langdetect" },
@@ -433,6 +433,7 @@ dependencies = [
     { name = "python-docx" },
     { name = "python-dotenv" },
     { name = "python-magic" },
+    { name = "python-magic-bin", marker = "(platform_machine != 'aarch64' and platform_system == 'Linux' and sys_platform == 'win32') or (platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'win32')" },
     { name = "python-pptx" },
     { name = "readability-lxml" },
     { name = "validators" },
@@ -457,7 +458,7 @@ requires-dist = [
     { name = "bs4", specifier = ">=0.0.2" },
     { name = "dicttoxml", specifier = ">=1.7.16" },
     { name = "docling", specifier = ">=2.34.0" },
-    { name = "esperanto", extras = ["openai"], specifier = ">=1.2.0" },
+    { name = "esperanto", specifier = ">=1.2.0" },
     { name = "firecrawl-py", specifier = ">=2.7.0" },
     { name = "jinja2", specifier = ">=3.1.6" },
     { name = "langdetect", specifier = ">=1.0.9" },
@@ -471,6 +472,7 @@ requires-dist = [
     { name = "python-docx", specifier = ">=1.1.2" },
     { name = "python-dotenv", specifier = ">=1.1.0" },
     { name = "python-magic", specifier = ">=0.4.27" },
+    { name = "python-magic-bin", marker = "sys_platform == 'win32'", specifier = "==0.4.14" },
     { name = "python-pptx", specifier = ">=1.0.2" },
     { name = "readability-lxml", specifier = ">=0.8.4.1" },
     { name = "validators", specifier = ">=0.34.0" },
@@ -729,11 +731,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f9/79/5d74f2b8f9d73da83bfe80a39ff11505a2a285c03a869750db98cd89ddfd/esperanto-1.2.1-py3-none-any.whl", hash = "sha256:2fa41e5e35c847b1fe58395906d8877035f7e55d6429870d897781f7c9f17c42", size = 57680 },
 ]
 
-[package.optional-dependencies]
-openai = [
-    { name = "openai" },
-]
-
 [[package]]
 name = "et-xmlfile"
 version = "2.0.0"
@@ -2919,6 +2916,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3", size = 13840 },
 ]
 
+[[package]]
+name = "python-magic-bin"
+version = "0.4.14"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5a/5d/10b9ac745d9fd2f7151a2ab901e6bb6983dbd70e87c71111f54859d1ca2e/python_magic_bin-0.4.14-py2.py3-none-win32.whl", hash = "sha256:34a788c03adde7608028203e2dbb208f1f62225ad91518787ae26d603ae68892", size = 397784 },
+    { url = "https://files.pythonhosted.org/packages/07/c2/094e3d62b906d952537196603a23aec4bcd7c6126bf80eb14e6f9f4be3a2/python_magic_bin-0.4.14-py2.py3-none-win_amd64.whl", hash = "sha256:90be6206ad31071a36065a2fc169c5afb5e0355cbe6030e87641c6c62edc2b69", size = 409299 },
+]
+
 [[package]]
 name = "python-pptx"
 version = "1.0.2"