content-core 1.0.1__tar.gz → 1.0.3__tar.gz

{content_core-1.0.1 → content_core-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 1.0.1
+Version: 1.0.3
 Summary: Extract what matters from any media source
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE
@@ -11,7 +11,7 @@ Requires-Dist: asciidoc>=10.2.1
 Requires-Dist: bs4>=0.0.2
 Requires-Dist: dicttoxml>=1.7.16
 Requires-Dist: docling>=2.34.0
-Requires-Dist: esperanto[openai]>=1.2.0
+Requires-Dist: esperanto>=1.2.0
 Requires-Dist: firecrawl-py>=2.7.0
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: langdetect>=1.0.9
@@ -24,8 +24,10 @@ Requires-Dist: pillow>=10.4.0
 Requires-Dist: pymupdf>=1.25.5
 Requires-Dist: python-docx>=1.1.2
 Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: python-magic-bin==0.4.14; sys_platform == 'win32'
 Requires-Dist: python-magic>=0.4.27
 Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: pytubefix>=9.1.1
 Requires-Dist: readability-lxml>=0.8.4.1
 Requires-Dist: validators>=0.34.0
 Requires-Dist: youtube-transcript-api>=1.0.3

{content_core-1.0.1 → content_core-1.0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "content-core"
-version = "1.0.1"
+version = "1.0.3"
 description = "Extract what matters from any media source"
 readme = "README.md"
 homepage = "https://github.com/lfnovo/content-core"
@@ -11,7 +11,7 @@ requires-python = ">=3.10"
 dependencies = [
     "aiohttp>=3.11",
     "bs4>=0.0.2",
-    "esperanto[openai]>=1.2.0",
+    "esperanto>=1.2.0",
     "jinja2>=3.1.6",
     "langdetect>=1.0.9",
     "loguru>=0.7.3",
@@ -33,6 +33,8 @@ dependencies = [
     "docling>=2.34.0",
     "pillow>=10.4.0",
     "asciidoc>=10.2.1",
+    "python-magic-bin==0.4.14; sys_platform == 'win32'",
+    "pytubefix>=9.1.1",
 ]
 
 [project.scripts]

content_core-1.0.3/src/content_core/processors/youtube.py

@@ -0,0 +1,212 @@
+import re
+import ssl
+
+import aiohttp
+from bs4 import BeautifulSoup
+from youtube_transcript_api import YouTubeTranscriptApi  # type: ignore
+from youtube_transcript_api.formatters import TextFormatter  # type: ignore
+
+from content_core.common import ProcessSourceState
+from content_core.common.exceptions import NoTranscriptFound
+from content_core.config import CONFIG
+from content_core.logging import logger
+
+ssl._create_default_https_context = ssl._create_unverified_context
+
+
+async def get_video_title(video_id):
+    try:
+        url = f"https://www.youtube.com/watch?v={video_id}"
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                html = await response.text()
+
+        # BeautifulSoup doesn't support async operations
+        soup = BeautifulSoup(html, "html.parser")
+
+        # YouTube stores title in a meta tag
+        title = soup.find("meta", property="og:title")["content"]
+        return title
+
+    except Exception as e:
+        logger.error(f"Failed to get video title: {e}")
+        return None
+
+
+async def _extract_youtube_id(url):
+    """
+    Extract the YouTube video ID from a given URL using regular expressions.
+
+    Args:
+    url (str): The YouTube URL from which to extract the video ID.
+
+    Returns:
+    str: The extracted YouTube video ID or None if no valid ID is found.
+    """
+    # Define a regular expression pattern to capture the YouTube video ID
+    youtube_regex = (
+        r"(?:https?://)?"  # Optional scheme
+        r"(?:www\.)?"  # Optional www.
+        r"(?:"
+        r"youtu\.be/"  # Shortened URL
+        r"|youtube\.com"  # Main URL
+        r"(?:"  # Group start
+        r"/embed/"  # Embed URL
+        r"|/v/"  # Older video URL
+        r"|/watch\?v="  # Standard watch URL
+        r"|/watch\?.+&v="  # Other watch URL
+        r")"  # Group end
+        r")"  # End main group
+        r"([\w-]{11})"  # 11 characters (YouTube video ID)
+    )
+
+    # Search the URL for the pattern
+    match = re.search(youtube_regex, url)
+
+    # Return the video ID if a match is found
+    return match.group(1) if match else None
+
+
+async def get_best_transcript(video_id, preferred_langs=["en", "es", "pt"]):
+    try:
+        transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+
+        # First try: Manual transcripts in preferred languages
+        manual_transcripts = []
+        try:
+            for transcript in transcript_list:
+                if not transcript.is_generated and not transcript.is_translatable:
+                    manual_transcripts.append(transcript)
+
+            if manual_transcripts:
+                # Sort based on preferred language order
+                for lang in preferred_langs:
+                    for transcript in manual_transcripts:
+                        if transcript.language_code == lang:
+                            return transcript.fetch()
+                # If no preferred language found, return first manual transcript
+                return manual_transcripts[0].fetch()
+        except NoTranscriptFound:
+            pass
+
+        # Second try: Auto-generated transcripts in preferred languages
+        generated_transcripts = []
+        try:
+            for transcript in transcript_list:
+                if transcript.is_generated and not transcript.is_translatable:
+                    generated_transcripts.append(transcript)
+
+            if generated_transcripts:
+                # Sort based on preferred language order
+                for lang in preferred_langs:
+                    for transcript in generated_transcripts:
+                        if transcript.language_code == lang:
+                            return transcript.fetch()
+                # If no preferred language found, return first generated transcript
+                return generated_transcripts[0].fetch()
+        except NoTranscriptFound:
+            pass
+
+        # Last try: Translated transcripts in preferred languages
+        translated_transcripts = []
+        try:
+            for transcript in transcript_list:
+                if transcript.is_translatable:
+                    translated_transcripts.append(transcript)
+
+            if translated_transcripts:
+                # Sort based on preferred language order
+                for lang in preferred_langs:
+                    for transcript in translated_transcripts:
+                        if transcript.language_code == lang:
+                            return transcript.fetch()
+                # If no preferred language found, return translation to first preferred language
+                translation = translated_transcripts[0].translate(preferred_langs[0])
+                return translation.fetch()
+        except NoTranscriptFound:
+            pass
+
+        raise Exception("No suitable transcript found")
+
+    except Exception as e:
+        logger.error(f"Failed to get transcript for video {video_id}: {e}")
+        return None
+
+
+def extract_transcript_pytubefix(url, languages=["en", "es", "pt"]):
+    from pytubefix import YouTube
+
+    yt = YouTube(url)
+    print(yt.captions)
+
+    # Try to get captions in the preferred languages
+    if yt.captions:
+        for lang in languages:
+            if lang in yt.captions:
+                caption = yt.captions[lang]
+                break
+            elif f"a.{lang}" in yt.captions:
+                caption = yt.captions[f"a.{lang}"]
+                break
+        else:  # No preferred language found, use the first available
+            caption_key = next(iter(yt.captions))
+            caption = yt.captions[caption_key]
+
+        srt_captions = caption.generate_srt_captions()
+        txt_captions = caption.generate_txt_captions()
+        return txt_captions, srt_captions
+
+    return None, None
+
+
+async def extract_youtube_transcript(state: ProcessSourceState):
+    """
+    Parse the text file and print its content.
+    """
+
+    assert state.url, "No URL provided"
+    logger.warning(f"Extracting transcript from URL: {state.url}")
+    languages = CONFIG.get("youtube_transcripts", {}).get(
+        "preferred_languages", ["en", "es", "pt"]
+    )
+
+    # quick fix since transcripts api is not working for now
+    engine = "pytubefix"
+    video_id = await _extract_youtube_id(state.url)
+
+    try:
+        title = await get_video_title(video_id)
+    except Exception as e:
+        logger.critical(f"Failed to get video title for video_id: {video_id}")
+        logger.exception(e)
+        title = ""
+
+    if engine == "pytubefix":
+        formatted_content, transcript_raw = extract_transcript_pytubefix(
+            state.url, languages
+        )
+    if engine == "transcripts-api":
+        transcript = await get_best_transcript(video_id, languages)
+
+        logger.debug(f"Found transcript: {transcript}")
+        formatter = TextFormatter()
+
+        try:
+            formatted_content = formatter.format_transcript(transcript)
+        except Exception as e:
+            logger.critical(f"Failed to format transcript for video_id: {video_id}")
+            logger.exception(e)
+            formatted_content = ""
+
+        try:
+            transcript_raw = transcript.to_raw_data()
+        except Exception as e:
+            logger.critical(f"Failed to get raw transcript for video_id: {video_id}")
+            logger.exception(e)
+            transcript_raw = ""
+
+    return {
+        "content": formatted_content,
+        "title": title,
+        "metadata": {"video_id": video_id, "transcript": transcript_raw},
+    }

{content_core-1.0.1 → content_core-1.0.3}/uv.lock

@@ -410,7 +410,7 @@ wheels = [
 
 [[package]]
 name = "content-core"
-version = "1.0.1"
+version = "1.0.3"
 source = { editable = "." }
 dependencies = [
     { name = "ai-prompter" },
@@ -419,7 +419,7 @@ dependencies = [
     { name = "bs4" },
     { name = "dicttoxml" },
     { name = "docling" },
-    { name = "esperanto", extra = ["openai"] },
+    { name = "esperanto" },
     { name = "firecrawl-py" },
     { name = "jinja2" },
     { name = "langdetect" },
@@ -433,7 +433,9 @@ dependencies = [
     { name = "python-docx" },
     { name = "python-dotenv" },
     { name = "python-magic" },
+    { name = "python-magic-bin", marker = "(platform_machine != 'aarch64' and platform_system == 'Linux' and sys_platform == 'win32') or (platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'win32')" },
     { name = "python-pptx" },
+    { name = "pytubefix" },
     { name = "readability-lxml" },
     { name = "validators" },
     { name = "youtube-transcript-api" },
@@ -457,7 +459,7 @@ requires-dist = [
     { name = "bs4", specifier = ">=0.0.2" },
     { name = "dicttoxml", specifier = ">=1.7.16" },
     { name = "docling", specifier = ">=2.34.0" },
-    { name = "esperanto", extras = ["openai"], specifier = ">=1.2.0" },
+    { name = "esperanto", specifier = ">=1.2.0" },
     { name = "firecrawl-py", specifier = ">=2.7.0" },
     { name = "jinja2", specifier = ">=3.1.6" },
     { name = "langdetect", specifier = ">=1.0.9" },
@@ -471,7 +473,9 @@ requires-dist = [
     { name = "python-docx", specifier = ">=1.1.2" },
     { name = "python-dotenv", specifier = ">=1.1.0" },
     { name = "python-magic", specifier = ">=0.4.27" },
+    { name = "python-magic-bin", marker = "sys_platform == 'win32'", specifier = "==0.4.14" },
     { name = "python-pptx", specifier = ">=1.0.2" },
+    { name = "pytubefix", specifier = ">=9.1.1" },
     { name = "readability-lxml", specifier = ">=0.8.4.1" },
     { name = "validators", specifier = ">=0.34.0" },
     { name = "youtube-transcript-api", specifier = ">=1.0.3" },
@@ -729,11 +733,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f9/79/5d74f2b8f9d73da83bfe80a39ff11505a2a285c03a869750db98cd89ddfd/esperanto-1.2.1-py3-none-any.whl", hash = "sha256:2fa41e5e35c847b1fe58395906d8877035f7e55d6429870d897781f7c9f17c42", size = 57680 },
 ]
 
-[package.optional-dependencies]
-openai = [
-    { name = "openai" },
-]
-
 [[package]]
 name = "et-xmlfile"
 version = "2.0.0"
@@ -2919,6 +2918,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3", size = 13840 },
 ]
 
+[[package]]
+name = "python-magic-bin"
+version = "0.4.14"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5a/5d/10b9ac745d9fd2f7151a2ab901e6bb6983dbd70e87c71111f54859d1ca2e/python_magic_bin-0.4.14-py2.py3-none-win32.whl", hash = "sha256:34a788c03adde7608028203e2dbb208f1f62225ad91518787ae26d603ae68892", size = 397784 },
+    { url = "https://files.pythonhosted.org/packages/07/c2/094e3d62b906d952537196603a23aec4bcd7c6126bf80eb14e6f9f4be3a2/python_magic_bin-0.4.14-py2.py3-none-win_amd64.whl", hash = "sha256:90be6206ad31071a36065a2fc169c5afb5e0355cbe6030e87641c6c62edc2b69", size = 409299 },
+]
+
 [[package]]
 name = "python-pptx"
 version = "1.0.2"
@@ -2934,6 +2942,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788 },
 ]
 
+[[package]]
+name = "pytubefix"
+version = "9.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3c/06/8570fb8fc1296dae7f156e4de57c2b351856e7813873178e1cbb8045eef3/pytubefix-9.1.1.tar.gz", hash = "sha256:68946ab2192d7bb9d8fcc0fe73f634bb0ab0cd33f2c3c718e65c0c4fbdbccbb1", size = 734325 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5d/fd/80ba35c78cbd007bfdb71d83b64087cca10e671cae4eb77875c952a21734/pytubefix-9.1.1-py3-none-any.whl", hash = "sha256:cc1c9cca936b82fcbf136e4630639417072aa5fdacf54ec0426604ca81c33b77", size = 732005 },
+]
+
 [[package]]
 name = "pytz"
 version = "2025.2"

content_core-1.0.1/src/content_core/processors/youtube.py

@@ -1,195 +0,0 @@
-import asyncio
-import re
-import ssl
-
-import aiohttp
-from bs4 import BeautifulSoup
-from youtube_transcript_api import YouTubeTranscriptApi  # type: ignore
-from youtube_transcript_api.formatters import TextFormatter  # type: ignore
-
-from content_core.common import ProcessSourceState
-from content_core.common.exceptions import NoTranscriptFound
-from content_core.config import CONFIG
-from content_core.logging import logger
-
-ssl._create_default_https_context = ssl._create_unverified_context
-
-
-async def get_video_title(video_id):
-    try:
-        url = f"https://www.youtube.com/watch?v={video_id}"
-        async with aiohttp.ClientSession() as session:
-            async with session.get(url) as response:
-                html = await response.text()
-
-        # BeautifulSoup doesn't support async operations
-        soup = BeautifulSoup(html, "html.parser")
-
-        # YouTube stores title in a meta tag
-        title = soup.find("meta", property="og:title")["content"]
-        return title
-
-    except Exception as e:
-        logger.error(f"Failed to get video title: {e}")
-        return None
-
-
-async def _extract_youtube_id(url):
-    """
-    Extract the YouTube video ID from a given URL using regular expressions.
-
-    Args:
-    url (str): The YouTube URL from which to extract the video ID.
-
-    Returns:
-    str: The extracted YouTube video ID or None if no valid ID is found.
-    """
-    # Define a regular expression pattern to capture the YouTube video ID
-    youtube_regex = (
-        r"(?:https?://)?"  # Optional scheme
-        r"(?:www\.)?"  # Optional www.
-        r"(?:"
-        r"youtu\.be/"  # Shortened URL
-        r"|youtube\.com"  # Main URL
-        r"(?:"  # Group start
-        r"/embed/"  # Embed URL
-        r"|/v/"  # Older video URL
-        r"|/watch\?v="  # Standard watch URL
-        r"|/watch\?.+&v="  # Other watch URL
-        r")"  # Group end
-        r")"  # End main group
-        r"([\w-]{11})"  # 11 characters (YouTube video ID)
-    )
-
-    # Search the URL for the pattern
-    match = re.search(youtube_regex, url)
-
-    # Return the video ID if a match is found
-    return match.group(1) if match else None
-
-
-async def get_best_transcript(video_id, preferred_langs=["en", "es", "pt"]):
-    max_attempts = 5
-    for attempt in range(max_attempts):
-        try:
-            transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
-
-            # First try: Manual transcripts in preferred languages
-            manual_transcripts = []
-            try:
-                for transcript in transcript_list:
-                    if not transcript.is_generated and not transcript.is_translatable:
-                        manual_transcripts.append(transcript)
-
-                if manual_transcripts:
-                    # Sort based on preferred language order
-                    for lang in preferred_langs:
-                        for transcript in manual_transcripts:
-                            if transcript.language_code == lang:
-                                return transcript.fetch()
-                    # If no preferred language found, return first manual transcript
-                    return manual_transcripts[0].fetch()
-            except NoTranscriptFound:
-                pass
-
-            # Second try: Auto-generated transcripts in preferred languages
-            generated_transcripts = []
-            try:
-                for transcript in transcript_list:
-                    if transcript.is_generated and not transcript.is_translatable:
-                        generated_transcripts.append(transcript)
-
-                if generated_transcripts:
-                    # Sort based on preferred language order
-                    for lang in preferred_langs:
-                        for transcript in generated_transcripts:
-                            if transcript.language_code == lang:
-                                return transcript.fetch()
-                    # If no preferred language found, return first generated transcript
-                    return generated_transcripts[0].fetch()
-            except NoTranscriptFound:
-                pass
-
-            # Last try: Translated transcripts in preferred languages
-            translated_transcripts = []
-            try:
-                for transcript in transcript_list:
-                    if transcript.is_translatable:
-                        translated_transcripts.append(transcript)
-
-                if translated_transcripts:
-                    # Sort based on preferred language order
-                    for lang in preferred_langs:
-                        for transcript in translated_transcripts:
-                            if transcript.language_code == lang:
-                                return transcript.fetch()
-                    # If no preferred language found, return translation to first preferred language
-                    translation = translated_transcripts[0].translate(
-                        preferred_langs[0]
-                    )
-                    return translation.fetch()
-            except NoTranscriptFound:
-                pass
-
-            raise Exception("No suitable transcript found")
-
-        except Exception as e:
-            if e.__class__.__name__ == "ParserError":
-                logger.warning(
-                    f"ParserError on attempt {attempt+1}/5 for video {video_id}. Retrying..."
-                )
-                if attempt == max_attempts - 1:
-                    logger.error(
-                        f"Failed to get transcript for video {video_id} after {max_attempts} attempts due to repeated ParserError."
-                    )
-                    return None
-                await asyncio.sleep(2)
-                continue
-            else:
-                logger.error(f"Failed to get transcript for video {video_id}: {e}")
-                return None
-    return None
-
-
-async def extract_youtube_transcript(state: ProcessSourceState):
-    """
-    Parse the text file and print its content.
-    """
-
-    assert state.url, "No URL provided"
-    logger.warning(f"Extracting transcript from URL: {state.url}")
-    languages = CONFIG.get("youtube_transcripts", {}).get(
-        "preferred_languages", ["en", "es", "pt"]
-    )
-
-    video_id = await _extract_youtube_id(state.url)
-    transcript = await get_best_transcript(video_id, languages)
-
-    logger.debug(f"Found transcript: {transcript}")
-    formatter = TextFormatter()
-    try:
-        title = await get_video_title(video_id)
-    except Exception as e:
-        logger.critical(f"Failed to get video title for video_id: {video_id}")
-        logger.exception(e)
-        title = ""
-
-    try:
-        formatted_content = formatter.format_transcript(transcript)
-    except Exception as e:
-        logger.critical(f"Failed to format transcript for video_id: {video_id}")
-        logger.exception(e)
-        formatted_content = ""
-
-    try:
-        transcript_raw = transcript.to_raw_data()
-    except Exception as e:
-        logger.critical(f"Failed to get raw transcript for video_id: {video_id}")
-        logger.exception(e)
-        transcript_raw = ""
-
-    return {
-        "content": formatted_content,
-        "title": title,
-        "metadata": {"video_id": video_id, "transcript": transcript_raw},
-    }