lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/youtube/video_details.py

@@ -0,0 +1,263 @@
+from contextlib import contextmanager
+
+import googleapiclient
+import pandas as pd
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import BoolInput, MessageTextInput, SecretStrInput
+from lfx.schema.dataframe import DataFrame
+from lfx.template.field.base import Output
+
+
+class YouTubeVideoDetailsComponent(Component):
+    """A component that retrieves detailed information about YouTube videos."""
+
+    display_name: str = "YouTube Video Details"
+    description: str = "Retrieves detailed information and statistics about YouTube videos."
+    icon: str = "YouTube"
+
+    inputs = [
+        MessageTextInput(
+            name="video_url",
+            display_name="Video URL",
+            info="The URL of the YouTube video.",
+            tool_mode=True,
+            required=True,
+        ),
+        SecretStrInput(
+            name="api_key",
+            display_name="YouTube API Key",
+            info="Your YouTube Data API key.",
+            required=True,
+        ),
+        BoolInput(
+            name="include_statistics",
+            display_name="Include Statistics",
+            value=True,
+            info="Include video statistics (views, likes, comments).",
+        ),
+        BoolInput(
+            name="include_content_details",
+            display_name="Include Content Details",
+            value=True,
+            info="Include video duration, quality, and age restriction info.",
+            advanced=True,
+        ),
+        BoolInput(
+            name="include_tags",
+            display_name="Include Tags",
+            value=True,
+            info="Include video tags and keywords.",
+            advanced=True,
+        ),
+        BoolInput(
+            name="include_thumbnails",
+            display_name="Include Thumbnails",
+            value=True,
+            info="Include video thumbnail URLs in different resolutions.",
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(name="video_data", display_name="Video Data", method="get_video_details"),
+    ]
+
+    API_FORBIDDEN = 403
+    VIDEO_NOT_FOUND = 404
+
+    @contextmanager
+    def youtube_client(self):
+        """Context manager for YouTube API client."""
+        client = build("youtube", "v3", developerKey=self.api_key)
+        try:
+            yield client
+        finally:
+            client.close()
+
+    def _extract_video_id(self, video_url: str) -> str:
+        """Extracts the video ID from a YouTube URL."""
+        import re
+
+        patterns = [
+            r"(?:youtube\.com\/watch\?v=|youtu.be\/|youtube.com\/embed\/)([^&\n?#]+)",
+            r"youtube.com\/shorts\/([^&\n?#]+)",
+        ]
+
+        for pattern in patterns:
+            match = re.search(pattern, video_url)
+            if match:
+                return match.group(1)
+
+        return video_url.strip()
+
+    def _format_duration(self, duration: str) -> str:
+        """Formats the ISO 8601 duration to a readable format."""
+        import re
+
+        hours = 0
+        minutes = 0
+        seconds = 0
+
+        hours_match = re.search(r"(\d+)H", duration)
+        minutes_match = re.search(r"(\d+)M", duration)
+        seconds_match = re.search(r"(\d+)S", duration)
+
+        if hours_match:
+            hours = int(hours_match.group(1))
+        if minutes_match:
+            minutes = int(minutes_match.group(1))
+        if seconds_match:
+            seconds = int(seconds_match.group(1))
+
+        if hours > 0:
+            return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+        return f"{minutes:02d}:{seconds:02d}"
+
+    def get_video_details(self) -> DataFrame:
+        """Retrieves detailed information about a YouTube video and returns as DataFrame."""
+        try:
+            with self.youtube_client() as youtube:
+                # Extract video ID
+                video_id = self._extract_video_id(self.video_url)
+
+                # Prepare parts for the API request
+                parts = ["snippet"]
+                if self.include_statistics:
+                    parts.append("statistics")
+                if self.include_content_details:
+                    parts.append("contentDetails")
+
+                # Get video information
+                video_response = youtube.videos().list(part=",".join(parts), id=video_id).execute()
+
+                if not video_response["items"]:
+                    return DataFrame(pd.DataFrame({"error": ["Video not found"]}))
+
+                video_info = video_response["items"][0]
+                snippet = video_info["snippet"]
+
+                # Build video data dictionary
+                video_data = {
+                    "video_id": [video_id],
+                    "url": [f"https://www.youtube.com/watch?v={video_id}"],
+                    "title": [snippet["title"]],
+                    "description": [snippet["description"]],
+                    "published_at": [snippet["publishedAt"]],
+                    "channel_id": [snippet["channelId"]],
+                    "channel_title": [snippet["channelTitle"]],
+                    "category_id": [snippet.get("categoryId", "Unknown")],
+                    "live_broadcast_content": [snippet.get("liveBroadcastContent", "none")],
+                }
+
+                # Add thumbnails if requested
+                if self.include_thumbnails:
+                    for size, thumb in snippet["thumbnails"].items():
+                        video_data[f"thumbnail_{size}_url"] = [thumb["url"]]
+                        video_data[f"thumbnail_{size}_width"] = [thumb.get("width", 0)]
+                        video_data[f"thumbnail_{size}_height"] = [thumb.get("height", 0)]
+
+                # Add tags if requested
+                if self.include_tags and "tags" in snippet:
+                    video_data["tags"] = [", ".join(snippet["tags"])]
+                    video_data["tags_count"] = [len(snippet["tags"])]
+
+                # Add statistics if requested
+                if self.include_statistics and "statistics" in video_info:
+                    stats = video_info["statistics"]
+                    video_data.update(
+                        {
+                            "view_count": [int(stats.get("viewCount", 0))],
+                            "like_count": [int(stats.get("likeCount", 0))],
+                            "favorite_count": [int(stats.get("favoriteCount", 0))],
+                            "comment_count": [int(stats.get("commentCount", 0))],
+                        }
+                    )
+
+                # Add content details if requested
+                if self.include_content_details and "contentDetails" in video_info:
+                    content_details = video_info["contentDetails"]
+                    video_data.update(
+                        {
+                            "duration": [self._format_duration(content_details["duration"])],
+                            "dimension": [content_details.get("dimension", "2d")],
+                            "definition": [content_details.get("definition", "hd").upper()],
+                            "has_captions": [content_details.get("caption", "false") == "true"],
+                            "licensed_content": [content_details.get("licensedContent", False)],
+                            "projection": [content_details.get("projection", "rectangular")],
+                            "has_custom_thumbnails": [content_details.get("hasCustomThumbnail", False)],
+                        }
+                    )
+
+                    # Add content rating if available
+                    if "contentRating" in content_details:
+                        rating_info = content_details["contentRating"]
+                        video_data["content_rating"] = [str(rating_info)]
+
+                # Create DataFrame with organized columns
+                video_df = pd.DataFrame(video_data)
+
+                # Organize columns in logical groups
+                basic_cols = [
+                    "video_id",
+                    "title",
+                    "url",
+                    "channel_id",
+                    "channel_title",
+                    "published_at",
+                    "category_id",
+                    "live_broadcast_content",
+                    "description",
+                ]
+
+                stat_cols = ["view_count", "like_count", "favorite_count", "comment_count"]
+
+                content_cols = [
+                    "duration",
+                    "dimension",
+                    "definition",
+                    "has_captions",
+                    "licensed_content",
+                    "projection",
+                    "has_custom_thumbnails",
+                    "content_rating",
+                ]
+
+                tag_cols = ["tags", "tags_count"]
+
+                thumb_cols = [col for col in video_df.columns if col.startswith("thumbnail_")]
+
+                # Reorder columns based on what's included
+                ordered_cols = basic_cols.copy()
+
+                if self.include_statistics:
+                    ordered_cols.extend([col for col in stat_cols if col in video_df.columns])
+
+                if self.include_content_details:
+                    ordered_cols.extend([col for col in content_cols if col in video_df.columns])
+
+                if self.include_tags:
+                    ordered_cols.extend([col for col in tag_cols if col in video_df.columns])
+
+                if self.include_thumbnails:
+                    ordered_cols.extend(sorted(thumb_cols))
+
+                # Add any remaining columns
+                remaining_cols = [col for col in video_df.columns if col not in ordered_cols]
+                ordered_cols.extend(remaining_cols)
+
+                return DataFrame(video_df[ordered_cols])
+
+        except (HttpError, googleapiclient.errors.HttpError) as e:
+            error_message = f"YouTube API error: {e!s}"
+            if e.resp.status == self.API_FORBIDDEN:
+                error_message = "API quota exceeded or access forbidden."
+            elif e.resp.status == self.VIDEO_NOT_FOUND:
+                error_message = "Video not found."
+
+            return DataFrame(pd.DataFrame({"error": [error_message]}))
+
+        except KeyError as e:
+            return DataFrame(pd.DataFrame({"error": [str(e)]}))

lfx/components/youtube/youtube_transcripts.py

@@ -0,0 +1,206 @@
+import re
+
+import pandas as pd
+from youtube_transcript_api import NoTranscriptFound, TranscriptsDisabled, YouTubeTranscriptApi
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import DropdownInput, IntInput, MultilineInput
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.schema.message import Message
+from lfx.template.field.base import Output
+
+
+class YouTubeTranscriptsComponent(Component):
+    """A component that extracts spoken content from YouTube videos as transcripts."""
+
+    display_name: str = "YouTube Transcripts"
+    description: str = "Extracts spoken content from YouTube videos with multiple output options."
+    icon: str = "YouTube"
+    name = "YouTubeTranscripts"
+
+    inputs = [
+        MultilineInput(
+            name="url",
+            display_name="Video URL",
+            info="Enter the YouTube video URL to get transcripts from.",
+            tool_mode=True,
+            required=True,
+        ),
+        IntInput(
+            name="chunk_size_seconds",
+            display_name="Chunk Size (seconds)",
+            value=60,
+            info="The size of each transcript chunk in seconds.",
+        ),
+        DropdownInput(
+            name="translation",
+            display_name="Translation Language",
+            advanced=True,
+            options=["", "en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "hi", "ar", "id"],
+            info="Translate the transcripts to the specified language. Leave empty for no translation.",
+        ),
+    ]
+
+    outputs = [
+        Output(name="dataframe", display_name="Chunks", method="get_dataframe_output"),
+        Output(name="message", display_name="Transcript", method="get_message_output"),
+        Output(name="data_output", display_name="Transcript + Source", method="get_data_output"),
+    ]
+
+    def _extract_video_id(self, url: str) -> str:
+        """Extract video ID from YouTube URL."""
+        patterns = [
+            r"(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/)([^&\n?#]+)",
+            r"youtube\.com\/watch\?.*?v=([^&\n?#]+)",
+        ]
+        for pattern in patterns:
+            match = re.search(pattern, url)
+            if match:
+                return match.group(1)
+        msg = f"Could not extract video ID from URL: {url}"
+        raise ValueError(msg)
+
+    def _load_transcripts(self, *, as_chunks: bool = True):
+        """Internal method to load transcripts from YouTube."""
+        try:
+            video_id = self._extract_video_id(self.url)
+        except ValueError as e:
+            msg = f"Invalid YouTube URL: {e}"
+            raise ValueError(msg) from e
+
+        try:
+            # Use new v1.0+ API - create instance
+            api = YouTubeTranscriptApi()
+            transcript_list = api.list(video_id)
+
+            # Get transcript in specified language or default to English
+            if self.translation:
+                # Get any available transcript and translate it
+                transcript = transcript_list.find_transcript(["en"])
+                transcript = transcript.translate(self.translation)
+            else:
+                # Try to get transcript in available languages
+                try:
+                    transcript = transcript_list.find_transcript(["en"])
+                except NoTranscriptFound:
+                    # Try auto-generated English
+                    transcript = transcript_list.find_generated_transcript(["en"])
+
+            # Fetch the transcript data
+            transcript_data = api.fetch(transcript.video_id, [transcript.language_code])
+
+        except (TranscriptsDisabled, NoTranscriptFound) as e:
+            error_type = type(e).__name__
+            msg = (
+                f"Could not retrieve transcripts for video '{video_id}'. "
+                "Possible reasons:\n"
+                "1. This video does not have captions/transcripts enabled\n"
+                "2. The video is private, restricted, or deleted\n"
+                f"\nTechnical error ({error_type}): {e}"
+            )
+            raise RuntimeError(msg) from e
+        except Exception as e:
+            error_type = type(e).__name__
+            msg = (
+                f"Could not retrieve transcripts for video '{video_id}'. "
+                "Possible reasons:\n"
+                "1. This video does not have captions/transcripts enabled\n"
+                "2. The video is private, restricted, or deleted\n"
+                "3. YouTube is blocking automated requests\n"
+                f"\nTechnical error ({error_type}): {e}"
+            )
+            raise RuntimeError(msg) from e
+
+        if as_chunks:
+            # Group into chunks based on chunk_size_seconds
+            return self._chunk_transcript(transcript_data)
+        # Return as continuous text
+        return transcript_data
+
+    def _chunk_transcript(self, transcript_data):
+        """Group transcript segments into time-based chunks."""
+        chunks = []
+        current_chunk = []
+        chunk_start = 0
+
+        for segment in transcript_data:
+            # Handle both dict (old API) and object (new API) formats
+            segment_start = segment.start if hasattr(segment, "start") else segment["start"]
+
+            # If this segment starts beyond the current chunk window, start a new chunk
+            if segment_start - chunk_start >= self.chunk_size_seconds and current_chunk:
+                chunk_text = " ".join(s.text if hasattr(s, "text") else s["text"] for s in current_chunk)
+                chunks.append({"start": chunk_start, "text": chunk_text})
+                current_chunk = []
+                chunk_start = segment_start
+
+            current_chunk.append(segment)
+
+        # Add the last chunk
+        if current_chunk:
+            chunk_text = " ".join(s.text if hasattr(s, "text") else s["text"] for s in current_chunk)
+            chunks.append({"start": chunk_start, "text": chunk_text})
+
+        return chunks
+
+    def get_dataframe_output(self) -> DataFrame:
+        """Provides transcript output as a DataFrame with timestamp and text columns."""
+        try:
+            chunks = self._load_transcripts(as_chunks=True)
+
+            # Create DataFrame with timestamp and text columns
+            data = []
+            for chunk in chunks:
+                start_seconds = int(chunk["start"])
+                start_minutes = start_seconds // 60
+                start_seconds_remainder = start_seconds % 60
+                timestamp = f"{start_minutes:02d}:{start_seconds_remainder:02d}"
+                data.append({"timestamp": timestamp, "text": chunk["text"]})
+
+            return DataFrame(pd.DataFrame(data))
+
+        except (TranscriptsDisabled, NoTranscriptFound, RuntimeError, ValueError) as exc:
+            error_msg = f"Failed to get YouTube transcripts: {exc!s}"
+            return DataFrame(pd.DataFrame({"error": [error_msg]}))
+
+    def get_message_output(self) -> Message:
+        """Provides transcript output as continuous text."""
+        try:
+            transcript_data = self._load_transcripts(as_chunks=False)
+            # Handle both dict (old API) and object (new API) formats
+            result = " ".join(
+                segment.text if hasattr(segment, "text") else segment["text"] for segment in transcript_data
+            )
+            return Message(text=result)
+
+        except (TranscriptsDisabled, NoTranscriptFound, RuntimeError, ValueError) as exc:
+            error_msg = f"Failed to get YouTube transcripts: {exc!s}"
+            return Message(text=error_msg)
+
+    def get_data_output(self) -> Data:
+        """Creates a structured data object with transcript and metadata.
+
+        Returns a Data object containing transcript text, video URL, and any error
+        messages that occurred during processing. The object includes:
+        - 'transcript': continuous text from the entire video (concatenated if multiple parts)
+        - 'video_url': the input YouTube URL
+        - 'error': error message if an exception occurs
+        """
+        default_data = {"transcript": "", "video_url": self.url, "error": None}
+
+        try:
+            transcript_data = self._load_transcripts(as_chunks=False)
+            if not transcript_data:
+                default_data["error"] = "No transcripts found."
+                return Data(data=default_data)
+
+            # Combine all transcript segments - handle both dict and object formats
+            full_transcript = " ".join(
+                segment.text if hasattr(segment, "text") else segment["text"] for segment in transcript_data
+            )
+            return Data(data={"transcript": full_transcript, "video_url": self.url})
+
+        except (TranscriptsDisabled, NoTranscriptFound, RuntimeError, ValueError) as exc:
+            default_data["error"] = str(exc)
+            return Data(data=default_data)

lfx/components/zep/__init__.py

@@ -0,0 +1,3 @@
+from .zep import ZepChatMemory
+
+__all__ = ["ZepChatMemory"]

lfx/components/zep/zep.py

@@ -0,0 +1,45 @@
+from lfx.base.memory.model import LCChatMemoryComponent
+from lfx.field_typing.constants import Memory
+from lfx.inputs.inputs import DropdownInput, MessageTextInput, SecretStrInput
+
+
+class ZepChatMemory(LCChatMemoryComponent):
+    display_name = "Zep Chat Memory"
+    description = "Retrieves and store chat messages from Zep."
+    name = "ZepChatMemory"
+    icon = "ZepMemory"
+    legacy = True
+    replacement = ["helpers.Memory"]
+
+    inputs = [
+        MessageTextInput(name="url", display_name="Zep URL", info="URL of the Zep instance."),
+        SecretStrInput(name="api_key", display_name="Zep API Key", info="API Key for the Zep instance."),
+        DropdownInput(
+            name="api_base_path",
+            display_name="API Base Path",
+            options=["api/v1", "api/v2"],
+            value="api/v1",
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="session_id", display_name="Session ID", info="Session ID for the message.", advanced=True
+        ),
+    ]
+
+    def build_message_history(self) -> Memory:
+        try:
+            # Monkeypatch API_BASE_PATH to
+            # avoid 404
+            # This is a workaround for the local Zep instance
+            # cloud Zep works with v2
+            import zep_python.zep_client
+            from zep_python import ZepClient
+            from zep_python.langchain import ZepChatMessageHistory
+
+            zep_python.zep_client.API_BASE_PATH = self.api_base_path
+        except ImportError as e:
+            msg = "Could not import zep-python package. Please install it with `pip install zep-python`."
+            raise ImportError(msg) from e
+
+        zep_client = ZepClient(api_url=self.url, api_key=self.api_key)
+        return ZepChatMessageHistory(session_id=self.session_id, zep_client=zep_client)

lfx/constants.py

@@ -0,0 +1,6 @@
+"""Constants for lfx package."""
+
+from pathlib import Path
+
+# Base path for components - will be in lfx package when components are moved
+BASE_COMPONENTS_PATH = str(Path(__file__).parent / "components")

lfx/custom/__init__.py

@@ -0,0 +1,7 @@
+from lfx.custom.custom_component.component import Component
+from lfx.custom.custom_component.custom_component import CustomComponent
+
+from . import custom_component as custom_component
+from . import utils as utils
+
+__all__ = ["Component", "CustomComponent", "custom_component", "utils"]

lfx/custom/attributes.py

@@ -0,0 +1,87 @@
+from collections.abc import Callable
+
+import emoji
+
+from lfx.log.logger import logger
+
+
+def validate_icon(value: str):
+    # we are going to use the emoji library to validate the emoji
+    # emojis can be defined using the :emoji_name: syntax
+
+    if not value.startswith(":") and not value.endswith(":"):
+        return value
+    if not value.startswith(":") or not value.endswith(":"):
+        # emoji should have both starting and ending colons
+        # so if one of them is missing, we will raise
+        msg = f"Invalid emoji. {value} is not a valid emoji."
+        raise ValueError(msg)
+
+    emoji_value = emoji.emojize(value, variant="emoji_type")
+    if value == emoji_value:
+        logger.warning(f"Invalid emoji. {value} is not a valid emoji.")
+        return value
+    return emoji_value
+
+
+def getattr_return_str(value):
+    return str(value) if value else ""
+
+
+def getattr_return_bool(value):
+    if isinstance(value, bool):
+        return value
+    return None
+
+
+def getattr_return_int(value):
+    if isinstance(value, int):
+        return value
+    return None
+
+
+def getattr_return_list_of_str(value):
+    if isinstance(value, list):
+        return [str(val) for val in value]
+    return []
+
+
+def getattr_return_list_of_object(value):
+    if isinstance(value, list):
+        return value
+    return []
+
+
+def getattr_return_list_of_values_from_dict(value):
+    if isinstance(value, dict):
+        return list(value.values())
+    return []
+
+
+def getattr_return_dict(value):
+    if isinstance(value, dict):
+        return value
+    return {}
+
+
+ATTR_FUNC_MAPPING: dict[str, Callable] = {
+    "display_name": getattr_return_str,
+    "description": getattr_return_str,
+    "beta": getattr_return_bool,
+    "legacy": getattr_return_bool,
+    "replacement": getattr_return_list_of_str,
+    "documentation": getattr_return_str,
+    "priority": getattr_return_int,
+    "icon": validate_icon,
+    "minimized": getattr_return_bool,
+    "frozen": getattr_return_bool,
+    "is_input": getattr_return_bool,
+    "is_output": getattr_return_bool,
+    "conditional_paths": getattr_return_list_of_str,
+    "_outputs_map": getattr_return_list_of_values_from_dict,
+    "_inputs": getattr_return_list_of_values_from_dict,
+    "outputs": getattr_return_list_of_object,
+    "inputs": getattr_return_list_of_object,
+    "metadata": getattr_return_dict,
+    "tool_mode": getattr_return_bool,
+}

lfx/custom/code_parser/__init__.py

@@ -0,0 +1,3 @@
+from .code_parser import CodeParser
+
+__all__ = ["CodeParser"]