lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/youtube/comments.py

@@ -0,0 +1,231 @@
+from contextlib import contextmanager
+
+import pandas as pd
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import BoolInput, DropdownInput, IntInput, MessageTextInput, SecretStrInput
+from lfx.schema.dataframe import DataFrame
+from lfx.template.field.base import Output
+
+
+class YouTubeCommentsComponent(Component):
+    """A component that retrieves comments from YouTube videos."""
+
+    display_name: str = "YouTube Comments"
+    description: str = "Retrieves and analyzes comments from YouTube videos."
+    icon: str = "YouTube"
+
+    # Constants
+    COMMENTS_DISABLED_STATUS = 403
+    NOT_FOUND_STATUS = 404
+    API_MAX_RESULTS = 100
+
+    inputs = [
+        MessageTextInput(
+            name="video_url",
+            display_name="Video URL",
+            info="The URL of the YouTube video to get comments from.",
+            tool_mode=True,
+            required=True,
+        ),
+        SecretStrInput(
+            name="api_key",
+            display_name="YouTube API Key",
+            info="Your YouTube Data API key.",
+            required=True,
+        ),
+        IntInput(
+            name="max_results",
+            display_name="Max Results",
+            value=20,
+            info="The maximum number of comments to return.",
+        ),
+        DropdownInput(
+            name="sort_by",
+            display_name="Sort By",
+            options=["time", "relevance"],
+            value="relevance",
+            info="Sort comments by time or relevance.",
+        ),
+        BoolInput(
+            name="include_replies",
+            display_name="Include Replies",
+            value=False,
+            info="Whether to include replies to comments.",
+            advanced=True,
+        ),
+        BoolInput(
+            name="include_metrics",
+            display_name="Include Metrics",
+            value=True,
+            info="Include metrics like like count and reply count.",
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(name="comments", display_name="Comments", method="get_video_comments"),
+    ]
+
+    def _extract_video_id(self, video_url: str) -> str:
+        """Extracts the video ID from a YouTube URL."""
+        import re
+
+        patterns = [
+            r"(?:youtube\.com\/watch\?v=|youtu.be\/|youtube.com\/embed\/)([^&\n?#]+)",
+            r"youtube.com\/shorts\/([^&\n?#]+)",
+        ]
+
+        for pattern in patterns:
+            match = re.search(pattern, video_url)
+            if match:
+                return match.group(1)
+
+        return video_url.strip()
+
+    def _process_reply(self, reply: dict, parent_id: str, *, include_metrics: bool = True) -> dict:
+        """Process a single reply comment."""
+        reply_snippet = reply["snippet"]
+        reply_data = {
+            "comment_id": reply["id"],
+            "parent_comment_id": parent_id,
+            "author": reply_snippet["authorDisplayName"],
+            "text": reply_snippet["textDisplay"],
+            "published_at": reply_snippet["publishedAt"],
+            "is_reply": True,
+        }
+        if include_metrics:
+            reply_data["like_count"] = reply_snippet["likeCount"]
+            reply_data["reply_count"] = 0  # Replies can't have replies
+
+        return reply_data
+
+    def _process_comment(
+        self, item: dict, *, include_metrics: bool = True, include_replies: bool = False
+    ) -> list[dict]:
+        """Process a single comment thread."""
+        comment = item["snippet"]["topLevelComment"]["snippet"]
+        comment_id = item["snippet"]["topLevelComment"]["id"]
+
+        # Basic comment data
+        processed_comments = [
+            {
+                "comment_id": comment_id,
+                "parent_comment_id": "",  # Empty for top-level comments
+                "author": comment["authorDisplayName"],
+                "author_channel_url": comment.get("authorChannelUrl", ""),
+                "text": comment["textDisplay"],
+                "published_at": comment["publishedAt"],
+                "updated_at": comment["updatedAt"],
+                "is_reply": False,
+            }
+        ]
+
+        # Add metrics if requested
+        if include_metrics:
+            processed_comments[0].update(
+                {
+                    "like_count": comment["likeCount"],
+                    "reply_count": item["snippet"]["totalReplyCount"],
+                }
+            )
+
+        # Add replies if requested
+        if include_replies and item["snippet"]["totalReplyCount"] > 0 and "replies" in item:
+            for reply in item["replies"]["comments"]:
+                reply_data = self._process_reply(reply, parent_id=comment_id, include_metrics=include_metrics)
+                processed_comments.append(reply_data)
+
+        return processed_comments
+
+    @contextmanager
+    def youtube_client(self):
+        """Context manager for YouTube API client."""
+        client = build("youtube", "v3", developerKey=self.api_key)
+        try:
+            yield client
+        finally:
+            client.close()
+
+    def get_video_comments(self) -> DataFrame:
+        """Retrieves comments from a YouTube video and returns as DataFrame."""
+        try:
+            # Extract video ID from URL
+            video_id = self._extract_video_id(self.video_url)
+
+            # Use context manager for YouTube API client
+            with self.youtube_client() as youtube:
+                comments_data = []
+                results_count = 0
+                request = youtube.commentThreads().list(
+                    part="snippet,replies",
+                    videoId=video_id,
+                    maxResults=min(self.API_MAX_RESULTS, self.max_results),
+                    order=self.sort_by,
+                    textFormat="plainText",
+                )
+
+                while request and results_count < self.max_results:
+                    response = request.execute()
+
+                    for item in response.get("items", []):
+                        if results_count >= self.max_results:
+                            break
+
+                        comments = self._process_comment(
+                            item, include_metrics=self.include_metrics, include_replies=self.include_replies
+                        )
+                        comments_data.extend(comments)
+                        results_count += 1
+
+                    # Get the next page if available and needed
+                    if "nextPageToken" in response and results_count < self.max_results:
+                        request = youtube.commentThreads().list(
+                            part="snippet,replies",
+                            videoId=video_id,
+                            maxResults=min(self.API_MAX_RESULTS, self.max_results - results_count),
+                            order=self.sort_by,
+                            textFormat="plainText",
+                            pageToken=response["nextPageToken"],
+                        )
+                    else:
+                        request = None
+
+                # Convert to DataFrame
+                comments_df = pd.DataFrame(comments_data)
+
+                # Add video metadata
+                comments_df["video_id"] = video_id
+                comments_df["video_url"] = self.video_url
+
+                # Sort columns for better organization
+                column_order = [
+                    "video_id",
+                    "video_url",
+                    "comment_id",
+                    "parent_comment_id",
+                    "is_reply",
+                    "author",
+                    "author_channel_url",
+                    "text",
+                    "published_at",
+                    "updated_at",
+                ]
+
+                if self.include_metrics:
+                    column_order.extend(["like_count", "reply_count"])
+
+                comments_df = comments_df[column_order]
+
+                return DataFrame(comments_df)
+
+        except HttpError as e:
+            error_message = f"YouTube API error: {e!s}"
+            if e.resp.status == self.COMMENTS_DISABLED_STATUS:
+                error_message = "Comments are disabled for this video or API quota exceeded."
+            elif e.resp.status == self.NOT_FOUND_STATUS:
+                error_message = "Video not found."
+
+            return DataFrame(pd.DataFrame({"error": [error_message]}))

lfx/components/youtube/playlist.py

@@ -0,0 +1,33 @@
+from pytube import Playlist  # Ensure you have pytube installed
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import MessageTextInput
+from lfx.schema.data import Data
+from lfx.schema.dataframe import DataFrame
+from lfx.template.field.base import Output
+
+
+class YouTubePlaylistComponent(Component):
+    display_name = "YouTube Playlist"
+    description = "Extracts all video URLs from a YouTube playlist."
+    icon = "YouTube"  # Replace with a suitable icon
+
+    inputs = [
+        MessageTextInput(
+            name="playlist_url",
+            display_name="Playlist URL",
+            info="URL of the YouTube playlist.",
+            required=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Video URLs", name="video_urls", method="extract_video_urls"),
+    ]
+
+    def extract_video_urls(self) -> DataFrame:
+        playlist_url = self.playlist_url
+        playlist = Playlist(playlist_url)
+        video_urls = [video.watch_url for video in playlist.videos]
+
+        return DataFrame([Data(data={"video_url": url}) for url in video_urls])

lfx/components/youtube/search.py

@@ -0,0 +1,120 @@
+from contextlib import contextmanager
+
+import pandas as pd
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import BoolInput, DropdownInput, IntInput, MessageTextInput, SecretStrInput
+from lfx.schema.dataframe import DataFrame
+from lfx.template.field.base import Output
+
+
+class YouTubeSearchComponent(Component):
+    """A component that searches YouTube videos."""
+
+    display_name: str = "YouTube Search"
+    description: str = "Searches YouTube videos based on query."
+    icon: str = "YouTube"
+
+    inputs = [
+        MessageTextInput(
+            name="query",
+            display_name="Search Query",
+            info="The search query to look for on YouTube.",
+            tool_mode=True,
+            required=True,
+        ),
+        SecretStrInput(
+            name="api_key",
+            display_name="YouTube API Key",
+            info="Your YouTube Data API key.",
+            required=True,
+        ),
+        IntInput(
+            name="max_results",
+            display_name="Max Results",
+            value=10,
+            info="The maximum number of results to return.",
+        ),
+        DropdownInput(
+            name="order",
+            display_name="Sort Order",
+            options=["relevance", "date", "rating", "title", "viewCount"],
+            value="relevance",
+            info="Sort order for the search results.",
+        ),
+        BoolInput(
+            name="include_metadata",
+            display_name="Include Metadata",
+            value=True,
+            info="Include video metadata like description and statistics.",
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(name="results", display_name="Search Results", method="search_videos"),
+    ]
+
+    @contextmanager
+    def youtube_client(self):
+        """Context manager for YouTube API client."""
+        client = build("youtube", "v3", developerKey=self.api_key)
+        try:
+            yield client
+        finally:
+            client.close()
+
+    def search_videos(self) -> DataFrame:
+        """Searches YouTube videos and returns results as DataFrame."""
+        try:
+            with self.youtube_client() as youtube:
+                search_response = (
+                    youtube.search()
+                    .list(
+                        q=self.query,
+                        part="id,snippet",
+                        maxResults=self.max_results,
+                        order=self.order,
+                        type="video",
+                    )
+                    .execute()
+                )
+
+                results = []
+                for search_result in search_response.get("items", []):
+                    video_id = search_result["id"]["videoId"]
+                    snippet = search_result["snippet"]
+
+                    result = {
+                        "video_id": video_id,
+                        "title": snippet["title"],
+                        "description": snippet["description"],
+                        "published_at": snippet["publishedAt"],
+                        "channel_title": snippet["channelTitle"],
+                        "thumbnail_url": snippet["thumbnails"]["default"]["url"],
+                    }
+
+                    if self.include_metadata:
+                        # Get video details for additional metadata
+                        video_response = youtube.videos().list(part="statistics,contentDetails", id=video_id).execute()
+
+                        if video_response.get("items"):
+                            video_details = video_response["items"][0]
+                            result.update(
+                                {
+                                    "view_count": int(video_details["statistics"]["viewCount"]),
+                                    "like_count": int(video_details["statistics"].get("likeCount", 0)),
+                                    "comment_count": int(video_details["statistics"].get("commentCount", 0)),
+                                    "duration": video_details["contentDetails"]["duration"],
+                                }
+                            )
+
+                    results.append(result)
+
+                return DataFrame(pd.DataFrame(results))
+
+        except HttpError as e:
+            error_message = f"YouTube API error: {e!s}"
+            return DataFrame(pd.DataFrame({"error": [error_message]}))

lfx/components/youtube/trending.py

@@ -0,0 +1,285 @@
+from contextlib import contextmanager
+
+import pandas as pd
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+from lfx.custom.custom_component.component import Component
+from lfx.inputs.inputs import BoolInput, DropdownInput, IntInput, SecretStrInput
+from lfx.log.logger import logger
+from lfx.schema.dataframe import DataFrame
+from lfx.template.field.base import Output
+
+HTTP_FORBIDDEN = 403
+HTTP_NOT_FOUND = 404
+MAX_API_RESULTS = 50
+
+
+class YouTubeTrendingComponent(Component):
+    """A component that retrieves trending videos from YouTube."""
+
+    display_name: str = "YouTube Trending"
+    description: str = "Retrieves trending videos from YouTube with filtering options."
+    icon: str = "YouTube"
+
+    # Dictionary of country codes and names
+    COUNTRY_CODES = {
+        "Global": "US",  # Default to US for global
+        "United States": "US",
+        "Brazil": "BR",
+        "United Kingdom": "GB",
+        "India": "IN",
+        "Japan": "JP",
+        "South Korea": "KR",
+        "Germany": "DE",
+        "France": "FR",
+        "Canada": "CA",
+        "Australia": "AU",
+        "Spain": "ES",
+        "Italy": "IT",
+        "Mexico": "MX",
+        "Russia": "RU",
+        "Netherlands": "NL",
+        "Poland": "PL",
+        "Argentina": "AR",
+    }
+
+    # Dictionary of video categories
+    VIDEO_CATEGORIES = {
+        "All": "0",
+        "Film & Animation": "1",
+        "Autos & Vehicles": "2",
+        "Music": "10",
+        "Pets & Animals": "15",
+        "Sports": "17",
+        "Travel & Events": "19",
+        "Gaming": "20",
+        "People & Blogs": "22",
+        "Comedy": "23",
+        "Entertainment": "24",
+        "News & Politics": "25",
+        "Education": "27",
+        "Science & Technology": "28",
+        "Nonprofits & Activism": "29",
+    }
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="YouTube API Key",
+            info="Your YouTube Data API key.",
+            required=True,
+        ),
+        DropdownInput(
+            name="region",
+            display_name="Region",
+            options=list(COUNTRY_CODES.keys()),
+            value="Global",
+            info="The region to get trending videos from.",
+        ),
+        DropdownInput(
+            name="category",
+            display_name="Category",
+            options=list(VIDEO_CATEGORIES.keys()),
+            value="All",
+            info="The category of videos to retrieve.",
+        ),
+        IntInput(
+            name="max_results",
+            display_name="Max Results",
+            value=10,
+            info="Maximum number of trending videos to return (1-50).",
+        ),
+        BoolInput(
+            name="include_statistics",
+            display_name="Include Statistics",
+            value=True,
+            info="Include video statistics (views, likes, comments).",
+        ),
+        BoolInput(
+            name="include_content_details",
+            display_name="Include Content Details",
+            value=True,
+            info="Include video duration and quality info.",
+            advanced=True,
+        ),
+        BoolInput(
+            name="include_thumbnails",
+            display_name="Include Thumbnails",
+            value=True,
+            info="Include video thumbnail URLs.",
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(name="trending_videos", display_name="Trending Videos", method="get_trending_videos"),
+    ]
+
+    max_results: int
+
+    def _format_duration(self, duration: str) -> str:
+        """Formats ISO 8601 duration to readable format."""
+        import re
+
+        # Remove 'PT' from the start of duration
+        duration = duration[2:]
+
+        hours = 0
+        minutes = 0
+        seconds = 0
+
+        # Extract hours, minutes and seconds
+        time_dict = {}
+        for time_unit in ["H", "M", "S"]:
+            match = re.search(r"(\d+)" + time_unit, duration)
+            if match:
+                time_dict[time_unit] = int(match.group(1))
+
+        if "H" in time_dict:
+            hours = time_dict["H"]
+        if "M" in time_dict:
+            minutes = time_dict["M"]
+        if "S" in time_dict:
+            seconds = time_dict["S"]
+
+        # Format the time string
+        if hours > 0:
+            return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+        return f"{minutes:02d}:{seconds:02d}"
+
+    @contextmanager
+    def youtube_client(self):
+        """Context manager for YouTube API client."""
+        client = build("youtube", "v3", developerKey=self.api_key)
+        try:
+            yield client
+        finally:
+            client.close()
+
+    def get_trending_videos(self) -> DataFrame:
+        """Retrieves trending videos from YouTube and returns as DataFrame."""
+        try:
+            # Validate max_results
+            if not 1 <= self.max_results <= MAX_API_RESULTS:
+                self.max_results = min(max(1, self.max_results), MAX_API_RESULTS)
+
+            # Use context manager for YouTube API client
+            with self.youtube_client() as youtube:
+                # Get country code
+                region_code = self.COUNTRY_CODES[self.region]
+
+                # Prepare API request parts
+                parts = ["snippet"]
+                if self.include_statistics:
+                    parts.append("statistics")
+                if self.include_content_details:
+                    parts.append("contentDetails")
+
+                # Prepare API request parameters
+                request_params = {
+                    "part": ",".join(parts),
+                    "chart": "mostPopular",
+                    "regionCode": region_code,
+                    "maxResults": self.max_results,
+                }
+
+                # Add category filter if not "All"
+                if self.category != "All":
+                    request_params["videoCategoryId"] = self.VIDEO_CATEGORIES[self.category]
+
+                # Get trending videos
+                request = youtube.videos().list(**request_params)
+                response = request.execute()
+
+                videos_data = []
+                for item in response.get("items", []):
+                    video_data = {
+                        "video_id": item["id"],
+                        "title": item["snippet"]["title"],
+                        "description": item["snippet"]["description"],
+                        "channel_id": item["snippet"]["channelId"],
+                        "channel_title": item["snippet"]["channelTitle"],
+                        "published_at": item["snippet"]["publishedAt"],
+                        "url": f"https://www.youtube.com/watch?v={item['id']}",
+                        "region": self.region,
+                        "category": self.category,
+                    }
+
+                    # Add thumbnails if requested
+                    if self.include_thumbnails:
+                        for size, thumb in item["snippet"]["thumbnails"].items():
+                            video_data[f"thumbnail_{size}_url"] = thumb["url"]
+                            video_data[f"thumbnail_{size}_width"] = thumb.get("width", 0)
+                            video_data[f"thumbnail_{size}_height"] = thumb.get("height", 0)
+
+                    # Add statistics if requested
+                    if self.include_statistics and "statistics" in item:
+                        video_data.update(
+                            {
+                                "view_count": int(item["statistics"].get("viewCount", 0)),
+                                "like_count": int(item["statistics"].get("likeCount", 0)),
+                                "comment_count": int(item["statistics"].get("commentCount", 0)),
+                            }
+                        )
+
+                    # Add content details if requested
+                    if self.include_content_details and "contentDetails" in item:
+                        content_details = item["contentDetails"]
+                        video_data.update(
+                            {
+                                "duration": self._format_duration(content_details["duration"]),
+                                "definition": content_details.get("definition", "hd").upper(),
+                                "has_captions": content_details.get("caption", "false") == "true",
+                                "licensed_content": content_details.get("licensedContent", False),
+                                "projection": content_details.get("projection", "rectangular"),
+                            }
+                        )
+
+                    videos_data.append(video_data)
+
+                # Convert to DataFrame
+                videos_df = pd.DataFrame(videos_data)
+
+                # Organize columns
+                column_order = [
+                    "video_id",
+                    "title",
+                    "channel_id",
+                    "channel_title",
+                    "category",
+                    "region",
+                    "published_at",
+                    "url",
+                    "description",
+                ]
+
+                if self.include_statistics:
+                    column_order.extend(["view_count", "like_count", "comment_count"])
+
+                if self.include_content_details:
+                    column_order.extend(["duration", "definition", "has_captions", "licensed_content", "projection"])
+
+                # Add thumbnail columns at the end if included
+                if self.include_thumbnails:
+                    thumbnail_cols = [col for col in videos_df.columns if col.startswith("thumbnail_")]
+                    column_order.extend(sorted(thumbnail_cols))
+
+                # Reorder columns, including any that might not be in column_order
+                remaining_cols = [col for col in videos_df.columns if col not in column_order]
+                videos_df = videos_df[column_order + remaining_cols]
+
+                return DataFrame(videos_df)
+
+        except HttpError as e:
+            error_message = f"YouTube API error: {e}"
+            if e.resp.status == HTTP_FORBIDDEN:
+                error_message = "API quota exceeded or access forbidden."
+            elif e.resp.status == HTTP_NOT_FOUND:
+                error_message = "Resource not found."
+
+            return DataFrame(pd.DataFrame({"error": [error_message]}))
+
+        except Exception as e:  # noqa: BLE001
+            logger.exception("An unexpected error occurred:")
+            return DataFrame(pd.DataFrame({"error": [str(e)]}))