lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/twelvelabs/pegasus_index.py

@@ -0,0 +1,311 @@
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+from tenacity import retry, stop_after_attempt, wait_exponential
+from twelvelabs import TwelveLabs
+
+from lfx.custom import Component
+from lfx.inputs import DataInput, DropdownInput, SecretStrInput, StrInput
+from lfx.io import Output
+from lfx.schema import Data
+
+
+class TwelveLabsError(Exception):
+    """Base exception for TwelveLabs errors."""
+
+
+class IndexCreationError(TwelveLabsError):
+    """Error raised when there's an issue with an index."""
+
+
+class TaskError(TwelveLabsError):
+    """Error raised when a task fails."""
+
+
+class TaskTimeoutError(TwelveLabsError):
+    """Error raised when a task times out."""
+
+
+class PegasusIndexVideo(Component):
+    """Indexes videos using TwelveLabs Pegasus API and adds the video ID to metadata."""
+
+    display_name = "TwelveLabs Pegasus Index Video"
+    description = "Index videos using TwelveLabs and add the video_id to metadata."
+    icon = "TwelveLabs"
+    name = "TwelveLabsPegasusIndexVideo"
+    documentation = "https://github.com/twelvelabs-io/twelvelabs-developer-experience/blob/main/integrations/Langflow/TWELVE_LABS_COMPONENTS_README.md"
+
+    inputs = [
+        DataInput(
+            name="videodata",
+            display_name="Video Data",
+            info="Video Data objects (from VideoFile or SplitVideo)",
+            is_list=True,
+            required=True,
+        ),
+        SecretStrInput(
+            name="api_key", display_name="TwelveLabs API Key", info="Enter your TwelveLabs API Key.", required=True
+        ),
+        DropdownInput(
+            name="model_name",
+            display_name="Model",
+            info="Pegasus model to use for indexing",
+            options=["pegasus1.2"],
+            value="pegasus1.2",
+            advanced=False,
+        ),
+        StrInput(
+            name="index_name",
+            display_name="Index Name",
+            info="Name of the index to use. If the index doesn't exist, it will be created.",
+            required=False,
+        ),
+        StrInput(
+            name="index_id",
+            display_name="Index ID",
+            info="ID of an existing index to use. If provided, index_name will be ignored.",
+            required=False,
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Indexed Data", name="indexed_data", method="index_videos", output_types=["Data"], is_list=True
+        ),
+    ]
+
+    def _get_or_create_index(self, client: TwelveLabs) -> tuple[str, str]:
+        """Get existing index or create new one.
+
+        Returns (index_id, index_name).
+        """
+        # First check if index_id is provided and valid
+        if hasattr(self, "index_id") and self.index_id:
+            try:
+                index = client.index.retrieve(id=self.index_id)
+            except (ValueError, KeyError) as e:
+                if not hasattr(self, "index_name") or not self.index_name:
+                    error_msg = "Invalid index ID provided and no index name specified for fallback"
+                    raise IndexCreationError(error_msg) from e
+            else:
+                return self.index_id, index.name
+
+        # If index_name is provided, try to find it
+        if hasattr(self, "index_name") and self.index_name:
+            try:
+                # List all indexes and find by name
+                indexes = client.index.list()
+                for idx in indexes:
+                    if idx.name == self.index_name:
+                        return idx.id, idx.name
+
+                # If we get here, index wasn't found - create it
+                index = client.index.create(
+                    name=self.index_name,
+                    models=[
+                        {
+                            "name": self.model_name if hasattr(self, "model_name") else "pegasus1.2",
+                            "options": ["visual", "audio"],
+                        }
+                    ],
+                )
+            except (ValueError, KeyError) as e:
+                error_msg = f"Error with index name {self.index_name}"
+                raise IndexCreationError(error_msg) from e
+            else:
+                return index.id, index.name
+
+        # If we get here, neither index_id nor index_name was provided
+        error_msg = "Either index_name or index_id must be provided"
+        raise IndexCreationError(error_msg)
+
+    def on_task_update(self, task: Any, video_path: str) -> None:
+        """Callback for task status updates.
+
+        Updates the component status with the current task status.
+        """
+        video_name = Path(video_path).name
+        status_msg = f"Indexing {video_name}... Status: {task.status}"
+        self.status = status_msg
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=5, max=60), reraise=True)
+    def _check_task_status(
+        self,
+        client: TwelveLabs,
+        task_id: str,
+        video_path: str,
+    ) -> Any:
+        """Check task status once.
+
+        Makes a single API call to check the status of a task.
+        """
+        task = client.task.retrieve(id=task_id)
+        self.on_task_update(task, video_path)
+        return task
+
+    def _wait_for_task_completion(
+        self, client: TwelveLabs, task_id: str, video_path: str, max_retries: int = 120, sleep_time: int = 10
+    ) -> Any:
+        """Wait for task completion with timeout and improved error handling.
+
+        Polls the task status until completion or timeout.
+        """
+        retries = 0
+        consecutive_errors = 0
+        max_consecutive_errors = 5
+        video_name = Path(video_path).name
+
+        while retries < max_retries:
+            try:
+                self.status = f"Checking task status for {video_name} (attempt {retries + 1})"
+                task = self._check_task_status(client, task_id, video_path)
+
+                if task.status == "ready":
+                    self.status = f"Indexing for {video_name} completed successfully!"
+                    return task
+                if task.status == "failed":
+                    error_msg = f"Task failed for {video_name}: {getattr(task, 'error', 'Unknown error')}"
+                    self.status = error_msg
+                    raise TaskError(error_msg)
+                if task.status == "error":
+                    error_msg = f"Task encountered an error for {video_name}: {getattr(task, 'error', 'Unknown error')}"
+                    self.status = error_msg
+                    raise TaskError(error_msg)
+
+                time.sleep(sleep_time)
+                retries += 1
+                elapsed_time = retries * sleep_time
+                self.status = f"Indexing {video_name}... {elapsed_time}s elapsed"
+
+            except (ValueError, KeyError) as e:
+                consecutive_errors += 1
+                error_msg = f"Error checking task status for {video_name}: {e!s}"
+                self.status = error_msg
+
+                if consecutive_errors >= max_consecutive_errors:
+                    too_many_errors = f"Too many consecutive errors checking task status for {video_name}"
+                    raise TaskError(too_many_errors) from e
+
+                time.sleep(sleep_time * (2**consecutive_errors))
+                continue
+
+        timeout_msg = f"Timeout waiting for indexing of {video_name} after {max_retries * sleep_time} seconds"
+        self.status = timeout_msg
+        raise TaskTimeoutError(timeout_msg)
+
+    def _upload_video(self, client: TwelveLabs, video_path: str, index_id: str) -> str:
+        """Upload a single video and return its task ID.
+
+        Uploads a video file to the specified index and returns the task ID.
+        """
+        video_name = Path(video_path).name
+        with Path(video_path).open("rb") as video_file:
+            self.status = f"Uploading {video_name} to index {index_id}..."
+            task = client.task.create(index_id=index_id, file=video_file)
+            task_id = task.id
+            self.status = f"Upload complete for {video_name}. Task ID: {task_id}"
+            return task_id
+
+    def index_videos(self) -> list[Data]:
+        """Indexes each video and adds the video_id to its metadata."""
+        if not self.videodata:
+            self.status = "No video data provided."
+            return []
+
+        if not self.api_key:
+            error_msg = "TwelveLabs API Key is required"
+            raise IndexCreationError(error_msg)
+
+        if not (hasattr(self, "index_name") and self.index_name) and not (hasattr(self, "index_id") and self.index_id):
+            error_msg = "Either index_name or index_id must be provided"
+            raise IndexCreationError(error_msg)
+
+        client = TwelveLabs(api_key=self.api_key)
+        indexed_data_list: list[Data] = []
+
+        # Get or create the index
+        try:
+            index_id, index_name = self._get_or_create_index(client)
+            self.status = f"Using index: {index_name} (ID: {index_id})"
+        except IndexCreationError as e:
+            self.status = f"Failed to get/create TwelveLabs index: {e!s}"
+            raise
+
+        # First, validate all videos and create a list of valid ones
+        valid_videos: list[tuple[Data, str]] = []
+        for video_data_item in self.videodata:
+            if not isinstance(video_data_item, Data):
+                self.status = f"Skipping invalid data item: {video_data_item}"
+                continue
+
+            video_info = video_data_item.data
+            if not isinstance(video_info, dict):
+                self.status = f"Skipping item with invalid data structure: {video_info}"
+                continue
+
+            video_path = video_info.get("text")
+            if not video_path or not isinstance(video_path, str):
+                self.status = f"Skipping item with missing or invalid video path: {video_info}"
+                continue
+
+            if not Path(video_path).exists():
+                self.status = f"Video file not found, skipping: {video_path}"
+                continue
+
+            valid_videos.append((video_data_item, video_path))
+
+        if not valid_videos:
+            self.status = "No valid videos to process."
+            return []
+
+        # Upload all videos first and collect their task IDs
+        upload_tasks: list[tuple[Data, str, str]] = []  # (data_item, video_path, task_id)
+        for data_item, video_path in valid_videos:
+            try:
+                task_id = self._upload_video(client, video_path, index_id)
+                upload_tasks.append((data_item, video_path, task_id))
+            except (ValueError, KeyError) as e:
+                self.status = f"Failed to upload {video_path}: {e!s}"
+                continue
+
+        # Now check all tasks in parallel using a thread pool
+        with ThreadPoolExecutor(max_workers=min(10, len(upload_tasks))) as executor:
+            futures = []
+            for data_item, video_path, task_id in upload_tasks:
+                future = executor.submit(self._wait_for_task_completion, client, task_id, video_path)
+                futures.append((data_item, video_path, future))
+
+            # Process results as they complete
+            for data_item, video_path, future in futures:
+                try:
+                    completed_task = future.result()
+                    if completed_task.status == "ready":
+                        video_id = completed_task.video_id
+                        video_name = Path(video_path).name
+                        self.status = f"Video {video_name} indexed successfully. Video ID: {video_id}"
+
+                        # Add video_id to the metadata
+                        video_info = data_item.data
+                        if "metadata" not in video_info:
+                            video_info["metadata"] = {}
+                        elif not isinstance(video_info["metadata"], dict):
+                            self.status = f"Warning: Overwriting non-dict metadata for {video_path}"
+                            video_info["metadata"] = {}
+
+                        video_info["metadata"].update(
+                            {"video_id": video_id, "index_id": index_id, "index_name": index_name}
+                        )
+
+                        updated_data_item = Data(data=video_info)
+                        indexed_data_list.append(updated_data_item)
+                except (TaskError, TaskTimeoutError) as e:
+                    self.status = f"Failed to process {video_path}: {e!s}"
+
+        if not indexed_data_list:
+            self.status = "No videos were successfully indexed."
+        else:
+            self.status = f"Finished indexing {len(indexed_data_list)}/{len(self.videodata)} videos."
+
+        return indexed_data_list

lfx/components/twelvelabs/split_video.py

@@ -0,0 +1,291 @@
+import hashlib
+import math
+import subprocess
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from lfx.custom import Component
+from lfx.inputs import BoolInput, DropdownInput, HandleInput, IntInput
+from lfx.schema import Data
+from lfx.template import Output
+
+
+class SplitVideoComponent(Component):
+    """A component that splits a video into multiple clips of specified duration using FFmpeg."""
+
+    display_name = "Split Video"
+    description = "Split a video into multiple clips of specified duration."
+    icon = "TwelveLabs"
+    name = "SplitVideo"
+    documentation = "https://github.com/twelvelabs-io/twelvelabs-developer-experience/blob/main/integrations/Langflow/TWELVE_LABS_COMPONENTS_README.md"
+
+    inputs = [
+        HandleInput(
+            name="videodata",
+            display_name="Video Data",
+            info="Input video data from VideoFile component",
+            required=True,
+            input_types=["Data"],
+        ),
+        IntInput(
+            name="clip_duration",
+            display_name="Clip Duration (seconds)",
+            info="Duration of each clip in seconds",
+            required=True,
+            value=30,
+        ),
+        DropdownInput(
+            name="last_clip_handling",
+            display_name="Last Clip Handling",
+            info=(
+                "How to handle the final clip when it would be shorter than the specified duration:\n"
+                "- Truncate: Skip the final clip entirely if it's shorter than the specified duration\n"
+                "- Overlap Previous: Start the final clip earlier to maintain full duration, "
+                "overlapping with previous clip\n"
+                "- Keep Short: Keep the final clip at its natural length, even if shorter than specified duration"
+            ),
+            options=["Truncate", "Overlap Previous", "Keep Short"],
+            value="Overlap Previous",
+            required=True,
+        ),
+        BoolInput(
+            name="include_original",
+            display_name="Include Original Video",
+            info="Whether to include the original video in the output",
+            value=False,
+        ),
+    ]
+
+    outputs = [
+        Output(
+            name="clips",
+            display_name="Video Clips",
+            method="process",
+            output_types=["Data"],
+        ),
+    ]
+
+    def get_video_duration(self, video_path: str) -> float:
+        """Get video duration using FFmpeg."""
+        try:
+            # Validate video path to prevent shell injection
+            if not isinstance(video_path, str) or any(c in video_path for c in ";&|`$(){}[]<>*?!#~"):
+                error_msg = "Invalid video path"
+                raise ValueError(error_msg)
+
+            cmd = [
+                "ffprobe",
+                "-v",
+                "error",
+                "-show_entries",
+                "format=duration",
+                "-of",
+                "default=noprint_wrappers=1:nokey=1",
+                video_path,
+            ]
+            result = subprocess.run(  # noqa: S603
+                cmd,
+                capture_output=True,
+                text=True,
+                check=False,
+                shell=False,  # Explicitly set shell=False for security
+            )
+            if result.returncode != 0:
+                error_msg = f"FFprobe error: {result.stderr}"
+                raise RuntimeError(error_msg)
+            return float(result.stdout.strip())
+        except Exception as e:
+            self.log(f"Error getting video duration: {e!s}", "ERROR")
+            raise
+
+    def get_output_dir(self, video_path: str) -> str:
+        """Create a unique output directory for clips based on video name and timestamp."""
+        # Get the video filename without extension
+        path_obj = Path(video_path)
+        base_name = path_obj.stem
+
+        # Create a timestamp
+        timestamp = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d_%H-%M-%S")
+
+        # Create a unique hash from the video path
+        path_hash = hashlib.sha256(video_path.encode()).hexdigest()[:8]
+
+        # Create the output directory path
+        output_dir = Path(path_obj.parent) / f"clips_{base_name}_{timestamp}_{path_hash}"
+
+        # Create the directory if it doesn't exist
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        return str(output_dir)
+
+    def process_video(self, video_path: str, clip_duration: int, *, include_original: bool) -> list[Data]:
+        """Process video and split it into clips using FFmpeg."""
+        try:
+            # Get video duration
+            total_duration = self.get_video_duration(video_path)
+
+            # Calculate number of clips (ceiling to include partial clip)
+            num_clips = math.ceil(total_duration / clip_duration)
+            self.log(
+                f"Total duration: {total_duration}s, Clip duration: {clip_duration}s, Number of clips: {num_clips}"
+            )
+
+            # Create output directory for clips
+            output_dir = self.get_output_dir(video_path)
+
+            # Get original video info
+            path_obj = Path(video_path)
+            original_filename = path_obj.name
+            original_name = path_obj.stem
+
+            # List to store all video paths (including original if requested)
+            video_paths: list[Data] = []
+
+            # Add original video if requested
+            if include_original:
+                original_data: dict[str, Any] = {
+                    "text": video_path,
+                    "metadata": {
+                        "source": video_path,
+                        "type": "video",
+                        "clip_index": -1,  # -1 indicates original video
+                        "duration": int(total_duration),  # Convert to int
+                        "original_video": {
+                            "name": original_name,
+                            "filename": original_filename,
+                            "path": video_path,
+                            "duration": int(total_duration),  # Convert to int
+                            "total_clips": int(num_clips),
+                            "clip_duration": int(clip_duration),
+                        },
+                    },
+                }
+                video_paths.append(Data(data=original_data))
+
+            # Split video into clips
+            for i in range(int(num_clips)):  # Convert num_clips to int for range
+                start_time = float(i * clip_duration)  # Convert to float for time calculations
+                end_time = min(float((i + 1) * clip_duration), total_duration)
+                duration = end_time - start_time
+
+                # Handle last clip if it's shorter
+                if i == int(num_clips) - 1 and duration < clip_duration:  # Convert num_clips to int for comparison
+                    if self.last_clip_handling == "Truncate":
+                        # Skip if the last clip would be too short
+                        continue
+                    if self.last_clip_handling == "Overlap Previous" and i > 0:
+                        # Start from earlier to make full duration
+                        start_time = total_duration - clip_duration
+                        duration = clip_duration
+                    # For "Keep Short", we use the original start_time and duration
+
+                # Skip if duration is too small (less than 1 second)
+                if duration < 1:
+                    continue
+
+                # Generate output path
+                output_path = Path(output_dir) / f"clip_{i:03d}.mp4"
+                output_path_str = str(output_path)
+
+                try:
+                    # Use FFmpeg to split the video
+                    cmd = [
+                        "ffmpeg",
+                        "-i",
+                        video_path,
+                        "-ss",
+                        str(start_time),
+                        "-t",
+                        str(duration),
+                        "-c:v",
+                        "libx264",
+                        "-c:a",
+                        "aac",
+                        "-y",  # Overwrite output file if it exists
+                        output_path_str,
+                    ]
+
+                    result = subprocess.run(  # noqa: S603
+                        cmd,
+                        capture_output=True,
+                        text=True,
+                        check=False,
+                        shell=False,  # Explicitly set shell=False for security
+                    )
+                    if result.returncode != 0:
+                        error_msg = f"FFmpeg error: {result.stderr}"
+                        raise RuntimeError(error_msg)
+
+                    # Create timestamp string for metadata
+                    start_min = int(start_time // 60)
+                    start_sec = int(start_time % 60)
+                    end_min = int(end_time // 60)
+                    end_sec = int(end_time % 60)
+                    timestamp_str = f"{start_min:02d}:{start_sec:02d} - {end_min:02d}:{end_sec:02d}"
+
+                    # Create Data object for the clip
+                    clip_data: dict[str, Any] = {
+                        "text": output_path_str,
+                        "metadata": {
+                            "source": video_path,
+                            "type": "video",
+                            "clip_index": i,
+                            "start_time": float(start_time),
+                            "end_time": float(end_time),
+                            "duration": float(duration),
+                            "original_video": {
+                                "name": original_name,
+                                "filename": original_filename,
+                                "path": video_path,
+                                "duration": int(total_duration),
+                                "total_clips": int(num_clips),
+                                "clip_duration": int(clip_duration),
+                            },
+                            "clip": {
+                                "index": i,
+                                "total": int(num_clips),
+                                "duration": float(duration),
+                                "start_time": float(start_time),
+                                "end_time": float(end_time),
+                                "timestamp": timestamp_str,
+                            },
+                        },
+                    }
+                    video_paths.append(Data(data=clip_data))
+
+                except Exception as e:
+                    self.log(f"Error processing clip {i}: {e!s}", "ERROR")
+                    raise
+
+            self.log(f"Created {len(video_paths)} clips in {output_dir}")
+        except Exception as e:
+            self.log(f"Error processing video: {e!s}", "ERROR")
+            raise
+        else:
+            return video_paths
+
+    def process(self) -> list[Data]:
+        """Process the input video and return a list of Data objects containing the clips."""
+        try:
+            # Get the input video path from the previous component
+            if not hasattr(self, "videodata") or not isinstance(self.videodata, list) or len(self.videodata) != 1:
+                error_msg = "Please provide exactly one video"
+                raise ValueError(error_msg)
+
+            video_path = self.videodata[0].data.get("text")
+            if not video_path or not Path(video_path).exists():
+                error_msg = "Invalid video path"
+                raise ValueError(error_msg)
+
+            # Validate video path to prevent shell injection
+            if not isinstance(video_path, str) or any(c in video_path for c in ";&|`$(){}[]<>*?!#~"):
+                error_msg = "Invalid video path contains unsafe characters"
+                raise ValueError(error_msg)
+
+            # Process the video
+            return self.process_video(video_path, self.clip_duration, include_original=self.include_original)
+
+        except Exception as e:
+            self.log(f"Error in split video component: {e!s}", "ERROR")
+            raise

lfx/components/twelvelabs/text_embeddings.py

@@ -0,0 +1,57 @@
+from twelvelabs import TwelveLabs
+
+from lfx.base.embeddings.model import LCEmbeddingsModel
+from lfx.field_typing import Embeddings
+from lfx.io import DropdownInput, FloatInput, IntInput, SecretStrInput
+
+
+class TwelveLabsTextEmbeddings(Embeddings):
+    def __init__(self, api_key: str, model: str) -> None:
+        self.client = TwelveLabs(api_key=api_key)
+        self.model = model
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        all_embeddings: list[list[float]] = []
+        for text in texts:
+            if not text:
+                continue
+
+            result = self.client.embed.create(model_name=self.model, text=text)
+
+            if result.text_embedding and result.text_embedding.segments:
+                for segment in result.text_embedding.segments:
+                    all_embeddings.append([float(x) for x in segment.embeddings_float])
+                    break  # Only take first segment for now
+
+        return all_embeddings
+
+    def embed_query(self, text: str) -> list[float]:
+        result = self.client.embed.create(model_name=self.model, text=text)
+
+        if result.text_embedding and result.text_embedding.segments:
+            return [float(x) for x in result.text_embedding.segments[0].embeddings_float]
+        return []
+
+
+class TwelveLabsTextEmbeddingsComponent(LCEmbeddingsModel):
+    display_name = "TwelveLabs Text Embeddings"
+    description = "Generate embeddings using TwelveLabs text embedding models."
+    icon = "TwelveLabs"
+    name = "TwelveLabsTextEmbeddings"
+    documentation = "https://github.com/twelvelabs-io/twelvelabs-developer-experience/blob/main/integrations/Langflow/TWELVE_LABS_COMPONENTS_README.md"
+
+    inputs = [
+        SecretStrInput(name="api_key", display_name="TwelveLabs API Key", value="TWELVELABS_API_KEY", required=True),
+        DropdownInput(
+            name="model",
+            display_name="Model",
+            advanced=False,
+            options=["Marengo-retrieval-2.7"],
+            value="Marengo-retrieval-2.7",
+        ),
+        IntInput(name="max_retries", display_name="Max Retries", value=3, advanced=True),
+        FloatInput(name="request_timeout", display_name="Request Timeout", advanced=True),
+    ]
+
+    def build_embeddings(self) -> Embeddings:
+        return TwelveLabsTextEmbeddings(api_key=self.api_key, model=self.model)