camel-ai 0.2.45__py3-none-any.whl → 0.2.47__py3-none-any.whl

camel/toolkits/search_toolkit.py

@@ -1064,6 +1064,172 @@ class SearchToolkit(BaseToolkit):
         except Exception as e:
             return {"error": f"Exa search failed: {e!s}"}
 
+    @api_keys_required([(None, 'TONGXIAO_API_KEY')])
+    def search_alibaba_tongxiao(
+        self,
+        query: str,
+        time_range: Literal[
+            "OneDay", "OneWeek", "OneMonth", "OneYear", "NoLimit"
+        ] = "NoLimit",
+        industry: Optional[
+            Literal[
+                "finance",
+                "law",
+                "medical",
+                "internet",
+                "tax",
+                "news_province",
+                "news_center",
+            ]
+        ] = None,
+        page: int = 1,
+        return_main_text: bool = False,
+        return_markdown_text: bool = True,
+        enable_rerank: bool = True,
+    ) -> Dict[str, Any]:
+        r"""Query the Alibaba Tongxiao search API and return search results.
+
+        A powerful search API optimized for Chinese language queries with
+        features:
+        - Enhanced Chinese language understanding
+        - Industry-specific filtering (finance, law, medical, etc.)
+        - Structured data with markdown formatting
+        - Result reranking for relevance
+        - Time-based filtering
+
+        Args:
+            query (str): The search query string (length >= 1 and <= 100).
+            time_range (Literal["OneDay", "OneWeek", "OneMonth", "OneYear",
+                "NoLimit"]): Time frame filter for search results.
+                (default: :obj:`"NoLimit"`)
+            industry (Optional[Literal["finance", "law", "medical",
+                "internet", "tax", "news_province", "news_center"]]):
+                Industry-specific search filter. When specified, only returns
+                results from sites in the specified industries. Multiple
+                industries can be comma-separated.
+                (default: :obj:`None`)
+            page (int): Page number for results pagination.
+                (default: :obj:`1`)
+            return_main_text (bool): Whether to include the main text of the
+                webpage in results. (default: :obj:`True`)
+            return_markdown_text (bool): Whether to include markdown formatted
+                content in results. (default: :obj:`True`)
+            enable_rerank (bool): Whether to enable result reranking. If
+                response time is critical, setting this to False can reduce
+                response time by approximately 140ms. (default: :obj:`True`)
+
+        Returns:
+            Dict[str, Any]: A dictionary containing either search results with
+                'requestId' and 'results' keys, or an 'error' key with error
+                message. Each result contains title, snippet, url and other
+                metadata.
+        """
+        TONGXIAO_API_KEY = os.getenv("TONGXIAO_API_KEY")
+
+        # Validate query length
+        if not query or len(query) > 100:
+            return {
+                "error": "Query length must be between 1 and 100 characters"
+            }
+
+        # API endpoint and parameters
+        base_url = "https://cloud-iqs.aliyuncs.com/search/genericSearch"
+        headers = {
+            "X-API-Key": TONGXIAO_API_KEY,
+        }
+
+        # Convert boolean parameters to string for compatibility with requests
+        params: Dict[str, Union[str, int]] = {
+            "query": query,
+            "timeRange": time_range,
+            "page": page,
+            "returnMainText": str(return_main_text).lower(),
+            "returnMarkdownText": str(return_markdown_text).lower(),
+            "enableRerank": str(enable_rerank).lower(),
+        }
+
+        # Only add industry parameter if specified
+        if industry is not None:
+            params["industry"] = industry
+
+        try:
+            # Send GET request with proper typing for params
+            response = requests.get(
+                base_url, headers=headers, params=params, timeout=10
+            )
+
+            # Check response status
+            if response.status_code != 200:
+                return {
+                    "error": (
+                        f"Alibaba Tongxiao API request failed with status "
+                        f"code {response.status_code}: {response.text}"
+                    )
+                }
+
+            # Parse JSON response
+            data = response.json()
+
+            # Extract and format pageItems
+            page_items = data.get("pageItems", [])
+            results = []
+            for idx, item in enumerate(page_items):
+                # Create a simplified result structure
+                result = {
+                    "result_id": idx + 1,
+                    "title": item.get("title", ""),
+                    "snippet": item.get("snippet", ""),
+                    "url": item.get("link", ""),
+                    "hostname": item.get("hostname", ""),
+                }
+
+                # Only include additional fields if they exist and are
+                # requested
+                if "summary" in item and item.get("summary"):
+                    result["summary"] = item["summary"]
+                elif (
+                    return_main_text
+                    and "mainText" in item
+                    and item.get("mainText")
+                ):
+                    result["summary"] = item["mainText"]
+
+                if (
+                    return_main_text
+                    and "mainText" in item
+                    and item.get("mainText")
+                ):
+                    result["main_text"] = item["mainText"]
+
+                if (
+                    return_markdown_text
+                    and "markdownText" in item
+                    and item.get("markdownText")
+                ):
+                    result["markdown_text"] = item["markdownText"]
+
+                if "score" in item:
+                    result["score"] = item["score"]
+
+                if "publishTime" in item:
+                    result["publish_time"] = item["publishTime"]
+
+                results.append(result)
+
+            # Return a simplified structure
+            return {
+                "request_id": data.get("requestId", ""),
+                "results": results,
+            }
+
+        except requests.exceptions.RequestException as e:
+            return {"error": f"Alibaba Tongxiao search request failed: {e!s}"}
+        except Exception as e:
+            return {
+                "error": f"Unexpected error during Alibaba Tongxiao "
+                f"search: {e!s}"
+            }
+
     def get_tools(self) -> List[FunctionTool]:
         r"""Returns a list of FunctionTool objects representing the
         functions in the toolkit.
@@ -1084,4 +1250,5 @@ class SearchToolkit(BaseToolkit):
             FunctionTool(self.search_baidu),
             FunctionTool(self.search_bing),
             FunctionTool(self.search_exa),
+            FunctionTool(self.search_alibaba_tongxiao),
         ]

camel/toolkits/video_analysis_toolkit.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
+import io
 import os
 import tempfile
 from pathlib import Path
@@ -92,9 +93,15 @@ class VideoAnalysisToolkit(BaseToolkit):
             transcription using OpenAI's audio models. Requires a valid OpenAI
             API key. When disabled, video analysis will be based solely on
             visual content. (default: :obj:`False`)
+        frame_interval (float, optional): Interval in seconds between frames
+            to extract from the video. (default: :obj:`4.0`)
+        output_language (str, optional): The language for output responses.
+            (default: :obj:`"English"`)
+        cookies_path (Optional[str]): The path to the cookies file
+            for the video service in Netscape format. (default: :obj:`None`)
         timeout (Optional[float]): The timeout value for API requests
-                in seconds. If None, no timeout is applied.
-                (default: :obj:`None`)
+            in seconds. If None, no timeout is applied.
+            (default: :obj:`None`)
     """
 
     @dependencies_required("ffmpeg", "scenedetect")
@@ -103,27 +110,29 @@ class VideoAnalysisToolkit(BaseToolkit):
         download_directory: Optional[str] = None,
         model: Optional[BaseModelBackend] = None,
         use_audio_transcription: bool = False,
+        frame_interval: float = 4.0,
+        output_language: str = "English",
+        cookies_path: Optional[str] = None,
         timeout: Optional[float] = None,
     ) -> None:
         super().__init__(timeout=timeout)
         self._cleanup = download_directory is None
         self._temp_files: list[str] = []  # Track temporary files for cleanup
         self._use_audio_transcription = use_audio_transcription
+        self.output_language = output_language
+        self.frame_interval = frame_interval
 
         self._download_directory = Path(
             download_directory or tempfile.mkdtemp()
         ).resolve()
 
         self.video_downloader_toolkit = VideoDownloaderToolkit(
-            download_directory=str(self._download_directory)
+            download_directory=str(self._download_directory),
+            cookies_path=cookies_path,
         )
 
         try:
             self._download_directory.mkdir(parents=True, exist_ok=True)
-        except FileExistsError:
-            raise ValueError(
-                f"{self._download_directory} is not a valid directory."
-            )
         except OSError as e:
             raise ValueError(
                 f"Error creating directory {self._download_directory}: {e}"
@@ -137,16 +146,18 @@ class VideoAnalysisToolkit(BaseToolkit):
             # Import ChatAgent at runtime to avoid circular imports
             from camel.agents import ChatAgent
 
-            self.vl_agent = ChatAgent(model=self.vl_model)
+            self.vl_agent = ChatAgent(
+                model=self.vl_model, output_language=self.output_language
+            )
         else:
             # If no model is provided, use default model in ChatAgent
             # Import ChatAgent at runtime to avoid circular imports
             from camel.agents import ChatAgent
 
-            self.vl_agent = ChatAgent()
+            self.vl_agent = ChatAgent(output_language=self.output_language)
             logger.warning(
-                "No vision-language model provided. Using default model in"
-                " ChatAgent."
+                "No vision-language model provided. Using default model in "
+                "ChatAgent."
             )
 
         # Initialize audio models only if audio transcription is enabled
@@ -179,16 +190,22 @@ class VideoAnalysisToolkit(BaseToolkit):
         # Clean up temporary directory if needed
         if self._cleanup and os.path.exists(self._download_directory):
             try:
-                import shutil
+                import sys
 
-                shutil.rmtree(self._download_directory)
-                logger.debug(
-                    f"Removed temporary directory: {self._download_directory}"
-                )
+                if getattr(sys, 'modules', None) is not None:
+                    import shutil
+
+                    shutil.rmtree(self._download_directory)
+                    logger.debug(
+                        f"Removed temp directory: {self._download_directory}"
+                    )
+            except (ImportError, AttributeError):
+                # Skip cleanup if interpreter is shutting down
+                pass
             except OSError as e:
                 logger.warning(
-                    f"Failed to remove temporary directory"
-                    f" {self._download_directory}: {e}"
+                    f"Failed to remove temporary directory "
+                    f"{self._download_directory}: {e}"
                 )
 
     def _extract_audio_from_video(
@@ -242,88 +259,217 @@ class VideoAnalysisToolkit(BaseToolkit):
             logger.error(f"Audio transcription failed: {e}")
             return "Audio transcription failed."
 
-    def _extract_keyframes(
-        self, video_path: str, num_frames: int, threshold: float = 25.0
-    ) -> List[Image.Image]:
-        r"""Extract keyframes from a video based on scene changes
-        and return them as PIL.Image.Image objects.
+    def _extract_keyframes(self, video_path: str) -> List[Image.Image]:
+        r"""Extract keyframes from a video based on scene changes and
+        regular intervals,and return them as PIL.Image.Image objects.
 
         Args:
             video_path (str): Path to the video file.
-            num_frames (int): Number of keyframes to extract.
-            threshold (float): The threshold value for scene change detection.
 
         Returns:
-            list: A list of PIL.Image.Image objects representing
+            List[Image.Image]: A list of PIL.Image.Image objects representing
                 the extracted keyframes.
+
+        Raises:
+            ValueError: If no frames could be extracted from the video.
         """
+        import cv2
+        import numpy as np
         from scenedetect import (  # type: ignore[import-untyped]
             SceneManager,
-            VideoManager,
+            open_video,
         )
         from scenedetect.detectors import (  # type: ignore[import-untyped]
             ContentDetector,
         )
 
-        if num_frames <= 0:
+        # Get video information
+        cap = cv2.VideoCapture(video_path)
+        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        fps = cap.get(cv2.CAP_PROP_FPS)
+        duration = total_frames / fps if fps > 0 else 0
+        cap.release()
+
+        frame_interval = self.frame_interval  # seconds
+        # Maximum number of frames to extract to avoid memory issues
+        MAX_FRAMES = 100
+        # Minimum time difference (in seconds) to consider frames as distinct
+        TIME_THRESHOLD = 1.0
+
+        # Calculate the total number of frames to extract
+        if duration <= 0 or fps <= 0:
             logger.warning(
-                f"Invalid num_frames: {num_frames}, using default of 1"
+                "Invalid video duration or fps, using default frame count"
             )
-            num_frames = 1
+            num_frames = 10
+        else:
+            num_frames = max(int(duration / frame_interval), 1)
+
+            if num_frames > MAX_FRAMES:
+                frame_interval = duration / MAX_FRAMES
+                num_frames = MAX_FRAMES
 
-        video_manager = VideoManager([video_path])
+            logger.info(
+                f"Video duration: {duration:.2f}s, target frames: {num_frames}"
+                f"at {frame_interval:.2f}s intervals"
+            )
+
+        # Use scene detection to extract keyframes
+        # Use open_video instead of VideoManager
+        video = open_video(video_path)
         scene_manager = SceneManager()
-        scene_manager.add_detector(ContentDetector(threshold=threshold))
+        scene_manager.add_detector(ContentDetector())
 
-        video_manager.set_duration()
-        video_manager.start()
-        scene_manager.detect_scenes(video_manager)
+        # Detect scenes using the modern API
+        scene_manager.detect_scenes(video)
 
         scenes = scene_manager.get_scene_list()
         keyframes: List[Image.Image] = []
 
-        # Handle case where no scenes are detected
-        if not scenes:
+        # If scene detection is successful, prioritize scene change points
+        if scenes:
+            logger.info(f"Detected {len(scenes)} scene changes")
+
+            if len(scenes) > num_frames:
+                scene_indices = np.linspace(
+                    0, len(scenes) - 1, num_frames, dtype=int
+                )
+                selected_scenes = [scenes[i] for i in scene_indices]
+            else:
+                selected_scenes = scenes
+
+            # Extract frames from scenes
+            for scene in selected_scenes:
+                try:
+                    # Get start time in seconds
+                    start_time = scene[0].get_seconds()
+                    frame = _capture_screenshot(video_path, start_time)
+                    keyframes.append(frame)
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to capture frame at scene change"
+                        f" {scene[0].get_seconds()}s: {e}"
+                    )
+
+        if len(keyframes) < num_frames and duration > 0:
+            logger.info(
+                f"Scene detection provided {len(keyframes)} frames, "
+                f"supplementing with regular interval frames"
+            )
+
+            existing_times = []
+            if scenes:
+                existing_times = [scene[0].get_seconds() for scene in scenes]
+
+            regular_frames = []
+            for i in range(num_frames):
+                time_sec = i * frame_interval
+
+                is_duplicate = False
+                for existing_time in existing_times:
+                    if abs(existing_time - time_sec) < TIME_THRESHOLD:
+                        is_duplicate = True
+                        break
+
+                if not is_duplicate:
+                    try:
+                        frame = _capture_screenshot(video_path, time_sec)
+                        regular_frames.append(frame)
+                    except Exception as e:
+                        logger.warning(
+                            f"Failed to capture frame at {time_sec}s: {e}"
+                        )
+
+            frames_needed = num_frames - len(keyframes)
+            if frames_needed > 0 and regular_frames:
+                if len(regular_frames) > frames_needed:
+                    indices = np.linspace(
+                        0, len(regular_frames) - 1, frames_needed, dtype=int
+                    )
+                    selected_frames = [regular_frames[i] for i in indices]
+                else:
+                    selected_frames = regular_frames
+
+                keyframes.extend(selected_frames)
+
+        if not keyframes:
             logger.warning(
-                "No scenes detected in video, capturing frames at "
-                "regular intervals"
+                "No frames extracted, falling back to simple interval"
+                "extraction"
             )
-            import cv2
-
-            cap = cv2.VideoCapture(video_path)
-            total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
-            fps = cap.get(cv2.CAP_PROP_FPS)
-            duration = total_frames / fps if fps > 0 else 0
-
-            if duration > 0 and total_frames > 0:
-                # Extract frames at regular intervals
-                interval = duration / min(num_frames, total_frames)
-                for i in range(min(num_frames, total_frames)):
-                    time_sec = i * interval
+            for i in range(
+                min(num_frames, 10)
+            ):  # Limit to a maximum of 10 frames to avoid infinite loops
+                time_sec = i * (duration / 10 if duration > 0 else 6.0)
+                try:
                     frame = _capture_screenshot(video_path, time_sec)
                     keyframes.append(frame)
-
-            cap.release()
-        else:
-            # Extract frames from detected scenes
-            for start_time, _ in scenes:
-                if len(keyframes) >= num_frames:
-                    break
-                frame = _capture_screenshot(video_path, start_time)
-                keyframes.append(frame)
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to capture frame at {time_sec}s: {e}"
+                    )
 
         if not keyframes:
-            logger.error("Failed to extract any keyframes from video")
-            raise ValueError("Failed to extract keyframes from video")
+            error_msg = (
+                f"Failed to extract any keyframes from video: {video_path}"
+            )
+            logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        # Normalize image sizes
+        normalized_keyframes = self._normalize_frames(keyframes)
+
+        logger.info(
+            f"Extracted and normalized {len(normalized_keyframes)} keyframes"
+        )
+        return normalized_keyframes
+
+    def _normalize_frames(
+        self, frames: List[Image.Image], target_width: int = 512
+    ) -> List[Image.Image]:
+        r"""Normalize the size of extracted frames.
 
-        logger.info(f"Extracted {len(keyframes)} keyframes")
-        return keyframes
+        Args:
+            frames (List[Image.Image]): List of frames to normalize.
+            target_width (int): Target width for normalized frames.
+
+        Returns:
+            List[Image.Image]: List of normalized frames.
+        """
+        normalized_frames: List[Image.Image] = []
+
+        for frame in frames:
+            # Get original dimensions
+            width, height = frame.size
+
+            # Calculate new height, maintaining aspect ratio
+            aspect_ratio = width / height
+            new_height = int(target_width / aspect_ratio)
+
+            # Resize image
+            resized_frame = frame.resize(
+                (target_width, new_height), Image.Resampling.LANCZOS
+            )
+
+            # Ensure the image has a proper format
+            if resized_frame.mode != 'RGB':
+                resized_frame = resized_frame.convert('RGB')
+
+            # Create a new image with explicit format
+            with io.BytesIO() as buffer:
+                resized_frame.save(buffer, format='JPEG')
+                buffer.seek(0)
+                formatted_frame = Image.open(buffer)
+                formatted_frame.load()  # Load the image data
+
+            normalized_frames.append(formatted_frame)
+
+        return normalized_frames
 
     def ask_question_about_video(
         self,
         video_path: str,
         question: str,
-        num_frames: int = 28,
     ) -> str:
         r"""Ask a question about the video.
 
@@ -331,24 +477,12 @@ class VideoAnalysisToolkit(BaseToolkit):
             video_path (str): The path to the video file.
                 It can be a local file or a URL (such as Youtube website).
             question (str): The question to ask about the video.
-            num_frames (int): The number of frames to extract from the video.
-                To be adjusted based on the length of the video.
-                (default: :obj:`28`)
 
         Returns:
             str: The answer to the question.
         """
         from urllib.parse import urlparse
 
-        if not question:
-            raise ValueError("Question cannot be empty")
-
-        if num_frames <= 0:
-            logger.warning(
-                f"Invalid num_frames: {num_frames}, using default of 28"
-            )
-            num_frames = 28
-
         parsed_url = urlparse(video_path)
         is_url = all([parsed_url.scheme, parsed_url.netloc])
 
@@ -374,7 +508,7 @@ class VideoAnalysisToolkit(BaseToolkit):
                 audio_path = self._extract_audio_from_video(video_path)
                 audio_transcript = self._transcribe_audio(audio_path)
 
-            video_frames = self._extract_keyframes(video_path, num_frames)
+            video_frames = self._extract_keyframes(video_path)
             prompt = VIDEO_QA_PROMPT.format(
                 audio_transcription=audio_transcript,
                 question=question,
@@ -385,7 +519,8 @@ class VideoAnalysisToolkit(BaseToolkit):
                 content=prompt,
                 image_list=video_frames,
             )
-
+            # Reset the agent to clear previous state
+            self.vl_agent.reset()
             response = self.vl_agent.step(msg)
             if not response or not response.msgs:
                 logger.error("Model returned empty response")
@@ -398,7 +533,7 @@ class VideoAnalysisToolkit(BaseToolkit):
             return answer
 
         except Exception as e:
-            error_message = f"Error processing video: {e!s}"
+            error_message = f"Error processing video: {e}"
             logger.error(error_message)
             return f"Error: {error_message}"
 

camel/toolkits/video_download_toolkit.py

@@ -102,10 +102,17 @@ class VideoDownloaderToolkit(BaseToolkit):
         Cleans up the downloaded video if they are stored in a temporary
         directory.
         """
-        import shutil
-
         if self._cleanup:
-            shutil.rmtree(self._download_directory, ignore_errors=True)
+            try:
+                import sys
+
+                if getattr(sys, 'modules', None) is not None:
+                    import shutil
+
+                    shutil.rmtree(self._download_directory, ignore_errors=True)
+            except (ImportError, AttributeError):
+                # Skip cleanup if interpreter is shutting down
+                pass
 
     def download_video(self, url: str) -> str:
         r"""Download the video and optionally split it into chunks.