camel-ai 0.2.68__py3-none-any.whl → 0.2.69a2__py3-none-any.whl

camel/toolkits/non_visual_browser_toolkit/snapshot.js

@@ -0,0 +1,188 @@
+(() => {
+    // Store each element as {text, priority, depth}
+    const elements = [];
+
+    // Maximum lines allowed before we start dropping lower-priority nodes
+    const MAX_LINES = 400;
+
+    // Priority helper – lower number = higher priority
+    function getPriority(tag, role, text) {
+        // 1. Interactive elements
+        if (["input", "button", "a", "select", "textarea"].includes(tag)) return 1;
+        if (["checkbox", "radio"].includes(role)) return 1;
+
+        // 2. Labels / descriptive adjacent text (label elements)
+        if (tag === "label") return 2;
+
+        // 3. General visible text
+        if (text) return 3;
+
+        // 4. Low-value structural nodes
+        return 4;
+    }
+
+    function isVisible(node) {
+        const rect = node.getBoundingClientRect();
+        if (rect.width === 0 || rect.height === 0) return false;
+
+        const style = window.getComputedStyle(node);
+        if (style.display === 'none' || style.visibility === 'hidden') return false;
+
+        return true;
+    }
+
+    function getRole(node) {
+        const tag = node.tagName.toLowerCase();
+        const type = node.getAttribute('type');
+
+        if (node.getAttribute('role')) return node.getAttribute('role');
+
+        if (tag === 'input') {
+            if (type === 'checkbox') return 'checkbox';
+            if (type === 'radio') return 'radio';
+            return 'input';
+        }
+
+        if (tag === 'button') return 'button';
+        if (tag === 'a') return 'link';
+        if (tag === 'select') return 'select';
+        if (tag === 'textarea') return 'textarea';
+        if (tag === 'p') return 'paragraph';
+        if (tag === 'span') return 'text';
+
+        return 'generic';
+    }
+
+    function getAccessibleName(node) {
+        if (node.hasAttribute('aria-label')) {
+            return node.getAttribute('aria-label');
+        }
+        if (node.hasAttribute('aria-labelledby')) {
+            const id = node.getAttribute('aria-labelledby');
+            const labelEl = document.getElementById(id);
+            if (labelEl) return labelEl.textContent.trim();
+        }
+        if (node.hasAttribute('title')) {
+            return node.getAttribute('title');
+        }
+
+        const tagName = node.tagName?.toLowerCase();
+        if (['style', 'script', 'meta', 'noscript', 'svg'].includes(tagName)) {
+            return '';
+        }
+
+        const text = node.textContent?.trim() || '';
+
+        // Ignore styles, tokens, or long CSS-like expressions
+        if (/^[.#]?[a-zA-Z0-9\-_]+\s*\{[^}]*\}/.test(text)) return '';
+        if ((text.match(/[;:{}]/g)?.length || 0) > 2) return '';
+
+        return text.replace(/[^\w\u4e00-\u9fa5\s\-.,?!'"（）()]/g, '').trim();
+    }
+
+    let refCounter = 1;
+
+    function traverse(node, depth) {
+        if (node.nodeType !== Node.ELEMENT_NODE) return;
+        if (!isVisible(node)) return;
+
+        const tagName = node.tagName.toLowerCase();
+        const text = getAccessibleName(node).slice(0, 50);
+
+        // Skip unlabeled links (anchors without any accessible name)
+        if (tagName === 'a' && !text) {
+            // Skip unlabeled links; process children if any
+            for (const child of node.children) {
+                traverse(child, depth + 1);
+            }
+            return;
+        }
+
+        const hasRoleOrText = ['button', 'a', 'input', 'select', 'textarea', 'p', 'span'].includes(tagName) ||
+                              node.getAttribute('role') || text;
+
+        if (hasRoleOrText) {
+            const role = getRole(node);
+            const ref = `e${refCounter++}`;
+            const label = text ? `"${text}"` : '';
+
+            // Raw line (without indent) – we will apply indentation later once we know
+            // which ancestor lines survive filtering so that indentation always reflects
+            // the visible hierarchy.
+            const lineText = `- ${role}${label ? ` ${label}` : ''} [ref=${ref}]`;
+            const priority = getPriority(tagName, role, text);
+
+            elements.push({ text: lineText, priority, depth });
+
+            // Always inject ref so Playwright can still locate the element even if line is later filtered out.
+            node.setAttribute('aria-ref', ref);
+        }
+
+        for (const child of node.children) {
+            traverse(child, depth + 1);
+        }
+    }
+
+    function processDocument(doc, depth = 0) {
+        try {
+            traverse(doc.body, depth);
+        } catch (e) {
+            // Handle docs without body (e.g., about:blank)
+        }
+
+        const frames = doc.querySelectorAll('iframe');
+        for (const frame of frames) {
+            try {
+                if (frame.contentDocument) {
+                    processDocument(frame.contentDocument, depth + 1);
+                }
+            } catch (e) {
+                // Skip cross-origin iframes
+            }
+        }
+    }
+
+    processDocument(document);
+
+    // Always drop priority-4 nodes (low-value structural or invisible)
+    let finalElements = elements.filter(el => el.priority <= 3);
+
+    // Additional size condensation when still exceeding MAX_LINES
+    if (finalElements.length > MAX_LINES) {
+        const filterBy = (maxPriority) => finalElements.filter(el => el.priority <= maxPriority);
+
+        // Progressively tighten: keep 1-3, then 1-2, finally only 1
+        for (const limit of [3, 2, 1]) {
+            const candidate = filterBy(limit);
+            if (candidate.length <= MAX_LINES || limit === 1) {
+                finalElements = candidate;
+                break;
+            }
+        }
+    }
+
+    // ------------------------------------------------------------------
+    // Re-apply indentation so that it matches the *visible* hierarchy only.
+    // Whenever an ancestor element is removed due to priority rules, its
+    // children will be re-indented one level up so the structure remains
+    // intuitive.
+    // ------------------------------------------------------------------
+    const outputLines = [];
+    const depthStack = []; // keeps track of kept original depths
+
+    for (const el of finalElements) {
+        // Pop depths that are not ancestors of current element
+        while (depthStack.length && depthStack[depthStack.length - 1] >= el.depth) {
+            depthStack.pop();
+        }
+
+        // Push the current depth so future descendants know their ancestor chain
+        depthStack.push(el.depth);
+
+        const compressedDepth = depthStack.length - 1; // root level has zero indent
+        const indent = '\t'.repeat(compressedDepth);
+        outputLines.push(indent + el.text);
+    }
+
+    return outputLines.join('\n');
+})();

camel/toolkits/non_visual_browser_toolkit/snapshot.py

@@ -0,0 +1,164 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, List, Optional
+
+if TYPE_CHECKING:
+    from playwright.async_api import Page
+
+# Logging support
+from camel.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class PageSnapshot:
+    """Utility for capturing YAML-like page snapshots and diff-only
+    variants."""
+
+    MAX_TIMEOUT_MS = 5000  # wait_for_load_state timeout
+
+    def __init__(self, page: "Page"):
+        self.page = page
+        self.snapshot_data: Optional[str] = None  # last full snapshot
+        self._last_url: Optional[str] = None
+        self.last_info: Dict[str, List[int] | bool] = {
+            "is_diff": False,
+            "priorities": [1, 2, 3],
+        }
+
+    # ---------------------------------------------------------------------
+    # Public API
+    # ---------------------------------------------------------------------
+    async def capture(
+        self, *, force_refresh: bool = False, diff_only: bool = False
+    ) -> str:
+        """Return current snapshot or just the diff to previous one."""
+        try:
+            current_url = self.page.url
+
+            # Serve cached copy (unless diff requested)
+            if (
+                not force_refresh
+                and current_url == self._last_url
+                and self.snapshot_data
+                and not diff_only
+            ):
+                return self.snapshot_data
+
+            # ensure DOM stability
+            await self.page.wait_for_load_state(
+                'domcontentloaded', timeout=self.MAX_TIMEOUT_MS
+            )
+
+            logger.debug("Capturing page snapshot …")
+            snapshot_text = await self._get_snapshot_direct()
+            formatted = self._format_snapshot(snapshot_text or "<empty>")
+
+            output = formatted
+            if diff_only and self.snapshot_data:
+                output = self._compute_diff(self.snapshot_data, formatted)
+
+            # update cache with *full* snapshot (not diff)
+            self._last_url = current_url
+            self.snapshot_data = formatted
+
+            # analyse priorities present (only for non-diff)
+            priorities_included = self._detect_priorities(
+                formatted if not diff_only else self.snapshot_data or formatted
+            )
+            self.last_info = {
+                "is_diff": diff_only and self.snapshot_data is not None,
+                "priorities": priorities_included,
+            }
+
+            logger.debug(
+                "Snapshot captured. Diff_only=%s, priorities=%s",
+                diff_only,
+                self.last_info["priorities"],
+            )
+            return output
+        except Exception as exc:
+            logger.error("Snapshot capture failed: %s", exc)
+            return f"Error: Could not capture page snapshot {exc}"
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+    _snapshot_js_cache: Optional[str] = None  # class-level cache
+
+    async def _get_snapshot_direct(self) -> Optional[str]:
+        try:
+            if PageSnapshot._snapshot_js_cache is None:
+                js_path = Path(__file__).parent / "snapshot.js"
+                PageSnapshot._snapshot_js_cache = js_path.read_text(
+                    encoding="utf-8"
+                )
+            return await self.page.evaluate(PageSnapshot._snapshot_js_cache)
+        except Exception as e:
+            logger.warning("Failed to execute snapshot JavaScript: %s", e)
+            return None
+
+    @staticmethod
+    def _format_snapshot(text: str) -> str:
+        return "\n".join(["- Page Snapshot", "```yaml", text, "```"])
+
+    @staticmethod
+    def _compute_diff(old: str, new: str) -> str:
+        if not old or not new:
+            return "- Page Snapshot (error: missing data for diff)"
+
+        import difflib
+
+        diff = list(
+            difflib.unified_diff(
+                old.splitlines(False),
+                new.splitlines(False),
+                fromfile='prev',
+                tofile='curr',
+                lineterm='',
+            )
+        )
+        if not diff:
+            return "- Page Snapshot (no structural changes)"
+        return "\n".join(["- Page Snapshot (diff)", "```diff", *diff, "```"])
+
+    # ------------------------------------------------------------------
+    def _detect_priorities(self, snapshot_yaml: str) -> List[int]:
+        """Return sorted list of priorities present (1,2,3)."""
+        priorities = set()
+        for line in snapshot_yaml.splitlines():
+            if '[ref=' not in line:
+                continue
+            lower_line = line.lower()
+            if any(
+                r in lower_line
+                for r in (
+                    "input",
+                    "button",
+                    "select",
+                    "textarea",
+                    "checkbox",
+                    "radio",
+                    "link",
+                )
+            ):
+                priorities.add(1)
+            elif 'label' in lower_line:
+                priorities.add(2)
+            else:
+                priorities.add(3)
+        if not priorities:
+            priorities.add(3)
+        return sorted(priorities)

camel/toolkits/pptx_toolkit.py

@@ -62,7 +62,7 @@ class PPTXToolkit(BaseToolkit):
             output_dir (str): The default directory for output files.
                 Defaults to the current working directory.
             timeout (Optional[float]): The timeout for the toolkit.
-                (default: :obj: `None`)
+                (default: :obj:`None`)
         """
         super().__init__(timeout=timeout)
         self.output_dir = Path(output_dir).resolve()
@@ -120,7 +120,7 @@ class PPTXToolkit(BaseToolkit):
             frame_paragraph: The paragraph to format.
             text (str): The text to format.
             set_color_to_white (bool): Whether to set the color to white.
-                (default: :obj: `False`)
+                (default: :obj:`False`)
         """
         from pptx.dml.color import RGBColor
 
@@ -170,7 +170,7 @@ class PPTXToolkit(BaseToolkit):
             flat_items_list (List[Tuple[str, int]]): The list of items to be
                 displayed.
             set_color_to_white (bool): Whether to set the font color to white.
-                (default: :obj: `False`)
+                (default: :obj:`False`)
         """
         if not flat_items_list:
             logger.warning("Empty bullet point list provided")
@@ -368,7 +368,7 @@ class PPTXToolkit(BaseToolkit):
                 supplied, it is resolved to self.output_dir.
             template (Optional[str]): The path to the template PPTX file.
                 Initializes a presentation from a given template file Or PPTX
-                file. (default: :obj: `None`)
+                file. (default: :obj:`None`)
 
         Returns:
             str: A success message indicating the file was created.

camel/toolkits/sympy_toolkit.py

@@ -39,7 +39,7 @@ class SymPyToolkit(BaseToolkit):
 
         Args:
             default_variable (str): The default variable for
-                operations (default: :obj: `x`)
+                operations (default: :obj:`x`)
         """
         super().__init__(timeout=timeout)
         self.default_variable = default_variable

camel/toolkits/task_planning_toolkit.py

@@ -32,7 +32,7 @@ class TaskPlanningToolkit(BaseToolkit):
 
         Args:
             timeout (Optional[float]): The timeout for the toolkit.
-                (default: :obj: `None`)
+                (default: :obj:`None`)
         """
         super().__init__(timeout=timeout)
 
@@ -53,7 +53,7 @@ class TaskPlanningToolkit(BaseToolkit):
                 string is the content for a new sub-task.
             original_task_id (Optional[str]): The id of the task to be
                 decomposed. If not provided, a new id will be generated.
-                (default: :obj: `None`)
+                (default: :obj:`None`)
 
         Returns:
             List[Task]: A list of newly created sub-task objects.
@@ -99,7 +99,7 @@ class TaskPlanningToolkit(BaseToolkit):
             sub_task_contents (List[str]): A list of strings, where each
                 string is the content for a new sub-task.
             original_task_id (Optional[str]): The id of the task to be
-                decomposed. (default: :obj: `None`)
+                decomposed. (default: :obj:`None`)
 
         Returns:
             List[Task]: Reordered or modified tasks.

camel/toolkits/thinking_toolkit.py

@@ -32,7 +32,7 @@ class ThinkingToolkit(BaseToolkit):
 
         Args:
             timeout (Optional[float]): The timeout for the toolkit.
-                (default: :obj: `None`)
+                (default: :obj:`None`)
         """
         super().__init__(timeout=timeout)
         self.plans: List[str] = []

camel/toolkits/video_analysis_toolkit.py

@@ -17,6 +17,7 @@ from __future__ import annotations
 
 import io
 import os
+import re
 import tempfile
 from pathlib import Path
 from typing import List, Optional
@@ -41,6 +42,11 @@ VIDEO_QA_PROMPT = """
 Analyze the provided video frames and corresponding audio transcription to \
 answer the given question(s) thoroughly and accurately.
 
+The transcriptions may come from two sources:
+  1. **Audio Transcription**: The spoken words in the video.
+  2. **Visual Text (OCR)**: Text extracted from the video frames (like \
+  captions, on-screen text, etc.).
+
 Instructions:
     1. Visual Analysis:
         - Examine the video frames to identify visible entities.
@@ -49,11 +55,13 @@ such as size, color, shape, texture, or behavior.
         - Note significant groupings, interactions, or contextual patterns \
 relevant to the analysis.
 
-    2. Audio Integration:
+    2. Audio and Text Integration:
         - Use the audio transcription to complement or clarify your visual \
 observations.
+        - Use the visual text (OCR) to get exact textual information that may \
+not be accurately readable from the images alone.
         - Identify names, descriptions, or contextual hints in the \
-transcription that help confirm or refine your visual analysis.
+transcriptions that help confirm or refine your visual analysis.
 
     3. Detailed Reasoning and Justification:
         - Provide a brief explanation of how you identified and distinguished \
@@ -65,7 +73,7 @@ your reasoning.
         - Specify the total number of distinct species or object types \
 identified in the video.
         - Describe the defining characteristics and any supporting evidence \
-from the video and transcription.
+from the video and transcription sources.
 
     5. Important Considerations:
         - Pay close attention to subtle differences that could distinguish \
@@ -76,6 +84,9 @@ similar-looking species or objects
 **Audio Transcription:**
 {audio_transcription}
 
+**Visual Text (OCR):**
+{visual_text}
+
 **Question:**
 {question}
 """
@@ -96,6 +107,8 @@ class VideoAnalysisToolkit(BaseToolkit):
             transcription using OpenAI's audio models. Requires a valid OpenAI
             API key. When disabled, video analysis will be based solely on
             visual content. (default: :obj:`False`)
+        use_ocr (bool, optional): Whether to enable OCR for extracting text
+            from video frames. (default: :obj:`False`)
         frame_interval (float, optional): Interval in seconds between frames
             to extract from the video. (default: :obj:`4.0`)
         output_language (str, optional): The language for output responses.
@@ -113,6 +126,7 @@ class VideoAnalysisToolkit(BaseToolkit):
         download_directory: Optional[str] = None,
         model: Optional[BaseModelBackend] = None,
         use_audio_transcription: bool = False,
+        use_ocr: bool = False,
         frame_interval: float = 4.0,
         output_language: str = "English",
         cookies_path: Optional[str] = None,
@@ -122,6 +136,7 @@ class VideoAnalysisToolkit(BaseToolkit):
         self._cleanup = download_directory is None
         self._temp_files: list[str] = []  # Track temporary files for cleanup
         self._use_audio_transcription = use_audio_transcription
+        self._use_ocr = use_ocr
         self.output_language = output_language
         self.frame_interval = frame_interval
 
@@ -211,6 +226,53 @@ class VideoAnalysisToolkit(BaseToolkit):
                     f"{self._download_directory}: {e}"
                 )
 
+    @dependencies_required("pytesseract", "cv2", "numpy")
+    def _extract_text_from_frame(self, frame: Image.Image) -> str:
+        r"""Extract text from a video frame using OCR.
+
+        Args:
+            frame (Image.Image): PIL image frame to process.
+
+        Returns:
+            str: Extracted text from the frame.
+        """
+        import cv2
+        import numpy as np
+        import pytesseract
+
+        try:
+            # Convert to OpenCV format for preprocessing
+            cv_image = cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR)
+
+            # Preprocessing for better OCR results
+            gray = cv2.cvtColor(cv_image, cv2.COLOR_BGR2GRAY)
+            blur = cv2.GaussianBlur(gray, (3, 3), 0)
+            _, threshold = cv2.threshold(
+                blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU
+            )
+
+            # Convert back to PIL image for OCR
+            preprocessed_frame = Image.fromarray(threshold)
+            return pytesseract.image_to_string(preprocessed_frame).strip()
+        except Exception as e:
+            logger.error(f"OCR failed: {e}")
+            return ""
+
+    def _process_extracted_text(self, text: str) -> str:
+        r"""Clean and format OCR-extracted text.
+
+        Args:
+            text (str): Raw extracted OCR text.
+
+        Returns:
+            str: Cleaned and formatted text.
+        """
+        # Filter irrelevant characters and noise
+        text = re.sub(r'[^\w\s,.?!:;\'"()-]', '', text)
+        # Remove excessive whitespace
+        text = re.sub(r'\s+', ' ', text).strip()
+        return text
+
     def _extract_audio_from_video(
         self, video_path: str, output_format: str = "mp3"
     ) -> str:
@@ -511,9 +573,21 @@ class VideoAnalysisToolkit(BaseToolkit):
                 audio_path = self._extract_audio_from_video(video_path)
                 audio_transcript = self._transcribe_audio(audio_path)
 
+            # Extract visual text with OCR
+            visual_text = ""
             video_frames = self._extract_keyframes(video_path)
+            # Build visual text only if OCR is enabled
+            if self._use_ocr:
+                for frame in video_frames:
+                    text = self._extract_text_from_frame(frame)
+                    processed = self._process_extracted_text(text)
+                    if processed:
+                        visual_text += processed + "\n"
+                visual_text = visual_text.strip() or "No visual text detected."
+
             prompt = VIDEO_QA_PROMPT.format(
                 audio_transcription=audio_transcript,
+                visual_text=visual_text,
                 question=question,
             )
 

{camel_ai-0.2.68.dist-info → camel_ai-0.2.69a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: camel-ai
-Version: 0.2.68
+Version: 0.2.69a2
 Summary: Communicative Agents for AI Society Study
 Project-URL: Homepage, https://www.camel-ai.org/
 Project-URL: Repository, https://github.com/camel-ai/camel
@@ -99,6 +99,7 @@ Requires-Dist: pymupdf<2,>=1.22.5; extra == 'all'
 Requires-Dist: pyobvector>=0.1.18; extra == 'all'
 Requires-Dist: pyowm<4,>=3.3.0; extra == 'all'
 Requires-Dist: pytelegrambotapi<5,>=4.18.0; extra == 'all'
+Requires-Dist: pytesseract>=0.3.13; extra == 'all'
 Requires-Dist: pytest-asyncio<0.24,>=0.23.0; extra == 'all'
 Requires-Dist: pytest-cov<5,>=4; extra == 'all'
 Requires-Dist: pytest<8,>=7; extra == 'all'
@@ -226,6 +227,7 @@ Provides-Extra: media-tools
 Requires-Dist: ffmpeg-python<0.3,>=0.2.0; extra == 'media-tools'
 Requires-Dist: imageio[pyav]<3,>=2.34.2; extra == 'media-tools'
 Requires-Dist: pydub<0.26,>=0.25.1; extra == 'media-tools'
+Requires-Dist: pytesseract>=0.3.13; extra == 'media-tools'
 Requires-Dist: scenedetect>=0.6.5.2; extra == 'media-tools'
 Requires-Dist: yt-dlp<2025,>=2024.11.4; extra == 'media-tools'
 Provides-Extra: model-platforms
@@ -252,6 +254,7 @@ Requires-Dist: ffmpeg-python<0.3,>=0.2.0; extra == 'owl'
 Requires-Dist: fpdf>=1.7.2; extra == 'owl'
 Requires-Dist: html2text>=2024.2.26; extra == 'owl'
 Requires-Dist: imageio[pyav]<3,>=2.34.2; extra == 'owl'
+Requires-Dist: markitdown==0.1.1; extra == 'owl'
 Requires-Dist: mcp-server-fetch==2025.1.17; extra == 'owl'
 Requires-Dist: mcp-simple-arxiv==0.2.2; extra == 'owl'
 Requires-Dist: newspaper3k<0.3,>=0.2.8; extra == 'owl'
@@ -266,6 +269,7 @@ Requires-Dist: pyautogui<0.10,>=0.9.54; extra == 'owl'
 Requires-Dist: pydub<0.26,>=0.25.1; extra == 'owl'
 Requires-Dist: pylatex>=1.4.2; extra == 'owl'
 Requires-Dist: pymupdf<2,>=1.22.5; extra == 'owl'
+Requires-Dist: pytesseract>=0.3.13; extra == 'owl'
 Requires-Dist: python-dotenv<2,>=1.0.0; extra == 'owl'
 Requires-Dist: python-pptx>=1.0.2; extra == 'owl'
 Requires-Dist: requests-oauthlib<2,>=1.3.1; extra == 'owl'