camel-ai 0.2.69a7__py3-none-any.whl → 0.2.71a1__py3-none-any.whl

camel/societies/workforce/workforce_logger.py

@@ -488,7 +488,6 @@ class WorkforceLogger:
             'worker_utilization': {},
             'current_pending_tasks': 0,
             'total_workforce_running_time_seconds': 0.0,
-            'avg_task_queue_time_seconds': 0.0,
         }
 
         task_start_times: Dict[str, float] = {}
@@ -499,54 +498,68 @@ class WorkforceLogger:
 
         tasks_handled_by_worker: Dict[str, int] = {}
 
+        # Helper function to check if a task is the main task (has no parent)
+        def is_main_task(task_id: str) -> bool:
+            return (
+                task_id in self._task_hierarchy
+                and self._task_hierarchy[task_id].get('parent') is None
+            )
+
         for entry in self.log_entries:
             event_type = entry['event_type']
             timestamp = datetime.fromisoformat(entry['timestamp'])
+            task_id = entry.get('task_id', '')
+
             if first_timestamp is None or timestamp < first_timestamp:
                 first_timestamp = timestamp
             if last_timestamp is None or timestamp > last_timestamp:
                 last_timestamp = timestamp
 
             if event_type == 'task_created':
-                kpis['total_tasks_created'] += 1
-                task_creation_timestamps[entry['task_id']] = timestamp
+                # Exclude main task from total count
+                if not is_main_task(task_id):
+                    kpis['total_tasks_created'] += 1
+                task_creation_timestamps[task_id] = timestamp
             elif event_type == 'task_assigned':
-                task_assignment_timestamps[entry['task_id']] = timestamp
+                task_assignment_timestamps[task_id] = timestamp
                 # Queue time tracking has been removed
 
             elif event_type == 'task_started':
                 # Store start time for processing time calculation
-                task_start_times[entry['task_id']] = timestamp.timestamp()
+                task_start_times[task_id] = timestamp.timestamp()
 
             elif event_type == 'task_completed':
-                kpis['total_tasks_completed'] += 1
-                # Count tasks handled by worker
-                if 'worker_id' in entry and entry['worker_id'] is not None:
-                    worker_id = entry['worker_id']
-                    tasks_handled_by_worker[worker_id] = (
-                        tasks_handled_by_worker.get(worker_id, 0) + 1
-                    )
+                # Exclude main task from total count
+                if not is_main_task(task_id):
+                    kpis['total_tasks_completed'] += 1
+                    # Count tasks handled by worker (only for non-main tasks)
+                    if 'worker_id' in entry and entry['worker_id'] is not None:
+                        worker_id = entry['worker_id']
+                        tasks_handled_by_worker[worker_id] = (
+                            tasks_handled_by_worker.get(worker_id, 0) + 1
+                        )
 
-                if entry['task_id'] in task_assignment_timestamps:
+                if task_id in task_assignment_timestamps:
                     completion_time = (
-                        timestamp
-                        - task_assignment_timestamps[entry['task_id']]
+                        timestamp - task_assignment_timestamps[task_id]
                     ).total_seconds()
                     # Store completion time in task hierarchy instead of KPIs
                     # array
-                    if entry['task_id'] in self._task_hierarchy:
-                        self._task_hierarchy[entry['task_id']][
+                    if task_id in self._task_hierarchy:
+                        self._task_hierarchy[task_id][
                             'completion_time_seconds'
                         ] = completion_time
 
             elif event_type == 'task_failed':
-                kpis['total_tasks_failed'] += 1
-                # Count tasks handled by worker (also for failed tasks)
-                if 'worker_id' in entry and entry['worker_id'] is not None:
-                    worker_id = entry['worker_id']
-                    tasks_handled_by_worker[worker_id] = (
-                        tasks_handled_by_worker.get(worker_id, 0) + 1
-                    )
+                # Exclude main task from total count
+                if not is_main_task(task_id):
+                    kpis['total_tasks_failed'] += 1
+                    # Count tasks handled by worker (only for non-main tasks)
+                    if 'worker_id' in entry and entry['worker_id'] is not None:
+                        worker_id = entry['worker_id']
+                        tasks_handled_by_worker[worker_id] = (
+                            tasks_handled_by_worker.get(worker_id, 0) + 1
+                        )
                 error_type = entry['error_type']
                 kpis['error_types_count'][error_type] = (
                     kpis['error_types_count'].get(error_type, 0) + 1

camel/storages/__init__.py

@@ -30,6 +30,7 @@ from .vectordb_storages.chroma import ChromaStorage
 from .vectordb_storages.faiss import FaissStorage
 from .vectordb_storages.milvus import MilvusStorage
 from .vectordb_storages.oceanbase import OceanBaseStorage
+from .vectordb_storages.pgvector import PgVectorStorage
 from .vectordb_storages.qdrant import QdrantStorage
 from .vectordb_storages.tidb import TiDBStorage
 from .vectordb_storages.weaviate import WeaviateStorage
@@ -53,5 +54,6 @@ __all__ = [
     'Mem0Storage',
     'OceanBaseStorage',
     'WeaviateStorage',
+    'PgVectorStorage',
     'ChromaStorage',
 ]

camel/storages/vectordb_storages/__init__.py

@@ -23,6 +23,7 @@ from .chroma import ChromaStorage
 from .faiss import FaissStorage
 from .milvus import MilvusStorage
 from .oceanbase import OceanBaseStorage
+from .pgvector import PgVectorStorage
 from .qdrant import QdrantStorage
 from .tidb import TiDBStorage
 from .weaviate import WeaviateStorage
@@ -40,4 +41,5 @@ __all__ = [
     'WeaviateStorage',
     'VectorRecord',
     'VectorDBStatus',
+    'PgVectorStorage',
 ]

camel/storages/vectordb_storages/pgvector.py

@@ -0,0 +1,349 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import json
+from typing import Any, Dict, List, Optional
+
+from camel.logger import get_logger
+from camel.storages.vectordb_storages import (
+    BaseVectorStorage,
+    VectorDBQuery,
+    VectorDBQueryResult,
+    VectorDBStatus,
+    VectorRecord,
+)
+from camel.types import VectorDistance
+from camel.utils import dependencies_required
+
+logger = get_logger(__name__)
+
+
+class PgVectorStorage(BaseVectorStorage):
+    r"""PgVectorStorage is an implementation of BaseVectorStorage for
+    PostgreSQL with pgvector extension.
+
+    This class provides methods to add, delete, query, and manage vector
+    records in a PostgreSQL database using the pgvector extension.
+    It supports different distance metrics for similarity search.
+
+    Args:
+        vector_dim (int): The dimension of the vectors to be stored.
+        conn_info (Dict[str, Any]): Connection information for
+            psycopg2.connect.
+        table_name (str, optional): Name of the table to store vectors.
+            (default: :obj:`None`)
+        distance (VectorDistance, optional): Distance metric for vector
+            comparison. (default: :obj:`VectorDistance.COSINE`)
+    """
+
+    @dependencies_required('psycopg', 'pgvector')
+    def __init__(
+        self,
+        vector_dim: int,
+        conn_info: Dict[str, Any],
+        table_name: Optional[str] = None,
+        distance: VectorDistance = VectorDistance.COSINE,
+        **kwargs: Any,
+    ) -> None:
+        r"""Initialize PgVectorStorage.
+
+        Args:
+            vector_dim (int): The dimension of the vectors.
+            conn_info (Dict[str, Any]): Connection info for psycopg2.connect.
+            table_name (str, optional): Table name. (default: :obj:`None`)
+            distance (VectorDistance, optional): Distance metric.
+                (default: :obj:`VectorDistance.COSINE`)
+        """
+        import psycopg
+        from pgvector.psycopg import register_vector
+
+        if vector_dim <= 0:
+            raise ValueError("vector_dim must be positive")
+
+        self.vector_dim = vector_dim
+        self.conn_info = conn_info
+        self.table_name = table_name or 'vectors'
+        self.distance = distance
+
+        try:
+            self._conn = psycopg.connect(**conn_info)
+            register_vector(self._conn)
+            self._ensure_table()
+            self._ensure_index()
+        except Exception as e:
+            logger.error(f"Failed to initialize PgVectorStorage: {e}")
+            raise
+
+    def _ensure_table(self) -> None:
+        r"""Ensure the vector table exists in the database.
+        Creates the table if it does not exist.
+        """
+        try:
+            from psycopg.sql import SQL, Identifier, Literal
+
+            with self._conn.cursor() as cur:
+                query = SQL("""
+                    CREATE TABLE IF NOT EXISTS {table} (
+                        id VARCHAR PRIMARY KEY,
+                        vector vector({dim}),
+                        payload JSONB
+                    )
+                """).format(
+                    table=Identifier(self.table_name),
+                    dim=Literal(self.vector_dim),
+                )
+                cur.execute(query)
+                self._conn.commit()
+        except Exception as e:
+            logger.error(f"Failed to create table {self.table_name}: {e}")
+            raise
+
+    def _ensure_index(self) -> None:
+        r"""Ensure vector similarity search index exists for better
+        performance.
+        """
+        try:
+            from psycopg.sql import SQL, Identifier
+
+            with self._conn.cursor() as cur:
+                index_name = f"{self.table_name}_vector_idx"
+                query = SQL("""
+                    CREATE INDEX IF NOT EXISTS {index_name} 
+                    ON {table} 
+                    USING hnsw (vector vector_cosine_ops)
+                """).format(
+                    index_name=Identifier(index_name),
+                    table=Identifier(self.table_name),
+                )
+                cur.execute(query)
+                self._conn.commit()
+        except Exception as e:
+            logger.warning(f"Failed to create vector index: {e}")
+
+    def add(self, records: List[VectorRecord], **kwargs: Any) -> None:
+        r"""Add or update vector records in the database.
+
+        Args:
+            records (List[VectorRecord]): List of vector records to
+                add or update.
+        """
+        if not records:
+            return
+
+        try:
+            with self._conn.cursor() as cur:
+                # Use batch insert for better performance
+                batch_data = []
+                for rec in records:
+                    if len(rec.vector) != self.vector_dim:
+                        raise ValueError(
+                            f"Vector dimension mismatch: expected "
+                            f"{self.vector_dim}, got {len(rec.vector)}"
+                        )
+
+                    batch_data.append(
+                        (
+                            rec.id,
+                            rec.vector,
+                            json.dumps(rec.payload)
+                            if rec.payload is not None
+                            else None,
+                        )
+                    )
+
+                # Use executemany for efficient batch insert
+                from psycopg.sql import SQL, Identifier
+
+                query = SQL("""
+                    INSERT INTO {table} (id, vector, payload)
+                    VALUES (%s, %s, %s)
+                    ON CONFLICT (id) DO UPDATE SET 
+                    vector=EXCLUDED.vector, 
+                    payload=EXCLUDED.payload
+                """).format(table=Identifier(self.table_name))
+
+                cur.executemany(query, batch_data)
+                self._conn.commit()
+        except Exception as e:
+            self._conn.rollback()
+            logger.error(f"Failed to add records: {e}")
+            raise
+
+    def delete(self, ids: List[str], **kwargs: Any) -> None:
+        r"""Delete vector records from the database by their IDs.
+
+        Args:
+            ids (List[str]): List of record IDs to delete.
+        """
+        from psycopg.sql import SQL, Identifier
+
+        if not ids:
+            return
+
+        try:
+            with self._conn.cursor() as cur:
+                query = SQL("DELETE FROM {table} WHERE id = ANY(%s)").format(
+                    table=Identifier(self.table_name)
+                )
+                cur.execute(query, (ids,))
+                self._conn.commit()
+        except Exception as e:
+            self._conn.rollback()
+            logger.error(f"Failed to delete records: {e}")
+            raise
+
+    def query(
+        self, query: VectorDBQuery, **kwargs: Any
+    ) -> List[VectorDBQueryResult]:
+        r"""Query the database for the most similar vectors to the given
+        query vector.
+
+        Args:
+            query (VectorDBQuery): Query object containing the query
+                vector and top_k.
+            **kwargs (Any): Additional keyword arguments for the query.
+
+        Returns:
+            List[VectorDBQueryResult]: List of query results sorted by
+                similarity.
+        """
+        if len(query.query_vector) != self.vector_dim:
+            raise ValueError(
+                f"Query vector dimension mismatch: "
+                f"expected {self.vector_dim}, got {len(query.query_vector)}"
+            )
+
+        try:
+            with self._conn.cursor() as cur:
+                # Fix distance metric mapping
+                metric_info = {
+                    VectorDistance.COSINE: ('<=>', 'ASC'),  # Cosine distance
+                    VectorDistance.EUCLIDEAN: (
+                        '<->',
+                        'ASC',
+                    ),  # Euclidean distance
+                    VectorDistance.DOT: (
+                        '<#>',
+                        'DESC',
+                    ),  # Negative dot product (higher is better)
+                }
+
+                if self.distance not in metric_info:
+                    raise ValueError(
+                        f"Unsupported distance metric: {self.distance}"
+                    )
+
+                metric, order = metric_info[self.distance]
+
+                from psycopg.sql import SQL, Identifier, Literal
+
+                query_sql = SQL("""
+                    SELECT id, vector, payload, (vector {} %s::vector) 
+                    AS similarity
+                    FROM {}
+                    ORDER BY similarity {}
+                    LIMIT %s
+                """).format(
+                    Literal(metric),
+                    Identifier(self.table_name),
+                    Literal(order),
+                )
+
+                cur.execute(query_sql, (query.query_vector, query.top_k))
+                results = []
+                for row in cur.fetchall():
+                    id, vector, payload, similarity = row
+                    results.append(
+                        VectorDBQueryResult.create(
+                            similarity=float(similarity),
+                            vector=list(vector),
+                            id=id,
+                            payload=payload,
+                        )
+                    )
+                return results
+        except Exception as e:
+            logger.error(f"Failed to query vectors: {e}")
+            raise
+
+    def status(self, **kwargs: Any) -> VectorDBStatus:
+        r"""Get the status of the vector database, including vector
+        dimension and count.
+
+        Args:
+            **kwargs (Any): Additional keyword arguments for the query.
+
+        Returns:
+            VectorDBStatus: Status object with vector dimension and count.
+        """
+        try:
+            with self._conn.cursor() as cur:
+                from psycopg.sql import SQL, Identifier
+
+                query = SQL('SELECT COUNT(*) FROM {}').format(
+                    Identifier(self.table_name)
+                )
+                cur.execute(query)
+                result = cur.fetchone()
+                count = result[0] if result else 0
+                return VectorDBStatus(
+                    vector_dim=self.vector_dim, vector_count=count
+                )
+        except Exception as e:
+            logger.error(f"Failed to get status: {e}")
+            raise
+
+    def clear(self) -> None:
+        r"""Remove all vectors from the storage by truncating the table."""
+        try:
+            with self._conn.cursor() as cur:
+                from psycopg.sql import SQL, Identifier
+
+                query = SQL("TRUNCATE TABLE {table}").format(
+                    table=Identifier(self.table_name)
+                )
+                cur.execute(query)
+                self._conn.commit()
+        except Exception as e:
+            self._conn.rollback()
+            logger.error(f"Failed to clear table: {e}")
+            raise
+
+    def load(self) -> None:
+        r"""Load the collection hosted on cloud service (no-op for pgvector).
+        This method is provided for interface compatibility.
+        """
+        # For PostgreSQL local/managed instances, no loading is required
+        pass
+
+    def close(self) -> None:
+        r"""Close the database connection."""
+        if hasattr(self, '_conn') and self._conn:
+            try:
+                self._conn.close()
+            except Exception as e:
+                logger.warning(f"Error closing connection: {e}")
+
+    def __del__(self) -> None:
+        r"""Ensure connection is closed when object is destroyed."""
+        self.close()
+
+    @property
+    def client(self) -> Any:
+        r"""Provides access to the underlying vector database client.
+
+        Returns:
+            Any: The underlying psycopg connection object.
+        """
+        return self._conn

camel/tasks/task.py

@@ -46,19 +46,35 @@ from .task_prompt import (
 logger = get_logger(__name__)
 
 
+class TaskValidationMode(Enum):
+    r"""Validation modes for different use cases."""
+
+    INPUT = "input"  # For validating task content before processing
+    OUTPUT = "output"  # For validating task results after completion
+
+
 def validate_task_content(
-    content: str, task_id: str = "unknown", min_length: int = 10
+    content: str,
+    task_id: str = "unknown",
+    min_length: int = 5,
+    mode: TaskValidationMode = TaskValidationMode.INPUT,
+    check_failure_patterns: bool = True,
 ) -> bool:
-    r"""Validates task result content to avoid silent failures.
-    It performs basic checks to ensure the content meets minimum
-    quality standards.
+    r"""Unified validation for task content and results to avoid silent
+    failures. Performs comprehensive checks to ensure content meets quality
+    standards.
 
     Args:
-        content (str): The task result content to validate.
+        content (str): The task content or result to validate.
         task_id (str): Task ID for logging purposes.
             (default: :obj:`"unknown"`)
         min_length (int): Minimum content length after stripping whitespace.
-            (default: :obj:`10`)
+            (default: :obj:`5`)
+        mode (TaskValidationMode): Validation mode - INPUT for task content,
+            OUTPUT for task results. (default: :obj:`TaskValidationMode.INPUT`)
+        check_failure_patterns (bool): Whether to check for failure indicators
+            in the content. Only effective in OUTPUT mode.
+            (default: :obj:`True`)
 
     Returns:
         bool: True if content passes validation, False otherwise.
@@ -85,14 +101,70 @@ def validate_task_content(
         )
         return False
 
+    # 4: For OUTPUT mode, check for failure patterns if enabled
+    if mode == TaskValidationMode.OUTPUT and check_failure_patterns:
+        content_lower = stripped_content.lower()
+
+        # Check for explicit failure indicators
+        failure_indicators = [
+            "i cannot complete",
+            "i cannot do",
+            "task failed",
+            "unable to complete",
+            "cannot be completed",
+            "failed to complete",
+            "i cannot",
+            "not possible",
+            "impossible to",
+            "cannot perform",
+        ]
+
+        if any(indicator in content_lower for indicator in failure_indicators):
+            logger.warning(
+                f"Task {task_id}: Failure indicator detected in result. "
+                f"Content preview: '{stripped_content[:100]}...'"
+            )
+            return False
+
+        # Check for responses that are just error messages or refusals
+        if content_lower.startswith(("error", "failed", "cannot", "unable")):
+            logger.warning(
+                f"Task {task_id}: Error/refusal pattern detected at start. "
+                f"Content preview: '{stripped_content[:100]}...'"
+            )
+            return False
+
     # All validation checks passed
     logger.debug(
-        f"Task {task_id}: Content validation passed "
+        f"Task {task_id}: {mode.value} validation passed "
         f"({len(stripped_content)} chars)"
     )
     return True
 
 
+def is_task_result_insufficient(task: "Task") -> bool:
+    r"""Check if a task result is insufficient and should be treated as failed.
+
+    This is a convenience wrapper around validate_task_content for backward
+    compatibility and semantic clarity when checking task results.
+
+    Args:
+        task (Task): The task to check.
+
+    Returns:
+        bool: True if the result is insufficient, False otherwise.
+    """
+    if not hasattr(task, 'result') or task.result is None:
+        return True
+
+    return not validate_task_content(
+        content=task.result,
+        task_id=task.id,
+        mode=TaskValidationMode.OUTPUT,
+        check_failure_patterns=True,
+    )
+
+
 def parse_response(
     response: str, task_id: Optional[str] = None
 ) -> List["Task"]:
@@ -157,6 +229,8 @@ class Task(BaseModel):
             (default: :obj:`""`)
         failure_count (int): The failure count for the task.
             (default: :obj:`0`)
+        assigned_worker_id (Optional[str]): The ID of the worker assigned to
+            this task. (default: :obj:`None`)
         additional_info (Optional[Dict[str, Any]]): Additional information for
             the task. (default: :obj:`None`)
         image_list (Optional[List[Image.Image]]): Optional list of PIL Image
@@ -187,6 +261,8 @@ class Task(BaseModel):
 
     failure_count: int = 0
 
+    assigned_worker_id: Optional[str] = None
+
     additional_info: Optional[Dict[str, Any]] = None
 
     image_list: Optional[List[Image.Image]] = None

camel/toolkits/file_write_toolkit.py

@@ -176,26 +176,40 @@ class FileWriteToolkit(BaseToolkit):
 
             doc = Document(documentclass="article")
             doc.packages.append(Command('usepackage', 'amsmath'))
-
             with doc.create(Section('Generated Content')):
                 for line in content.split('\n'):
-                    # Remove leading whitespace
                     stripped_line = line.strip()
-                    # Check if the line is intended as a standalone math
-                    # expression
+
+                    # Skip empty lines
+                    if not stripped_line:
+                        continue
+
+                    # Convert Markdown-like headers
+                    if stripped_line.startswith('## '):
+                        header = stripped_line[3:]
+                        doc.append(NoEscape(r'\subsection*{%s}' % header))
+                        continue
+                    elif stripped_line.startswith('# '):
+                        header = stripped_line[2:]
+                        doc.append(NoEscape(r'\section*{%s}' % header))
+                        continue
+                    elif stripped_line.strip() == '---':
+                        doc.append(NoEscape(r'\hrule'))
+                        continue
+
+                    # Detect standalone math expressions like $...$
                     if (
                         stripped_line.startswith('$')
                         and stripped_line.endswith('$')
                         and len(stripped_line) > 1
                     ):
-                        # Extract content between the '$' delimiters
                         math_data = stripped_line[1:-1]
                         doc.append(Math(data=math_data))
                     else:
-                        doc.append(NoEscape(line))
+                        doc.append(NoEscape(stripped_line))
                     doc.append(NoEscape(r'\par'))
 
-            doc.generate_pdf(str(file_path), clean_tex=False)
+                doc.generate_pdf(str(file_path), clean_tex=True)
 
             logger.info(f"Wrote PDF (with LaTeX) to {file_path}")
         else: