aiecs 1.0.0__py3-none-any.whl

aiecs/tools/task_tools/classfire_tool.py

@@ -0,0 +1,901 @@
+import os
+import re
+import logging
+import asyncio
+import time
+from typing import Dict, Any, List, Optional, Union, Tuple
+from enum import Enum
+
+from pydantic import BaseModel, Field, field_validator, ValidationError, ConfigDict
+
+# Lazy imports for heavy dependencies
+rake_nltk = None
+spacy = None
+
+def _init_heavy_dependencies():
+    """Initialize heavy dependencies when actually needed"""
+    global rake_nltk, spacy
+    
+    if rake_nltk is None:
+        try:
+            import rake_nltk as _rake_nltk
+            rake_nltk = _rake_nltk
+        except ImportError:
+            import logging
+            logging.getLogger(__name__).error("rake_nltk not available")
+            
+    if spacy is None:
+        try:
+            import spacy as _spacy
+            spacy = _spacy
+        except ImportError:
+            import logging
+            logging.getLogger(__name__).warning("spacy not available (optional)")
+
+from aiecs.tools import register_tool
+from aiecs.tools.base_tool import BaseTool
+from aiecs.tools.tool_executor import (
+    validate_input,
+)
+
+# Enums for configuration options
+class Language(str, Enum):
+    ENGLISH = "en"
+    CHINESE = "zh"
+    AUTO = "auto"
+
+class ModelType(str, Enum):
+    SPACY_ENGLISH = "en_core_web_sm"
+    SPACY_CHINESE = "zh_core_web_sm"
+
+@register_tool("classifier")
+class ClassifierTool(BaseTool):
+    """
+    Text classification, tokenization, POS tagging, NER, lemmatization, dependency parsing,
+    keyword extraction, and summarization tool.
+
+    Operations:
+      - classify: Sentiment or topic classification.
+      - tokenize: Tokenize text.
+      - pos_tag: Part-of-speech tagging.
+      - ner: Named entity recognition.
+      - lemmatize: Lemmatize tokens.
+      - dependency_parse: Dependency parsing.
+      - keyword_extract: Extract key phrases.
+      - summarize: Summarize text.
+      - batch_process: Process multiple texts with any operation.
+
+    Supports English (spaCy) and Chinese (Jieba, spaCy).
+    """
+
+    # Configuration schema
+    class Config(BaseModel):
+        """Configuration for the classifier tool"""
+        max_workers: int = Field(
+            default=min(32, (os.cpu_count() or 4) * 2),
+            description="Maximum number of worker threads"
+        )
+        pipeline_cache_ttl: int = Field(
+            default=3600,
+            description="Time-to-live for pipeline cache in seconds"
+        )
+        pipeline_cache_size: int = Field(
+            default=10,
+            description="Maximum number of pipeline cache entries"
+        )
+        max_text_length: int = Field(
+            default=10_000,
+            description="Maximum text length in characters"
+        )
+        spacy_model_en: str = Field(
+            default="en_core_web_sm",
+            description="spaCy model for English"
+        )
+        spacy_model_zh: str = Field(
+            default="zh_core_web_sm",
+            description="spaCy model for Chinese"
+        )
+        allowed_models: List[str] = Field(
+            default=[
+                "en_core_web_sm",
+                "zh_core_web_sm"
+            ],
+            description="List of allowed spaCy models"
+        )
+        rate_limit_enabled: bool = Field(
+            default=True,
+            description="Enable rate limiting"
+        )
+        rate_limit_requests: int = Field(
+            default=100,
+            description="Maximum requests per window"
+        )
+        rate_limit_window: int = Field(
+            default=60,
+            description="Rate limit window in seconds"
+        )
+        use_rake_for_english: bool = Field(
+            default=True,
+            description="Use RAKE for English phrase extraction"
+        )
+
+        model_config = ConfigDict(env_prefix="CLASSIFIER_TOOL_")
+
+    # Base schema for text operations
+    class BaseTextSchema(BaseModel):
+        """Base schema for text operations"""
+        text: str = Field(
+            description="Text to process"
+        )
+
+        @field_validator("text")
+        @classmethod
+        def check_length_and_content(cls, v: str) -> str:
+            if len(v) > 10_000:  # Using a constant here for validation
+                raise ValueError(f"Text length exceeds 10,000 characters")
+            # Check for malicious patterns (e.g., SQL injection)
+            if re.search(r'(\bSELECT\b|\bINSERT\b|\bDELETE\b|--|;|/\*)', v, re.IGNORECASE):
+                raise ValueError("Text contains potentially malicious content")
+            return v
+
+    # Input schemas for operations
+    class ClassifySchema(BaseTextSchema):
+        """Schema for text classification"""
+        model: Optional[str] = Field(
+            default=None,
+            description="Model to use for classification"
+        )
+        language: Optional[Language] = Field(
+            default=None,
+            description="Language of the text"
+        )
+
+        @field_validator("model")
+        @classmethod
+        def check_model(cls, v: Optional[str]) -> Optional[str]:
+            allowed_models = [
+                "en_core_web_sm",
+                "zh_core_web_sm"
+            ]
+            if v and v not in allowed_models:
+                raise ValueError(f"Model '{v}' not in allowed spaCy models: {allowed_models}")
+            return v
+
+    class TokenizeSchema(BaseTextSchema):
+        """Schema for text tokenization"""
+        language: Optional[Language] = Field(
+            default=None,
+            description="Language of the text"
+        )
+
+    class PosTagSchema(BaseTextSchema):
+        """Schema for part-of-speech tagging"""
+        language: Optional[Language] = Field(
+            default=None,
+            description="Language of the text"
+        )
+
+    class NERSchema(BaseTextSchema):
+        """Schema for named entity recognition"""
+        language: Optional[Language] = Field(
+            default=None,
+            description="Language of the text"
+        )
+
+    class LemmatizeSchema(BaseTextSchema):
+        """Schema for lemmatization"""
+        language: Optional[Language] = Field(
+            default=None,
+            description="Language of the text"
+        )
+
+    class DependencyParseSchema(BaseTextSchema):
+        """Schema for dependency parsing"""
+        language: Optional[Language] = Field(
+            default=None,
+            description="Language of the text"
+        )
+
+    class KeywordExtractSchema(BaseTextSchema):
+        """Schema for keyword extraction"""
+        top_k: int = Field(
+            default=10,
+            description="Number of keywords to extract"
+        )
+        language: Optional[Language] = Field(
+            default=None,
+            description="Language of the text"
+        )
+        extract_phrases: bool = Field(
+            default=True,
+            description="Whether to extract phrases or just keywords"
+        )
+
+    class SummarizeSchema(BaseTextSchema):
+        """Schema for text summarization"""
+        max_length: int = Field(
+            default=150,
+            description="Maximum length of the summary"
+        )
+        language: Optional[Language] = Field(
+            default=None,
+            description="Language of the text"
+        )
+
+    class BatchProcessSchema(BaseModel):
+        """Schema for batch processing"""
+        texts: List[str] = Field(
+            description="List of texts to process"
+        )
+        operation: str = Field(
+            description="Operation to perform on each text"
+        )
+        language: Optional[Language] = Field(
+            default=None,
+            description="Language of the texts"
+        )
+        model: Optional[str] = Field(
+            default=None,
+            description="Model to use for processing"
+        )
+        top_k: Optional[int] = Field(
+            default=None,
+            description="Number of keywords to extract (for keyword_extract)"
+        )
+        max_length: Optional[int] = Field(
+            default=None,
+            description="Maximum length of the summary (for summarize)"
+        )
+
+        @field_validator("texts")
+        @classmethod
+        def check_texts(cls, v: List[str]) -> List[str]:
+            for text in v:
+                if len(text) > 10_000:  # Using a constant here for validation
+                    raise ValueError(f"Text length exceeds 10,000 characters")
+                if re.search(r'(\bSELECT\b|\bINSERT\b|\bDELETE\b|--|;|/\*)', text, re.IGNORECASE):
+                    raise ValueError("Text contains potentially malicious content")
+            return v
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize ClassifierTool with settings and resources.
+
+        Args:
+            config (Dict, optional): Configuration overrides for ClassifierSettings.
+
+        Raises:
+            ValueError: If config contains invalid settings.
+        """
+        super().__init__(config)
+
+        # Parse configuration
+        self.config = self.Config(**(config or {}))
+
+        # Set up logger
+        self.logger = logging.getLogger(__name__)
+        if not self.logger.handlers:
+            handler = logging.StreamHandler()
+            handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
+            self.logger.addHandler(handler)
+        self.logger.setLevel(logging.INFO)
+
+        # Initialize resources
+        self._spacy_nlp = {}  # Language -> spaCy pipeline
+        self._metrics = {'requests': 0, 'cache_hits': 0, 'processing_time': []}
+        self._request_timestamps = []
+
+    def _get_sentiment_lexicon(self, language: str) -> Dict[str, float]:
+        """
+        Get sentiment lexicon for the specified language.
+
+        Args:
+            language (str): Language code ('en', 'zh').
+
+        Returns:
+            Dict[str, float]: Sentiment lexicon with word -> score mapping.
+        """
+        if language == 'en':
+            # Simple English sentiment lexicon
+            return {
+                'good': 1.0, 'great': 1.5, 'excellent': 2.0, 'amazing': 2.0, 'wonderful': 1.5,
+                'fantastic': 2.0, 'awesome': 1.5, 'perfect': 2.0, 'love': 1.5, 'like': 1.0,
+                'happy': 1.5, 'pleased': 1.0, 'satisfied': 1.0, 'positive': 1.0, 'best': 2.0,
+                'bad': -1.0, 'terrible': -2.0, 'awful': -2.0, 'horrible': -2.0, 'hate': -2.0,
+                'dislike': -1.0, 'sad': -1.5, 'angry': -1.5, 'disappointed': -1.5, 'negative': -1.0,
+                'worst': -2.0, 'poor': -1.0, 'fail': -1.5, 'wrong': -1.0, 'problem': -1.0
+            }
+        else:  # Chinese
+            return {
+                '好': 1.0, '很好': 1.5, '非常好': 2.0, '棒': 1.5, '优秀': 2.0, '完美': 2.0,
+                '喜欢': 1.5, '爱': 2.0, '满意': 1.0, '开心': 1.5, '高兴': 1.5, '积极': 1.0,
+                '坏': -1.0, '很坏': -1.5, '糟糕': -2.0, '讨厌': -2.0, '恨': -2.0, '失望': -1.5,
+                '生气': -1.5, '愤怒': -2.0, '消极': -1.0, '问题': -1.0, '错误': -1.0, '失败': -1.5
+            }
+
+    def _get_spacy(self, language: str) -> Any:
+        """
+        Get a spaCy pipeline for the specified language.
+
+        Args:
+            language (str): Language code ('en', 'zh').
+
+        Returns:
+            Any: spaCy NLP object.
+        """
+        global spacy
+        if spacy is None:
+            try:
+                import spacy as spacy_module
+                spacy = spacy_module
+            except ImportError:
+                raise ImportError("spaCy is required but not installed. Please install it with: pip install spacy")
+
+        model = self.config.spacy_model_zh if language == 'zh' else self.config.spacy_model_en
+        return spacy.load(model, disable=["textcat"])
+
+    def _detect_language(self, text: str) -> str:
+        """
+        Detect the language of the input text using character analysis.
+
+        Args:
+            text (str): Input text.
+
+        Returns:
+            str: Language code ('en', 'zh', or 'en' for unknown).
+        """
+        try:
+            # Count Chinese characters (CJK Unified Ideographs)
+            chinese_chars = sum(1 for char in text if '\u4e00' <= char <= '\u9fff')
+            total_chars = len([char for char in text if char.isalpha()])
+
+            if total_chars == 0:
+                return 'en'
+
+            # If more than 30% are Chinese characters, consider it Chinese
+            chinese_ratio = chinese_chars / total_chars
+            return 'zh' if chinese_ratio > 0.3 else 'en'
+        except Exception:
+            return 'en'
+
+    def _check_rate_limit(self) -> bool:
+        """
+        Check if the request is within rate limits.
+
+        Returns:
+            bool: True if within limits, False otherwise.
+        """
+        if not self.config.rate_limit_enabled:
+            return True
+
+        current_time = time.time()
+
+        # Get lock from executor
+        with self._executor.get_lock("rate_limit"):
+            # Remove timestamps outside the window
+            self._request_timestamps = [ts for ts in self._request_timestamps
+                                      if current_time - ts <= self.config.rate_limit_window]
+
+            # Check if we're at the limit
+            if len(self._request_timestamps) >= self.config.rate_limit_requests:
+                return False
+
+            # Add current timestamp
+            self._request_timestamps.append(current_time)
+            return True
+
+    def _extract_english_phrases(self, text: str, top_k: int) -> List[str]:
+        """
+        Extract key phrases from English text using RAKE.
+
+        Args:
+            text (str): Input text.
+            top_k (int): Number of phrases to extract.
+
+        Returns:
+            List[str]: Extracted phrases.
+        """
+        try:
+            # Initialize heavy dependencies if needed
+            _init_heavy_dependencies()
+            
+            if rake_nltk is None:
+                raise ImportError("rake_nltk not available")
+                
+            rake = rake_nltk.Rake()
+            rake.extract_keywords_from_text(text)
+            phrases = rake.get_ranked_phrases()[:top_k]
+            return phrases
+        except Exception as e:
+            self.logger.error(f"Error extracting English phrases: {e}")
+            # Fallback to simple keyword extraction
+            nlp = self._get_spacy('en')
+            doc = nlp(text)
+            keywords = [token.text for token in doc if token.pos_ in ('NOUN', 'PROPN')][:top_k]
+            return keywords
+
+    def _extract_chinese_phrases(self, text: str, top_k: int) -> List[str]:
+        """
+        Extract key phrases from Chinese text using spaCy.
+
+        Args:
+            text (str): Input text.
+            top_k (int): Number of phrases to extract.
+
+        Returns:
+            List[str]: Extracted phrases.
+        """
+        try:
+            nlp = self._get_spacy('zh')
+            doc = nlp(text)
+
+            # Extract noun phrases and named entities
+            phrases = []
+
+            # Add noun chunks
+            for chunk in doc.noun_chunks:
+                if len(chunk.text.strip()) > 1:
+                    phrases.append(chunk.text.strip())
+
+            # Add named entities
+            for ent in doc.ents:
+                if len(ent.text.strip()) > 1:
+                    phrases.append(ent.text.strip())
+
+            # Add important nouns and proper nouns
+            for token in doc:
+                if token.pos_ in ('NOUN', 'PROPN') and len(token.text.strip()) > 1:
+                    phrases.append(token.text.strip())
+
+            # Remove duplicates and return top_k
+            unique_phrases = list(dict.fromkeys(phrases))  # Preserve order
+            return unique_phrases[:top_k]
+
+        except Exception as e:
+            self.logger.error(f"Error extracting Chinese phrases with spaCy: {e}")
+            # Fallback to simple noun extraction
+            try:
+                nlp = self._get_spacy('zh')
+                doc = nlp(text)
+                nouns = [token.text for token in doc if token.pos_ in ('NOUN', 'PROPN')]
+                return nouns[:top_k]
+            except Exception:
+                return []
+
+    def _get_hf_pipeline(self, task: str, model: str):
+        """
+        Get a Hugging Face transformers pipeline for the specified task and model.
+
+        Args:
+            task (str): The task type (e.g., "summarization").
+            model (str): The model name.
+
+        Returns:
+            Any: Hugging Face pipeline object.
+
+        Raises:
+            ImportError: If transformers library is not available.
+            ValueError: If the pipeline creation fails.
+        """
+        try:
+            from transformers import pipeline
+            return pipeline(task, model=model)
+        except ImportError:
+            raise ImportError("transformers library is required for summarization but not installed. Please install it with: pip install transformers")
+        except Exception as e:
+            raise ValueError(f"Error creating pipeline for task '{task}' with model '{model}': {e}")
+
+    async def classify(self, text: str, model: Optional[str] = None, language: Optional[str] = None) -> List[Dict[str, Any]]:
+        """
+        Perform sentiment classification on text using spaCy and lexicon-based approach.
+
+        Args:
+            text (str): Text to classify.
+            model (Optional[str]): spaCy model to use (optional, auto-detected).
+            language (Optional[str]): Language of the text.
+
+        Returns:
+            List[Dict[str, Any]]: Classification results [{'label': str, 'score': float}].
+        """
+        if not self._check_rate_limit():
+            raise ValueError("Rate limit exceeded. Please try again later.")
+
+        language = language or self._detect_language(text)
+
+        # Get spaCy pipeline and sentiment lexicon
+        nlp = await asyncio.get_event_loop().run_in_executor(
+            None, self._get_spacy, language
+        )
+
+        sentiment_lexicon = self._get_sentiment_lexicon(language)
+
+        # Process text with spaCy
+        doc = await asyncio.get_event_loop().run_in_executor(
+            None, nlp, text
+        )
+
+        # Calculate sentiment score
+        sentiment_score = 0.0
+        word_count = 0
+
+        for token in doc:
+            if not token.is_stop and not token.is_punct and token.text.lower() in sentiment_lexicon:
+                sentiment_score += sentiment_lexicon[token.text.lower()]
+                word_count += 1
+
+        # Normalize score
+        if word_count > 0:
+            sentiment_score = sentiment_score / word_count
+
+        # Determine label and confidence
+        if sentiment_score > 0.1:
+            label = "POSITIVE"
+            confidence = min(0.9, 0.5 + abs(sentiment_score) * 0.4)
+        elif sentiment_score < -0.1:
+            label = "NEGATIVE"
+            confidence = min(0.9, 0.5 + abs(sentiment_score) * 0.4)
+        else:
+            label = "NEUTRAL"
+            confidence = 0.6
+
+        return [{"label": label, "score": confidence}]
+
+    async def tokenize(self, text: str, language: Optional[str] = None) -> List[str]:
+        """
+        Tokenize text into words or tokens using spaCy.
+
+        Args:
+            text (str): Text to tokenize.
+            language (Optional[str]): Language of the text.
+
+        Returns:
+            List[str]: List of tokens.
+        """
+        if not self._check_rate_limit():
+            raise ValueError("Rate limit exceeded. Please try again later.")
+
+        language = language or self._detect_language(text)
+
+        nlp = await asyncio.get_event_loop().run_in_executor(
+            None, self._get_spacy, language
+        )
+
+        doc = await asyncio.get_event_loop().run_in_executor(
+            None, nlp, text
+        )
+
+        return [token.text for token in doc]
+
+    async def pos_tag(self, text: str, language: Optional[str] = None) -> List[Tuple[str, str]]:
+        """
+        Perform part-of-speech tagging using spaCy, returning (token, pos) pairs.
+
+        Args:
+            text (str): Text to tag.
+            language (Optional[str]): Language of the text.
+
+        Returns:
+            List[Tuple[str, str]]: List of (token, POS tag) tuples.
+        """
+        if not self._check_rate_limit():
+            raise ValueError("Rate limit exceeded. Please try again later.")
+
+        language = language or self._detect_language(text)
+
+        nlp = await asyncio.get_event_loop().run_in_executor(
+            None, self._get_spacy, language
+        )
+
+        doc = await asyncio.get_event_loop().run_in_executor(
+            None, nlp, text
+        )
+
+        return [(token.text, token.pos_) for token in doc]
+
+    @validate_input(NERSchema)
+
+    async def ner(self, text: str, language: Optional[str] = None) -> List[Dict[str, Any]]:
+        """
+        Perform named entity recognition.
+
+        Args:
+            text (str): Text to analyze.
+            language (Optional[str]): Language of the text.
+
+        Returns:
+            List[Dict[str, Any]]: List of named entities with text, label, start, and end.
+        """
+        if not self._check_rate_limit():
+            raise ValueError("Rate limit exceeded. Please try again later.")
+
+        language = language or self._detect_language(text)
+
+        nlp = await asyncio.get_event_loop().run_in_executor(
+            None, self._get_spacy, language
+        )
+
+        doc = await asyncio.get_event_loop().run_in_executor(
+            None, nlp, text
+        )
+
+        return [
+            {"text": ent.text, "label": ent.label_, "start": ent.start_char, "end": ent.end_char}
+            for ent in doc.ents
+        ]
+
+    @validate_input(LemmatizeSchema)
+
+    async def lemmatize(self, text: str, language: Optional[str] = None) -> List[str]:
+        """
+        Lemmatize tokens in text using spaCy.
+
+        Args:
+            text (str): Text to lemmatize.
+            language (Optional[str]): Language of the text.
+
+        Returns:
+            List[str]: List of lemmatized tokens.
+        """
+        if not self._check_rate_limit():
+            raise ValueError("Rate limit exceeded. Please try again later.")
+
+        language = language or self._detect_language(text)
+
+        nlp = await asyncio.get_event_loop().run_in_executor(
+            None, self._get_spacy, language
+        )
+
+        doc = await asyncio.get_event_loop().run_in_executor(
+            None, nlp, text
+        )
+
+        # For Chinese, lemma might be the same as text, but spaCy handles it consistently
+        return [token.lemma_ for token in doc]
+
+    @validate_input(DependencyParseSchema)
+
+    async def dependency_parse(self, text: str, language: Optional[str] = None) -> List[Dict[str, Any]]:
+        """
+        Perform dependency parsing using spaCy (supports both English and Chinese).
+
+        Args:
+            text (str): Text to parse.
+            language (Optional[str]): Language of the text.
+
+        Returns:
+            List[Dict[str, Any]]: List of tokens with dependency information.
+        """
+        if not self._check_rate_limit():
+            raise ValueError("Rate limit exceeded. Please try again later.")
+
+        language = language or self._detect_language(text)
+
+        nlp = await asyncio.get_event_loop().run_in_executor(
+            None, self._get_spacy, language
+        )
+
+        doc = await asyncio.get_event_loop().run_in_executor(
+            None, nlp, text
+        )
+
+        return [
+            {
+                "text": token.text,
+                "head": token.head.text,
+                "dep": token.dep_,
+                "pos": token.pos_
+            }
+            for token in doc
+        ]
+
+    @validate_input(KeywordExtractSchema)
+
+    async def keyword_extract(self, text: str, top_k: int = 10, language: Optional[str] = None, extract_phrases: bool = True) -> List[str]:
+        """
+        Extract keywords or key phrases from text using spaCy.
+
+        Args:
+            text (str): Text to analyze.
+            top_k (int): Number of keywords to extract.
+            language (Optional[str]): Language of the text.
+            extract_phrases (bool): Whether to extract phrases or just keywords.
+
+        Returns:
+            List[str]: List of extracted keywords or phrases.
+        """
+        if not self._check_rate_limit():
+            raise ValueError("Rate limit exceeded. Please try again later.")
+
+        language = language or self._detect_language(text)
+
+        if language == 'zh':
+            if extract_phrases:
+                return await asyncio.get_event_loop().run_in_executor(
+                    None, self._extract_chinese_phrases, text, top_k
+                )
+            else:
+                # Extract simple keywords using spaCy
+                nlp = await asyncio.get_event_loop().run_in_executor(
+                    None, self._get_spacy, language
+                )
+
+                doc = await asyncio.get_event_loop().run_in_executor(
+                    None, nlp, text
+                )
+
+                keywords = [token.text for token in doc if token.pos_ in ('NOUN', 'PROPN')][:top_k]
+                return keywords
+        else:  # English or other languages
+            if extract_phrases and self.config.use_rake_for_english:
+                return await asyncio.get_event_loop().run_in_executor(
+                    None, self._extract_english_phrases, text, top_k
+                )
+            else:
+                nlp = await asyncio.get_event_loop().run_in_executor(
+                    None, self._get_spacy, language
+                )
+
+                doc = await asyncio.get_event_loop().run_in_executor(
+                    None, nlp, text
+                )
+
+                keywords = [token.text for token in doc if token.pos_ in ('NOUN', 'PROPN')][:top_k]
+                return keywords
+
+    @validate_input(SummarizeSchema)
+
+    async def summarize(self, text: str, max_length: int = 150, language: Optional[str] = None) -> str:
+        """
+        Summarize text.
+
+        Args:
+            text (str): Text to summarize.
+            max_length (int): Maximum length of the summary.
+            language (Optional[str]): Language of the text.
+
+        Returns:
+            str: Summarized text.
+        """
+        if not self._check_rate_limit():
+            raise ValueError("Rate limit exceeded. Please try again later.")
+
+        language = language or self._detect_language(text)
+        # Use appropriate models for summarization
+        if language == 'en':
+            model = "facebook/bart-large-cnn"
+        else:
+            # For Chinese and other languages, use a multilingual model
+            # For now, use t5-base, but consider using a Chinese-specific model in the future
+            model = "t5-base"
+
+        pipe = await asyncio.get_event_loop().run_in_executor(
+            None, self._get_hf_pipeline, "summarization", model
+        )
+
+        # Different models use different parameter names for length control
+        if model.startswith("t5"):
+            # T5 models use max_new_tokens instead of max_length
+            # For Chinese text, use a more conservative approach
+            if language == 'zh':
+                # Chinese text: use character count and be more conservative
+                input_chars = len(text)
+                max_new_tokens = min(max_length, max(input_chars // 4, 5))
+                min_new_tokens = 2
+            else:
+                # English text: use word count
+                input_words = len(text.split())
+                max_new_tokens = min(max_length, max(input_words // 2, 10))
+                min_new_tokens = 5
+            
+            result = await asyncio.get_event_loop().run_in_executor(
+                None, lambda: pipe(text, max_new_tokens=max_new_tokens, min_new_tokens=min_new_tokens, do_sample=False)[0]['summary_text']
+            )
+        else:
+            # BART and other models use max_length
+            if language == 'zh':
+                # Chinese text: use character count
+                input_chars = len(text)
+                max_len = min(max_length, max(input_chars // 4, 10))
+                min_len = 5
+            else:
+                # English text: use word count
+                input_words = len(text.split())
+                max_len = min(max_length, max(input_words // 2, 20))
+                min_len = 10
+            
+            result = await asyncio.get_event_loop().run_in_executor(
+                None, lambda: pipe(text, max_length=max_len, min_length=min_len, do_sample=False)[0]['summary_text']
+            )
+
+        return result
+
+    @validate_input(BatchProcessSchema)
+
+    async def batch_process(self, texts: List[str], operation: str, language: Optional[str] = None,
+                           model: Optional[str] = None, top_k: Optional[int] = None,
+                           max_length: Optional[int] = None) -> List[Any]:
+        """
+        Process multiple texts with the specified operation.
+
+        Args:
+            texts (List[str]): List of texts to process.
+            operation (str): Operation to perform on each text.
+            language (Optional[str]): Language of the texts.
+            model (Optional[str]): Model to use for processing.
+            top_k (Optional[int]): Number of keywords to extract (for keyword_extract).
+            max_length (Optional[int]): Maximum length of the summary (for summarize).
+
+        Returns:
+            List[Any]: List of operation results.
+        """
+        if not self._check_rate_limit():
+            raise ValueError("Rate limit exceeded. Please try again later.")
+
+        # Prepare operations to execute in batch
+        operations = []
+        for text in texts:
+            kwargs = {"text": text}
+            if language:
+                kwargs["language"] = language
+            if model and operation == "classify":
+                kwargs["model"] = model
+            if top_k and operation == "keyword_extract":
+                kwargs["top_k"] = top_k
+            if max_length and operation == "summarize":
+                kwargs["max_length"] = max_length
+
+            operations.append({"op": operation, "kwargs": kwargs})
+
+        # Execute batch operations
+        return await self.run_batch(operations)
+
+    async def health_check(self) -> Dict[str, Any]:
+        """
+        Perform a health check on the tool.
+
+        Returns:
+            Dict[str, Any]: Health check results.
+        """
+        result = {
+            "status": "ok",
+            "metrics": {
+                "requests": self._metrics["requests"],
+                "cache_hits": self._metrics["cache_hits"],
+                "avg_processing_time": sum(self._metrics["processing_time"]) / len(self._metrics["processing_time"])
+                                      if self._metrics["processing_time"] else 0.0
+            },
+            "config": {
+                "max_workers": self.config.max_workers,
+                "pipeline_cache_size": self.config.pipeline_cache_size,
+                "rate_limit_enabled": self.config.rate_limit_enabled,
+                "rate_limit_requests": self.config.rate_limit_requests,
+                "rate_limit_window": self.config.rate_limit_window
+            }
+        }
+
+        # Check if models can be loaded
+        try:
+            await asyncio.get_event_loop().run_in_executor(
+                None, self._get_spacy, "en"
+            )
+            result["models"] = {"spacy_en": "ok"}
+        except Exception as e:
+            result["status"] = "warning"
+            result["models"] = {"spacy_en": f"error: {str(e)}"}
+
+        return result
+
+    async def cleanup(self) -> None:
+        """
+        Clean up resources used by the tool.
+        """
+        # Clear spaCy models
+        self._spacy_nlp.clear()
+
+        # Clear metrics
+        self._metrics = {'requests': 0, 'cache_hits': 0, 'processing_time': []}
+
+        # Clear rate limiting data
+        self._request_timestamps = []
+
+        self.logger.info("ClassifierTool resources cleaned up")