spiderforce4ai 2.1__py3-none-any.whl → 2.4__py3-none-any.whl

spiderforce4ai/post_extraction_agent.py

@@ -0,0 +1,259 @@
+# post_extraction_agent.py
+
+from dataclasses import dataclass, asdict
+from typing import Any, Callable, Dict, List, Optional, Union
+import json
+import asyncio
+import time
+from pathlib import Path
+import aiofiles
+from litellm import completion
+from pydantic import BaseModel, Field
+import logging
+from datetime import datetime
+import re
+
+logger = logging.getLogger(__name__)
+
+class PostExtractionBuffer:
+    """Buffer system for tracking and retrying failed LLM requests."""
+    
+    def __init__(self, buffer_file: Optional[Path] = None):
+        # Generate a unique session ID using timestamp and random string
+        session_id = f"{int(time.time())}_{hex(hash(str(time.time())))[-6:]}"
+        
+        # Create unique buffer file path
+        if buffer_file:
+            # If buffer_file is provided, insert session_id before the extension
+            stem = buffer_file.stem
+            suffix = buffer_file.suffix
+            self.buffer_file = buffer_file.with_name(f"{stem}_{session_id}{suffix}")
+        else:
+            # Default buffer file with session_id
+            self.buffer_file = Path(f"post_extraction_buffer_{session_id}.json")
+            
+        self.failed_requests: Dict[str, Dict] = {}
+        self._load_buffer()
+    
+    def _load_buffer(self) -> None:
+        """Load failed requests from buffer file if it exists."""
+        if self.buffer_file.exists():
+            try:
+                with open(self.buffer_file, 'r') as f:
+                    self.failed_requests = json.load(f)
+            except Exception as e:
+                logger.error(f"Error loading buffer file: {e}")
+                self.failed_requests = {}
+
+    def _save_buffer(self) -> None:
+        """Save failed requests to buffer file."""
+        try:
+            with open(self.buffer_file, 'w') as f:
+                json.dump(self.failed_requests, f, indent=2)
+        except Exception as e:
+            logger.error(f"Error saving buffer file: {e}")
+
+    def add_failed_request(self, url: str, content: str, error: str) -> None:
+        """Add a failed request to the buffer."""
+        self.failed_requests[url] = {
+            "content": content,
+            "error": error,
+            "timestamp": datetime.now().isoformat(),
+            "attempts": self.failed_requests.get(url, {}).get("attempts", 0) + 1
+        }
+        self._save_buffer()
+
+    def remove_request(self, url: str) -> None:
+        """Remove a request from the buffer after successful processing."""
+        if url in self.failed_requests:
+            del self.failed_requests[url]
+            self._save_buffer()
+
+    def get_failed_requests(self) -> Dict[str, Dict]:
+        """Get all failed requests."""
+        return self.failed_requests
+
+    def get_retryable_requests(self, max_attempts: int = 3) -> Dict[str, Dict]:
+        """Get failed requests that haven't exceeded max retry attempts."""
+        return {
+            url: data for url, data in self.failed_requests.items()
+            if data.get("attempts", 0) < max_attempts
+        }
+
+class ExtractionTemplate(BaseModel):
+    """Base model for extraction template validation."""
+    template: Dict[str, Any] = Field(..., description="Template structure for extraction")
+    
+    class Config:
+        extra = "allow"
+        arbitrary_types_allowed = True
+
+    @classmethod
+    def validate_template_string(cls, template_str: str) -> bool:
+        """Validate a template string against the schema."""
+        try:
+            template_json = json.loads(template_str)
+            cls(template=template_json)
+            return True
+        except Exception as e:
+            logger.error(f"Template validation failed: {e}")
+            return False
+
+@dataclass
+class PostExtractionConfig:
+    """Configuration for post-extraction processing."""
+    model: str
+    messages: List[Dict[str, str]]
+    api_key: str
+    max_tokens: int = 1000
+    temperature: float = 0.7
+    base_url: Optional[str] = None
+    request_delay: float = 0.01  # 10 milliseconds default
+    max_retries: int = 3
+    retry_delay: float = 1.0
+    combine_output: bool = False
+    output_file: Optional[Path] = None
+    custom_transform_function: Optional[Callable] = None
+    buffer_file: Optional[Path] = None
+
+    def __post_init__(self):
+        if self.output_file:
+            self.output_file = Path(self.output_file)
+            self.output_file.parent.mkdir(parents=True, exist_ok=True)
+        
+        if self.buffer_file:
+            self.buffer_file = Path(self.buffer_file)
+            self.buffer_file.parent.mkdir(parents=True, exist_ok=True)
+
+class RateLimiter:
+    """Rate limiter for API calls."""
+    
+    def __init__(self, requests_per_minute: int = 60):
+        self.requests_per_minute = requests_per_minute
+        self.interval = 60 / requests_per_minute
+        self.last_request = 0
+        self._lock = asyncio.Lock()
+
+    async def acquire(self):
+        """Acquire rate limit slot."""
+        async with self._lock:
+            now = time.time()
+            if self.last_request:
+                elapsed = now - self.last_request
+                if elapsed < self.interval:
+                    await asyncio.sleep(self.interval - elapsed)
+            self.last_request = time.time()
+
+class PostExtractionAgent:
+    """Agent for processing extracted content using LLM models."""
+    
+    def __init__(self, config: PostExtractionConfig):
+        self.config = config
+        self.buffer = PostExtractionBuffer(config.buffer_file)
+        self.results: Dict[str, Any] = {}
+        self.rate_limiter = RateLimiter()
+        self._setup_output()
+        
+    def _setup_output(self) -> None:
+        """Setup output file if combining results."""
+        if self.config.combine_output and self.config.output_file:
+            self.config.output_file.parent.mkdir(parents=True, exist_ok=True)
+            if self.config.output_file.exists():
+                # Backup existing file
+                backup_path = self.config.output_file.with_suffix(f".bak_{int(time.time())}")
+                self.config.output_file.rename(backup_path)
+            self.config.output_file.touch()
+
+    async def _process_single_content(self, url: str, content: str) -> Optional[Dict]:
+        """Process a single piece of content through the LLM."""
+        try:
+            # Apply rate limiting
+            await self.rate_limiter.acquire()
+            
+            # Replace placeholder in messages with actual content
+            messages = [
+                {**msg, 'content': msg['content'].replace('{here_markdown_content}', content)}
+                for msg in self.config.messages
+            ]
+            
+            # Make LLM request
+            response = await completion(
+                model=self.config.model,
+                messages=messages,
+                max_tokens=self.config.max_tokens,
+                temperature=self.config.temperature,
+                api_key=self.config.api_key,
+                api_base=self.config.base_url
+            )
+            
+            # Parse response
+            try:
+                extracted_data = json.loads(response.choices[0].message.content)
+                self.buffer.remove_request(url)  # Remove from buffer if successful
+                return extracted_data
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON response from LLM: {e}")
+                
+        except Exception as e:
+            logger.error(f"Error processing {url}: {str(e)}")
+            self.buffer.add_failed_request(url, content, str(e))
+            return None
+
+    async def _save_result(self, url: str, result: Dict) -> None:
+        """Save individual or combined results."""
+        try:
+            if self.config.combine_output and self.config.output_file:
+                self.results[url] = result
+                async with aiofiles.open(self.config.output_file, 'w') as f:
+                    await f.write(json.dumps(self.results, indent=2))
+            elif not self.config.combine_output and self.config.output_file:
+                individual_file = self.config.output_file.parent / f"{url.replace('/', '_')}.json"
+                async with aiofiles.open(individual_file, 'w') as f:
+                    await f.write(json.dumps(result, indent=2))
+        except Exception as e:
+            logger.error(f"Error saving results for {url}: {str(e)}")
+
+    async def process_content(self, url: str, content: str) -> Optional[Dict]:
+        """Process content with retry mechanism."""
+        for attempt in range(self.config.max_retries):
+            result = await self._process_single_content(url, content)
+            if result:
+                # Apply custom transformation if provided
+                if self.config.custom_transform_function:
+                    try:
+                        result = self.config.custom_transform_function(result)
+                    except Exception as e:
+                        logger.error(f"Error in custom transform for {url}: {str(e)}")
+                
+                await self._save_result(url, result)
+                return result
+            
+            # Wait before retry
+            if attempt < self.config.max_retries - 1:
+                await asyncio.sleep(self.config.retry_delay)
+        
+        return None
+
+    async def process_bulk_content(self, content_map: Dict[str, str]) -> Dict[str, Optional[Dict]]:
+        """Process multiple pieces of content with rate limiting."""
+        results = {}
+        for url, content in content_map.items():
+            results[url] = await self.process_content(url, content)
+            await asyncio.sleep(self.config.request_delay)
+        return results
+
+    def retry_failed_requests(self) -> Dict[str, Optional[Dict]]:
+        """Retry all failed requests from the buffer."""
+        failed_requests = self.buffer.get_retryable_requests(self.config.max_retries)
+        return asyncio.run(self.process_bulk_content(
+            {url: data['content'] for url, data in failed_requests.items()}
+        ))
+
+    async def get_processing_stats(self) -> Dict[str, Any]:
+        """Get detailed processing statistics."""
+        return {
+            "total_processed": len(self.results),
+            "failed_requests": len(self.buffer.get_failed_requests()),
+            "retryable_requests": len(self.buffer.get_retryable_requests(self.config.max_retries)),
+            "success_rate": len(self.results) / (len(self.results) + len(self.buffer.get_failed_requests())) * 100 if self.results else 0
+        }

{spiderforce4ai-2.1.dist-info → spiderforce4ai-2.4.dist-info}/METADATA

@@ -1,16 +1,24 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 2.1
-Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
+Version: 2.4
+Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service with LLM post-processing
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz
 Author-email: Piotr Tamulewicz <pt@petertam.pro>
-License: MIT
+Project-URL: Homepage, https://petertam.pro
+Project-URL: Documentation, https://petertam.pro/docs/spiderforce4ai
+Project-URL: Repository, https://github.com/yourusername/spiderforce4ai
+Project-URL: Bug Tracker, https://github.com/yourusername/spiderforce4ai/issues
+Keywords: web-scraping,markdown,html-to-markdown,llm,ai,content-extraction,async,parallel-processing
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 Requires-Dist: aiohttp>=3.8.0
@@ -18,6 +26,36 @@ Requires-Dist: asyncio>=3.4.3
 Requires-Dist: rich>=10.0.0
 Requires-Dist: aiofiles>=0.8.0
 Requires-Dist: httpx>=0.24.0
+Requires-Dist: litellm>=1.26.0
+Requires-Dist: pydantic>=2.6.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: aiofiles>=23.2.1
+Requires-Dist: et-xmlfile>=1.1.0
+Requires-Dist: multidict>=6.0.4
+Requires-Dist: openai>=1.12.0
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: yarl>=1.9.4
+Requires-Dist: typing_extensions>=4.9.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.1; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: black>=23.7.0; extra == "dev"
+Requires-Dist: isort>=5.12.0; extra == "dev"
+Requires-Dist: mypy>=1.4.1; extra == "dev"
+Requires-Dist: ruff>=0.1.8; extra == "dev"
+Requires-Dist: pre-commit>=3.5.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest>=7.4.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.21.1; extra == "test"
+Requires-Dist: pytest-cov>=4.1.0; extra == "test"
+Requires-Dist: pytest-mock>=3.12.0; extra == "test"
+Requires-Dist: coverage>=7.4.0; extra == "test"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.1.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=1.3.0; extra == "docs"
+Requires-Dist: myst-parser>=2.0.0; extra == "docs"
 Dynamic: author
 Dynamic: home-page
 Dynamic: requires-python

spiderforce4ai-2.4.dist-info/RECORD

@@ -0,0 +1,7 @@
+spiderforce4ai/__init__.py,sha256=JClWyqGGCVC6yxuK4TpJ7a-7iP6ueD20oKc0ERHxnyU,38701
+spiderforce4ai/post_extraction_agent.py,sha256=yZ17xdOtkNMDRGqqudNBZIb6N9bcsjOwbzPB6D5kJHg,10540
+spiderforce4ai-2.4.dist-info/METADATA,sha256=UNtth74KAHCNOngozhN2es3z4vY6J7SiKfTaIi0fYTI,9010
+spiderforce4ai-2.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+spiderforce4ai-2.4.dist-info/entry_points.txt,sha256=ibARQxOlDiL1ho12zbDZt4Uq5RKSIk_qk159ZlZ46hc,59
+spiderforce4ai-2.4.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
+spiderforce4ai-2.4.dist-info/RECORD,,

spiderforce4ai-2.4.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+spiderforce4ai = spiderforce4ai.cli:main

spiderforce4ai-2.1.dist-info/RECORD

@@ -1,5 +0,0 @@
-spiderforce4ai/__init__.py,sha256=6WskofS5vOJuPhdwoCRvcOmWVimCKJxtkkP_pshrrlo,35805
-spiderforce4ai-2.1.dist-info/METADATA,sha256=bK_85RBFEAmDTZgo2oCPKgDNd-dqfYvRJoBl92Zk-i8,7183
-spiderforce4ai-2.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-spiderforce4ai-2.1.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
-spiderforce4ai-2.1.dist-info/RECORD,,