unrealon 1.0.5__tar.gz → 1.0.6__tar.gz

{unrealon-1.0.5 → unrealon-1.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unrealon
-Version: 1.0.5
+Version: 1.0.6
 Summary: AI-powered web scraping platform with real-time orchestration
 License: MIT
 Author: Unrealon Team

{unrealon-1.0.5 → unrealon-1.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "unrealon"
-version = "1.0.5"
+version = "1.0.6"
 description = "AI-powered web scraping platform with real-time orchestration"
 authors = ["Unrealon Team <dev@unrealon.com>"]
 readme = "README.md"

{unrealon-1.0.5 → unrealon-1.0.6}/unrealon/sdk_config.py

@@ -8,7 +8,7 @@ import os
 from pydantic import BaseModel, Field, ConfigDict
 
 # Simple version constants
-VERSION = "1.0.5"
+VERSION = "1.0.6"
 
 # Project info
 AUTHOR = "UnrealOn Team"

{unrealon-1.0.5 → unrealon-1.0.6}/unrealon_llm/src/modules/html_processor/base_processor.py

@@ -10,6 +10,7 @@ import json
 import random
 from typing import Type
 import traceback
+import re
 
 from unrealon_llm.src.core import SmartLLMClient
 from unrealon_llm.src.dto import ChatMessage, MessageRole
@@ -77,6 +78,10 @@ class BaseHTMLProcessor(ABC):
         """Return extraction prompt template for this processor type"""
         pass
 
+    def _trim_system_prompt(self, system_prompt: str) -> str:
+        """Trim system prompt to remove empty lines"""
+        return "\n".join(system_prompt.split("\n")[1:])
+
     async def extract_patterns(self, html_content: str) -> ExtractionResult:
         """
         Extract patterns from HTML using LLM intelligence
@@ -116,15 +121,27 @@ class BaseHTMLProcessor(ABC):
             prompt_tokens=0,
             details={
                 "full_prompt": prompt[:2000] + "..." if len(prompt) > 2000 else prompt,
-                "schema_json": json.dumps(self.schema_class.model_json_schema(), indent=2)
-            }
+                "schema_json": json.dumps(
+                    self.schema_class.model_json_schema(), indent=2
+                ),
+            },
         )
 
+        # Add critical format requirements to the prompt
+        SYSTEM_PROMPT = f"""
+        You are an HTML-to-JSON expert at analyzing {self.processor_type} pages. 
+        You MUST return JSON that EXACTLY matches the Pydantic schema provided. 
+        RESPOND ONLY WITH VALID JSON. 
+        NO EXPLANATIONS, NO TEXT, ONLY JSON! 
+        Include ALL required fields from the schema! 
+        CRITICAL: The 'selectors' field must be a DICTIONARY/OBJECT, not a list!
+        """
+
         # Prepare LLM messages
         messages = [
             ChatMessage(
                 role=MessageRole.SYSTEM,
-                content=f"You are an HTML-to-JSON expert at analyzing {self.processor_type} pages. You MUST return JSON that EXACTLY matches the Pydantic schema provided. RESPOND ONLY WITH VALID JSON. NO EXPLANATIONS, NO TEXT, ONLY JSON! Include ALL required fields from the schema!",
+                content=self._trim_system_prompt(SYSTEM_PROMPT),
             ),
             ChatMessage(
                 role=MessageRole.USER,
@@ -144,8 +161,7 @@ class BaseHTMLProcessor(ABC):
         try:
             # Call LLM
             response = await self.llm_client.chat_completion(
-                messages, 
-                response_model=self.schema_class
+                messages, response_model=self.schema_class
             )
 
             # Log full LLM response for debugging
@@ -167,7 +183,7 @@ class BaseHTMLProcessor(ABC):
             )
 
             # Use the validated model from LLM response
-            if hasattr(response, 'extracted_model') and response.extracted_model:
+            if hasattr(response, "extracted_model") and response.extracted_model:
                 validated_model = response.extracted_model
                 validated_result = validated_model.model_dump()
                 logger.log_html_analysis_completed(
@@ -203,8 +219,36 @@ class BaseHTMLProcessor(ABC):
                             "raw_llm_response": result_data,
                         },
                     )
-                    # Fall back to raw data
-                    validated_result = result_data
+
+                    # 🔥 SMART FALLBACK: Try to fix common LLM format issues
+                    try:
+                        fixed_data = self._fix_llm_response_format(result_data, str(e))
+                        validated_model = self.schema_class(**fixed_data)
+                        validated_result = validated_model.model_dump()
+                        logger.log_html_analysis_completed(
+                            selectors_generated=len(str(fixed_data)),
+                            confidence_score=fixed_data.get("confidence", 0.0),
+                            details={
+                                "processor_type": self.processor_type,
+                                "validation_success": True,
+                                "schema_matched": True,
+                                "format_fixed": True,
+                            },
+                        )
+                    except Exception as fix_error:
+                        logger.log_html_analysis_failed(
+                            error_message=f"Format fixing also failed: {str(fix_error)}",
+                            details={
+                                "processor_type": self.processor_type,
+                                "validation_error": str(e),
+                                "fix_error": str(fix_error),
+                                "raw_llm_response": result_data,
+                            },
+                        )
+                        # Final fallback: create minimal valid structure
+                        validated_result = self._create_fallback_result(
+                            result_data, str(e)
+                        )
 
             # Create Pydantic processing metadata
             processing_info = ProcessingInfo(
@@ -253,12 +297,20 @@ class BaseHTMLProcessor(ABC):
 
         # Add random number to bypass any caching
         cache_buster = random.randint(100000, 999999)
-        
-        schema_prompt = f"""PYDANTIC 2 SCHEMA (Request #{cache_buster}):
-{schema_json}
 
-CRITICAL: Return JSON that EXACTLY matches this schema structure!
-The response must include ALL required fields: detected_item_type, extraction_strategy, confidence, selectors, documentation."""
+        schema_prompt = f"""
+        PYDANTIC 2 SCHEMA (Request #{cache_buster}):
+        {schema_json}
+
+        🚨 CRITICAL FORMAT REQUIREMENTS:
+        1. Return JSON that EXACTLY matches this schema structure!
+        2. The response must include ALL required fields: detected_item_type, extraction_strategy, confidence, selectors, documentation
+        3. The "selectors" field MUST be a DICTIONARY/OBJECT with field names as keys and arrays of CSS selectors as values
+        4. Example: "selectors": {{"title": ["h1.title", ".product-name"], "price": [".price", ".cost"]}}
+        5. DO NOT return "selectors" as a list: ❌ ["h1.title", ".price"]
+        6. DO return "selectors" as a dictionary: ✅ {{"title": ["h1.title"], "price": [".price"]}}
+        """
+        schema_prompt = self._trim_system_prompt(schema_prompt)
 
         return prompt_template.format(
             processor_type=self.processor_type,
@@ -292,3 +344,72 @@ The response must include ALL required fields: detected_item_type, extraction_st
         estimated_cost = (total_tokens / 1_000_000) * 0.25
 
         return estimated_cost
+
+    def _fix_llm_response_format(self, result_data: dict, error_message: str) -> dict:
+        """Fix common LLM response format issues."""
+        fixed_data = result_data.copy()
+
+        # Fix selectors if it's a list instead of dict
+        if "selectors" in fixed_data and isinstance(fixed_data["selectors"], list):
+            logger.log_html_analysis_failed(
+                error_message="Fixing selectors format: list -> dict",
+                details={
+                    "processor_type": self.processor_type,
+                    "original_selectors": fixed_data["selectors"],
+                },
+            )
+
+            # Convert list to dict with generic field names
+            selectors_list = fixed_data["selectors"]
+            fixed_data["selectors"] = {}
+
+            # Try to intelligently map list items to field names
+            field_names = ["item", "title", "price", "description", "image", "link"]
+            for i, selector in enumerate(selectors_list):
+                if i < len(field_names):
+                    field_name = field_names[i]
+                else:
+                    field_name = f"field_{i+1}"
+
+                # Convert single selector to list
+                if isinstance(selector, str):
+                    fixed_data["selectors"][field_name] = [selector]
+                elif isinstance(selector, list):
+                    fixed_data["selectors"][field_name] = selector
+                else:
+                    fixed_data["selectors"][field_name] = [str(selector)]
+
+        # Ensure all required fields exist
+        required_fields = [
+            "detected_item_type",
+            "extraction_strategy",
+            "confidence",
+            "selectors",
+            "documentation",
+        ]
+        for field in required_fields:
+            if field not in fixed_data:
+                if field == "detected_item_type":
+                    fixed_data[field] = "unknown"
+                elif field == "extraction_strategy":
+                    fixed_data[field] = "fallback_strategy"
+                elif field == "confidence":
+                    fixed_data[field] = 0.1
+                elif field == "selectors":
+                    fixed_data[field] = {}
+                elif field == "documentation":
+                    fixed_data[field] = (
+                        "Extraction completed with fallback processing due to format issues."
+                    )
+
+        return fixed_data
+
+    def _create_fallback_result(self, result_data: dict, error_message: str) -> dict:
+        """Create a minimal valid result when all else fails."""
+        return {
+            "detected_item_type": "unknown",
+            "extraction_strategy": "fallback_strategy",
+            "confidence": 0.1,
+            "selectors": {},
+            "documentation": f"Extraction failed due to validation error: {error_message}. Raw data: {str(result_data)[:500]}...",
+        }

unrealon-1.0.6/unrealon_llm/src/modules/html_processor/details_processor.py

@@ -0,0 +1,85 @@
+"""
+Details Processor
+
+Universal processor for detail/product/item pages.
+Handles ANY type of detail pages: product details, service info, article content, job descriptions, etc.
+"""
+
+from typing import Type
+
+from .base_processor import BaseHTMLProcessor
+from .models import UniversalExtractionSchema
+
+
+class DetailsProcessor(BaseHTMLProcessor):
+    """Universal details page pattern extractor"""
+
+    def get_processor_type(self) -> str:
+        """Return processor type identifier"""
+        return "details"
+
+    def get_schema_class(self) -> Type[UniversalExtractionSchema]:
+        """Return Pydantic schema class for details extraction"""
+        return UniversalExtractionSchema
+
+    def get_extraction_prompt_template(self) -> str:
+        """Return details-specific extraction prompt template"""
+
+        prompt = """{schema}
+        [__TASK_DESCRIPTION__]
+        Analyze this DETAILS/PRODUCT/ITEM page and generate universal extraction patterns.
+        PROCESSOR TYPE: {processor_type}
+        THIS IS A DETAILS PAGE containing information about a single item/product/service/article.
+        [/__TASK_DESCRIPTION__]
+
+        [__CRITICAL_FORMAT_REQUIREMENTS__]
+        🚨 SELECTORS FORMAT: The "selectors" field MUST be a DICTIONARY/OBJECT, NOT a list!
+        Example of CORRECT format:
+        "selectors": {{
+            "title": ["h1.product-title", "h1.page-title", ".item-name"],
+            "price": [".price", ".cost", "span[data-price]", ".product-price"],
+            "description": [".description", ".product-desc", ".item-details"],
+            "images": ["img.product-image", ".gallery img", "img[src*='product']"],
+            "specifications": [".specs", ".product-specs", ".item-specifications"],
+            "reviews": [".reviews", ".product-reviews", ".customer-reviews"]
+        }}
+
+        ❌ WRONG format (DO NOT USE):
+        "selectors": ["h1.title", ".price", ".description"]
+
+        ✅ CORRECT format (USE THIS):
+        "selectors": {{
+            "title": ["h1.title", ".product-name", "h1[itemprop='name']"],
+            "price": [".price", ".cost", "span[data-price]"],
+            "description": [".description", ".product-desc", ".item-details"]
+        }}
+        [/__CRITICAL_FORMAT_REQUIREMENTS__]
+
+        [__INSTRUCTIONS__]
+        YOUR TASK:
+        Analyze this details page and generate extraction patterns for ANY type of item.
+        This could be: product details, service info, article content, job description, real estate listing, person profile, etc.
+
+        CRITICAL REQUIREMENTS:
+        1. The "selectors" field MUST be a DICTIONARY with field names as keys and arrays of CSS selectors as values
+        2. Include comprehensive markdown documentation
+        3. Provide real examples from the actual HTML
+        4. Explain the page structure and best extraction approach
+        5. Include confidence scores and fallback strategies
+        6. Document any special handling needed
+
+        ANALYZE THE HTML AND DETERMINE:
+        - What type of item this page describes
+        - What information is available (specs, pricing, reviews, etc.)
+        - How content is structured and organized
+        - What actions are possible (buy, contact, etc.)
+        - Best extraction strategy for this specific page
+        [/__INSTRUCTIONS__]
+
+        [__HTML_CONTENT__]
+        HTML CONTENT (first 50KB):
+        {html_content}
+        [/__HTML_CONTENT__]
+        """
+
+        return self._trim_system_prompt(prompt)

unrealon-1.0.6/unrealon_llm/src/modules/html_processor/listing_processor.py

@@ -0,0 +1,91 @@
+"""
+Listing Processor
+
+Universal processor for listing/catalog pages.
+Handles ANY type of listings: products, services, articles, real estate, jobs, etc.
+"""
+
+from typing import Type
+
+from .base_processor import BaseHTMLProcessor
+from .models import UniversalExtractionSchema
+
+
+class ListingProcessor(BaseHTMLProcessor):
+    """Universal listing page pattern extractor"""
+
+    def get_processor_type(self) -> str:
+        """Return processor type identifier"""
+        return "listing"
+
+    def get_schema_class(self) -> Type[UniversalExtractionSchema]:
+        """Return Pydantic schema class for listing extraction"""
+        return UniversalExtractionSchema
+
+    def get_extraction_prompt_template(self) -> str:
+        """Return listing-specific extraction prompt template"""
+
+        prompt = """{schema}
+
+        [__TASK_DESCRIPTION__]
+        Analyze this LISTING/CATALOG page and generate universal extraction patterns.
+        PROCESSOR TYPE: {processor_type}
+        THIS IS A LISTING PAGE containing multiple items arranged in a list or grid.
+        [/__TASK_DESCRIPTION__]
+
+        [__CRITICAL_FORMAT_REQUIREMENTS__]
+        🚨 SELECTORS FORMAT: The "selectors" field MUST be a DICTIONARY/OBJECT, NOT a list!
+        Example of CORRECT format:
+        "selectors": {{
+            "items_container": ["div.product-grid", "ul.product-list", "div.items"],
+            "item_title": ["h3.product-title", "a.product-link", ".item-name"],
+            "item_price": [".price", ".cost", "span[data-price]"],
+            "item_image": ["img.product-image", ".item-img", "img[src*='product']"],
+            "pagination": [".pagination", ".page-nav", "nav[aria-label='pagination']"]
+        }}
+
+        ❌ WRONG format (DO NOT USE):
+        "selectors": ["div.product", "h3.title", ".price"]
+
+        ✅ CORRECT format (USE THIS):
+        "selectors": {{
+            "items": ["div.product", "li.item", ".product-card"],
+            "titles": ["h3.title", ".product-name", "a[title]"],
+            "prices": [".price", ".cost", "span[data-price]"]
+        }}
+        [/__CRITICAL_FORMAT_REQUIREMENTS__]
+
+        [__INSTRUCTIONS__]
+        YOUR TASK:
+        Analyze this listing page and generate extraction patterns for ANY type of items.
+        This could be: products, services, articles, jobs, real estate, people, cars, etc.
+
+        CRITICAL REQUIREMENTS:
+        1. The "selectors" field MUST be a DICTIONARY with field names as keys and arrays of CSS selectors as values
+        2. This is a LISTING PAGE with multiple items
+        3. Focus on identifying item containers and individual item patterns
+        4. Detect ANY type of items - not just products!
+        5. Provide multiple fallback selectors for reliability
+        6. Include pagination and navigation patterns
+        7. Use realistic confidence scores (0.1-1.0)
+        8. Auto-detect what type of content this listing contains
+        9. Provide extraction strategy advice
+        10. Look for structured data (JSON-LD, microdata)
+        11. Generate patterns that work with BeautifulSoup4 .select() method
+        12. RETURN JSON that EXACTLY matches the Pydantic schema above!
+
+        ANALYZE THE HTML AND DETERMINE:
+        - What type of items are listed (products, services, articles, etc.)
+        - How items are structured and contained
+        - What navigation elements exist
+        - What metadata is available
+        - Best extraction strategy for this specific page
+        [/__INSTRUCTIONS__]
+
+        [__HTML_CONTENT__]
+        HTML CONTENT (first 50KB):
+        {html_content}
+        [/__HTML_CONTENT__]
+        """
+
+        return self._trim_system_prompt(prompt)

unrealon-1.0.5/unrealon_llm/src/modules/html_processor/details_processor.py

@@ -1,61 +0,0 @@
-"""
-Details Processor
-
-Universal processor for detail/product/item pages.
-Handles ANY type of detail pages: product details, service info, article content, job descriptions, etc.
-"""
-
-from typing import Type
-
-from .base_processor import BaseHTMLProcessor
-from .models import UniversalExtractionSchema
-
-
-class DetailsProcessor(BaseHTMLProcessor):
-    """Universal details page pattern extractor"""
-
-    def get_processor_type(self) -> str:
-        """Return processor type identifier"""
-        return "details"
-
-    def get_schema_class(self) -> Type[UniversalExtractionSchema]:
-        """Return Pydantic schema class for details extraction"""
-        return UniversalExtractionSchema
-
-    def get_extraction_prompt_template(self) -> str:
-        """Return details-specific extraction prompt template"""
-
-        return """{schema}
-
-[__TASK_DESCRIPTION__]
-Analyze this DETAILS/PRODUCT/ITEM page and generate universal extraction patterns.
-PROCESSOR TYPE: {processor_type}
-THIS IS A DETAILS PAGE containing information about a single item/product/service/article.
-[/__TASK_DESCRIPTION__]
-
-[__INSTRUCTIONS__]
-YOUR TASK:
-Analyze this details page and generate extraction patterns for ANY type of item.
-This could be: product details, service info, article content, job description, real estate listing, person profile, etc.
-
-CRITICAL REQUIREMENTS:
-1. Return simple CSS selectors in the "selectors" object
-2. Include comprehensive markdown documentation
-3. Provide real examples from the actual HTML
-4. Explain the page structure and best extraction approach
-5. Include confidence scores and fallback strategies
-6. Document any special handling needed
-
-ANALYZE THE HTML AND DETERMINE:
-- What type of item this page describes
-- What information is available (specs, pricing, reviews, etc.)
-- How content is structured and organized
-- What actions are possible (buy, contact, etc.)
-- Best extraction strategy for this specific page
-[/__INSTRUCTIONS__]
-
-[__HTML_CONTENT__]
-HTML CONTENT (first 50KB):
-{html_content}
-[/__HTML_CONTENT__]
-"""

unrealon-1.0.5/unrealon_llm/src/modules/html_processor/listing_processor.py

@@ -1,67 +0,0 @@
-"""
-Listing Processor
-
-Universal processor for listing/catalog pages.
-Handles ANY type of listings: products, services, articles, real estate, jobs, etc.
-"""
-
-from typing import Type
-
-from .base_processor import BaseHTMLProcessor
-from .models import UniversalExtractionSchema
-
-
-class ListingProcessor(BaseHTMLProcessor):
-    """Universal listing page pattern extractor"""
-
-    def get_processor_type(self) -> str:
-        """Return processor type identifier"""
-        return "listing"
-
-    def get_schema_class(self) -> Type[UniversalExtractionSchema]:
-        """Return Pydantic schema class for listing extraction"""
-        return UniversalExtractionSchema
-
-    def get_extraction_prompt_template(self) -> str:
-        """Return listing-specific extraction prompt template"""
-
-        return """{schema}
-
-[__TASK_DESCRIPTION__]
-Analyze this LISTING/CATALOG page and generate universal extraction patterns.
-PROCESSOR TYPE: {processor_type}
-THIS IS A LISTING PAGE containing multiple items arranged in a list or grid.
-[/__TASK_DESCRIPTION__]
-
-[__INSTRUCTIONS__]
-YOUR TASK:
-Analyze this listing page and generate extraction patterns for ANY type of items.
-This could be: products, services, articles, jobs, real estate, people, cars, etc.
-
-CRITICAL REQUIREMENTS:
-1. Return simple CSS selectors in the "selectors" object
-2. This is a LISTING PAGE with multiple items
-3. Focus on identifying item containers and individual item patterns
-4. Detect ANY type of items - not just products!
-5. Provide multiple fallback selectors for reliability
-6. Include pagination and navigation patterns
-7. Use realistic confidence scores (0.1-1.0)
-8. Auto-detect what type of content this listing contains
-9. Provide extraction strategy advice
-10. Look for structured data (JSON-LD, microdata)
-11. Generate patterns that work with BeautifulSoup4 .select() method
-12. RETURN JSON that EXACTLY matches the Pydantic schema above!
-
-ANALYZE THE HTML AND DETERMINE:
-- What type of items are listed (products, services, articles, etc.)
-- How items are structured and contained
-- What navigation elements exist
-- What metadata is available
-- Best extraction strategy for this specific page
-[/__INSTRUCTIONS__]
-
-[__HTML_CONTENT__]
-HTML CONTENT (first 50KB):
-{html_content}
-[/__HTML_CONTENT__]
-"""