unrealon 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

unrealon/sdk_config.py

@@ -8,7 +8,7 @@ import os
 from pydantic import BaseModel, Field, ConfigDict
 
 # Simple version constants
-VERSION = "1.0.5"
+VERSION = "1.0.7"
 
 # Project info
 AUTHOR = "UnrealOn Team"

{unrealon-1.0.5.dist-info → unrealon-1.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unrealon
-Version: 1.0.5
+Version: 1.0.7
 Summary: AI-powered web scraping platform with real-time orchestration
 License: MIT
 Author: Unrealon Team
@@ -186,7 +186,7 @@ UNREALON_BROWSER_PROFILE_DIR=system/browser_profiles
 # API Keys
 UNREALON_OPENROUTER_API_KEY=sk-or-v1-your-openrouter-key
 UNREALON_SERVER_URL=wss://api.unrealon.com
-UNREALON_SERVER_API_KEY=up_dev_your-api-key
+UNREALON_API_KEY=up_dev_your-api-key
 
 # Runtime Limits
 UNREALON_LLM_DAILY_LIMIT=1.0
@@ -247,7 +247,7 @@ class ParserSettings(BaseSettings):
     # API Keys
     OPENROUTER_API_KEY: str
     SERVER_URL: str
-    SERVER_API_KEY: str
+    API_KEY: str
     
     # Runtime Limits
     LLM_DAILY_LIMIT: float = Field(default=1.0)
@@ -317,7 +317,7 @@ class CustomAutoConfig(AutoConfig):
         """Override daemon config with custom settings."""
         return DaemonModeConfig(
             server_url=parser_settings.SERVER_URL,
-            api_key=parser_settings.SERVER_API_KEY,
+            api_key=parser_settings.API_KEY,
             auto_reconnect=True,
             connection_timeout=30,
             heartbeat_interval=30,

{unrealon-1.0.5.dist-info → unrealon-1.0.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unrealon/__init__.py,sha256=IWhHl4jLgYR5HjEtHR1_-BF3tUkXJpP67IqqekdsVxk,494
-unrealon/sdk_config.py,sha256=Q771leToYwbnG_Oi64DObYHtYFViS0U1H8S9mo3JnoQ,2118
+unrealon/sdk_config.py,sha256=V9QDoWS74obOJv4PotCHbXSWuI78NeIujzM8Lx_j-pc,2118
 unrealon_browser/README.md,sha256=9pP6RrfMGHtdT5uDLFAUB1e4nNGzZudXViEo1940gKw,396
 unrealon_browser/__init__.py,sha256=pLHhYYhYJmzrED1Chi3uQAETVFxqQrikwEAwO2TTZ70,631
 unrealon_browser/pyproject.toml,sha256=_PTGU6Pwh7antWDqZeA6KU-Vx3Xw4jwTlU_Wgt6t0Cg,4945
@@ -32,7 +32,7 @@ unrealon_driver/src/cli/__init__.py,sha256=6AE6FJoXxhr5bMGn9PVuavryEsvcjMiGFbQdn
 unrealon_driver/src/cli/main.py,sha256=wX0TxXyGG_X1La97zrELOujCI5i7mHnFeSP7d-r9FP8,1725
 unrealon_driver/src/cli/simple.py,sha256=rx8t8FSviddFYNdI18DJB6G3FLKndO_X56z69Iy2Z3Q,20087
 unrealon_driver/src/config/__init__.py,sha256=w-oB-HFHxscdxg9Ayw1t9jkRESDX9pTV5ggDrgt_Mrc,204
-unrealon_driver/src/config/auto_config.py,sha256=vnR4okInNSWgQj8gS8KyEAXW9VLNydcx_LMX99TEI7I,17025
+unrealon_driver/src/config/auto_config.py,sha256=e_7WPfFO8tl3JE-HPSzTfNVGJVEGKl6wuXO3z7qLeUk,16999
 unrealon_driver/src/core/__init__.py,sha256=dMsabllFmGA9PojjS9Y5DzrTzJgKA1BxvayAhl8qi3M,320
 unrealon_driver/src/core/exceptions.py,sha256=k5lqYqD100Q-d7Jyy1-dEm6gGnv-AxHFhpCaZywfxvo,8225
 unrealon_driver/src/core/parser.py,sha256=Wg4qSeCRsZSSLti-vCiYksnGXXR8ez0ZFXJc0PDiYnM,20950
@@ -43,7 +43,7 @@ unrealon_driver/src/dto/events.py,sha256=w69HZOKiKvopNpfMEhijCSZJWMY19f44wkPMnWH
 unrealon_driver/src/dto/execution.py,sha256=WMgKKfYiWHLwQ0sict16_QXbIBJGqLzWDma1F7CHYzQ,10583
 unrealon_driver/src/dto/services.py,sha256=BNqk2SQCRRnrIDsZJ1MbIniwZR6l3-kzqqRJgoAf3o8,10315
 unrealon_driver/src/execution/__init__.py,sha256=3lkrR6HTZL4SMXToyklQPXF4MelmRKVdoWWlnvPp72w,580
-unrealon_driver/src/execution/daemon_mode.py,sha256=IJyVhdoFNgiI3Vahe7sYTkqwrAQdEClVWdLA-BxXEtA,11110
+unrealon_driver/src/execution/daemon_mode.py,sha256=tgC4e-qAQj0R1_NCWwYtEE66yc_aFliVHSG1zGHeJJU,11022
 unrealon_driver/src/execution/interactive_mode.py,sha256=UbHpF6bMIQPrA-VBP3QIZ1KE6wAygwkVMzHoplevf6I,3245
 unrealon_driver/src/execution/modes.py,sha256=CuEJdcCkZSDpp_o668x9IiqOFZpvCgJ15j6bI75HvrA,1404
 unrealon_driver/src/execution/scheduled_mode.py,sha256=r-f2cUHNMxiBibTeta4c0g92eRhIZmSSvKrKnVDnRGI,7323
@@ -91,9 +91,9 @@ unrealon_llm/src/managers/cost_manager.py,sha256=Bu4LUWcKB9JSwIz2m5FxAe5iEN3dVRX
 unrealon_llm/src/managers/request_manager.py,sha256=oMsn2x1P6AF_6C84kmJrl_SCTHpWzgBkR50M9-bIyd0,10702
 unrealon_llm/src/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unrealon_llm/src/modules/html_processor/__init__.py,sha256=-Z3ExABUbUn0l8GEZhbW3WYFJGv7uheGpftiyOji0yg,552
-unrealon_llm/src/modules/html_processor/base_processor.py,sha256=BUSK3wou_KWxxr9nFgFAK_HrZCbVjt44EwM6zGZLIP8,10930
-unrealon_llm/src/modules/html_processor/details_processor.py,sha256=XAQFlf0OvfcgjXDd7cAV_LUf07WrnS_3V2ia3zHPZTA,2033
-unrealon_llm/src/modules/html_processor/listing_processor.py,sha256=PSdFlSlZLZ_dIUS8d_umJKBepPDRyijEv9zruioX2r4,2271
+unrealon_llm/src/modules/html_processor/base_processor.py,sha256=azggKCsE52cUnRAFi-7bak7LZPrYrrH5tJFWxI-HoBo,16423
+unrealon_llm/src/modules/html_processor/details_processor.py,sha256=eiayCPIB9p4PSDtMUEgdPhtb8yu-LqN-WkFd86BuhE4,3479
+unrealon_llm/src/modules/html_processor/listing_processor.py,sha256=h8D79XnVsWkEqcbDoVhcsbS1LYV2AngVo1ROy5oRD7o,3669
 unrealon_llm/src/modules/html_processor/models/__init__.py,sha256=Hn-ztJWI4SokVxhx-cakm5h_Xhjg0rn9_IXyToIjwRE,420
 unrealon_llm/src/modules/html_processor/models/processing_models.py,sha256=SoOE8KCwivU3FGLK3fxx0rKtKXyI-C7ibEHyWCApZAQ,1460
 unrealon_llm/src/modules/html_processor/models/universal_model.py,sha256=Zi3L_t4rcwHpbJKADWSdK7yBErVIGBf3ZRCsZxUsMDM,1738
@@ -240,7 +240,7 @@ unrealon_sdk/src/internal/http_client.py,sha256=uU3BdNYj4ZL16y0BpBxOtWLOo-pE-8LW
 unrealon_sdk/src/internal/websocket_client.py,sha256=1TteTv_6dUMXS5xTXwld6XB2Q0hcOsycLv9l_KEB8aA,15700
 unrealon_sdk/src/provider.py,sha256=kyKjUjuo6s8hcTld8gIc7aO4SM7ozhsUlIM0EXOyblw,14104
 unrealon_sdk/src/utils.py,sha256=nj8a83a7p_RXA985yRdHQxPr2S4rwKiwp1wD3qj7EEU,5440
-unrealon-1.0.5.dist-info/LICENSE,sha256=eEH8mWZW49YMpl4Sh5MtKqkZ8aVTzKQXiNPEnvL14ns,1070
-unrealon-1.0.5.dist-info/METADATA,sha256=NHGstQVOsZSNxDp15PXd7oNIi7NiLvvmOJXbOg0h9Dw,29082
-unrealon-1.0.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-unrealon-1.0.5.dist-info/RECORD,,
+unrealon-1.0.7.dist-info/LICENSE,sha256=eEH8mWZW49YMpl4Sh5MtKqkZ8aVTzKQXiNPEnvL14ns,1070
+unrealon-1.0.7.dist-info/METADATA,sha256=0Ge4C8jAuQrfnYPdBpo8BA60yLUhDGfm8b7gIzG87Cc,29061
+unrealon-1.0.7.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+unrealon-1.0.7.dist-info/RECORD,,

unrealon_driver/src/config/auto_config.py

@@ -204,8 +204,8 @@ class AutoConfigBase(BaseModel):
     def _create_websocket_config(self) -> WebSocketConfig:
         """Create WebSocket configuration."""
         return WebSocketConfig(
-            server_url=os.getenv("WEBSOCKET_URL"),
-            api_key=os.getenv("WEBSOCKET_API_KEY"),
+            server_url=os.getenv("SERVER_URL"),
+            api_key=os.getenv("API_KEY"),
             parser_name=self.parser_id,
             auto_reconnect=True,
             max_reconnect_attempts=10 if self.environment == "production" else 5,
@@ -265,8 +265,8 @@ class AutoConfigBase(BaseModel):
     def _create_daemon_config(self) -> DaemonModeConfig:
         """Create daemon mode configuration."""
         return DaemonModeConfig(
-            server_url=os.getenv("WEBSOCKET_URL"),
-            api_key=os.getenv("WEBSOCKET_API_KEY"),
+            server_url=os.getenv("SERVER_URL"),
+            api_key=os.getenv("API_KEY"),
             auto_reconnect=True,
             connection_timeout=30,
             heartbeat_interval=30,

unrealon_driver/src/execution/daemon_mode.py

@@ -48,12 +48,12 @@ class DaemonMode:
             self.logger.info(f"🔌 Starting daemon mode for: {self.parser.parser_name}")
 
         # Get connection parameters
-        websocket_url = server or self.config.server_url or kwargs.get("websocket_url")
-        websocket_api_key = (
-            api_key or self.config.api_key or kwargs.get("websocket_api_key")
+        server_url = server or self.config.server_url or kwargs.get("server_url")
+        api_key = (
+            api_key or self.config.api_key or kwargs.get("api_key")
         )
 
-        if not websocket_url:
+        if not server_url:
             if self.logger:
                 self.logger.error(
                     "❌ WebSocket server URL not configured for daemon mode!"
@@ -61,7 +61,7 @@ class DaemonMode:
                 self.logger.info("   Set server URL in config or pass as parameter")
             return
 
-        if not websocket_api_key:
+        if not api_key:
             if self.logger:
                 self.logger.error(
                     "❌ WebSocket API key not configured for daemon mode!"
@@ -71,16 +71,16 @@ class DaemonMode:
 
         if self.logger:
             self.logger.info(f"🔌 Starting WebSocket daemon mode")
-            self.logger.info(f"   Server: {websocket_url}")
+            self.logger.info(f"   Server: {server_url}")
             self.logger.info(f"   Parser: {self.parser.parser_id}")
             self.logger.info(
-                f"   API Key: {'***' + websocket_api_key[-4:] if len(websocket_api_key) > 4 else '***'}"
+                f"   API Key: {'***' + api_key[-4:] if len(api_key) > 4 else '***'}"
             )
 
         # Setup WebSocket service configuration with type safety
         websocket_config = WebSocketConfig(
-            server_url=websocket_url,
-            api_key=websocket_api_key,
+            server_url=server_url,
+            api_key=api_key,
             parser_name=self.parser.parser_name,
             auto_reconnect=self.config.auto_reconnect,
             health_check_interval=self.config.health_check_interval,
@@ -110,7 +110,7 @@ class DaemonMode:
 
             # Connect to WebSocket server
             success = await self.parser.websocket.connect(
-                server_url=websocket_url, api_key=websocket_api_key
+                server_url=server_url, api_key=api_key
             )
 
             if not success:

unrealon_llm/src/modules/html_processor/base_processor.py

@@ -10,6 +10,7 @@ import json
 import random
 from typing import Type
 import traceback
+import re
 
 from unrealon_llm.src.core import SmartLLMClient
 from unrealon_llm.src.dto import ChatMessage, MessageRole
@@ -77,6 +78,10 @@ class BaseHTMLProcessor(ABC):
         """Return extraction prompt template for this processor type"""
         pass
 
+    def _trim_system_prompt(self, system_prompt: str) -> str:
+        """Trim system prompt to remove empty lines"""
+        return "\n".join(system_prompt.split("\n")[1:])
+
     async def extract_patterns(self, html_content: str) -> ExtractionResult:
         """
         Extract patterns from HTML using LLM intelligence
@@ -116,15 +121,27 @@ class BaseHTMLProcessor(ABC):
             prompt_tokens=0,
             details={
                 "full_prompt": prompt[:2000] + "..." if len(prompt) > 2000 else prompt,
-                "schema_json": json.dumps(self.schema_class.model_json_schema(), indent=2)
-            }
+                "schema_json": json.dumps(
+                    self.schema_class.model_json_schema(), indent=2
+                ),
+            },
         )
 
+        # Add critical format requirements to the prompt
+        SYSTEM_PROMPT = f"""
+        You are an HTML-to-JSON expert at analyzing {self.processor_type} pages. 
+        You MUST return JSON that EXACTLY matches the Pydantic schema provided. 
+        RESPOND ONLY WITH VALID JSON. 
+        NO EXPLANATIONS, NO TEXT, ONLY JSON! 
+        Include ALL required fields from the schema! 
+        CRITICAL: The 'selectors' field must be a DICTIONARY/OBJECT, not a list!
+        """
+
         # Prepare LLM messages
         messages = [
             ChatMessage(
                 role=MessageRole.SYSTEM,
-                content=f"You are an HTML-to-JSON expert at analyzing {self.processor_type} pages. You MUST return JSON that EXACTLY matches the Pydantic schema provided. RESPOND ONLY WITH VALID JSON. NO EXPLANATIONS, NO TEXT, ONLY JSON! Include ALL required fields from the schema!",
+                content=self._trim_system_prompt(SYSTEM_PROMPT),
             ),
             ChatMessage(
                 role=MessageRole.USER,
@@ -144,8 +161,7 @@ class BaseHTMLProcessor(ABC):
         try:
             # Call LLM
             response = await self.llm_client.chat_completion(
-                messages, 
-                response_model=self.schema_class
+                messages, response_model=self.schema_class
             )
 
             # Log full LLM response for debugging
@@ -167,7 +183,7 @@ class BaseHTMLProcessor(ABC):
             )
 
             # Use the validated model from LLM response
-            if hasattr(response, 'extracted_model') and response.extracted_model:
+            if hasattr(response, "extracted_model") and response.extracted_model:
                 validated_model = response.extracted_model
                 validated_result = validated_model.model_dump()
                 logger.log_html_analysis_completed(
@@ -203,8 +219,36 @@ class BaseHTMLProcessor(ABC):
                             "raw_llm_response": result_data,
                         },
                     )
-                    # Fall back to raw data
-                    validated_result = result_data
+
+                    # 🔥 SMART FALLBACK: Try to fix common LLM format issues
+                    try:
+                        fixed_data = self._fix_llm_response_format(result_data, str(e))
+                        validated_model = self.schema_class(**fixed_data)
+                        validated_result = validated_model.model_dump()
+                        logger.log_html_analysis_completed(
+                            selectors_generated=len(str(fixed_data)),
+                            confidence_score=fixed_data.get("confidence", 0.0),
+                            details={
+                                "processor_type": self.processor_type,
+                                "validation_success": True,
+                                "schema_matched": True,
+                                "format_fixed": True,
+                            },
+                        )
+                    except Exception as fix_error:
+                        logger.log_html_analysis_failed(
+                            error_message=f"Format fixing also failed: {str(fix_error)}",
+                            details={
+                                "processor_type": self.processor_type,
+                                "validation_error": str(e),
+                                "fix_error": str(fix_error),
+                                "raw_llm_response": result_data,
+                            },
+                        )
+                        # Final fallback: create minimal valid structure
+                        validated_result = self._create_fallback_result(
+                            result_data, str(e)
+                        )
 
             # Create Pydantic processing metadata
             processing_info = ProcessingInfo(
@@ -253,12 +297,20 @@ class BaseHTMLProcessor(ABC):
 
         # Add random number to bypass any caching
         cache_buster = random.randint(100000, 999999)
-        
-        schema_prompt = f"""PYDANTIC 2 SCHEMA (Request #{cache_buster}):
-{schema_json}
 
-CRITICAL: Return JSON that EXACTLY matches this schema structure!
-The response must include ALL required fields: detected_item_type, extraction_strategy, confidence, selectors, documentation."""
+        schema_prompt = f"""
+        PYDANTIC 2 SCHEMA (Request #{cache_buster}):
+        {schema_json}
+
+        🚨 CRITICAL FORMAT REQUIREMENTS:
+        1. Return JSON that EXACTLY matches this schema structure!
+        2. The response must include ALL required fields: detected_item_type, extraction_strategy, confidence, selectors, documentation
+        3. The "selectors" field MUST be a DICTIONARY/OBJECT with field names as keys and arrays of CSS selectors as values
+        4. Example: "selectors": {{"title": ["h1.title", ".product-name"], "price": [".price", ".cost"]}}
+        5. DO NOT return "selectors" as a list: ❌ ["h1.title", ".price"]
+        6. DO return "selectors" as a dictionary: ✅ {{"title": ["h1.title"], "price": [".price"]}}
+        """
+        schema_prompt = self._trim_system_prompt(schema_prompt)
 
         return prompt_template.format(
             processor_type=self.processor_type,
@@ -292,3 +344,72 @@ The response must include ALL required fields: detected_item_type, extraction_st
         estimated_cost = (total_tokens / 1_000_000) * 0.25
 
         return estimated_cost
+
+    def _fix_llm_response_format(self, result_data: dict, error_message: str) -> dict:
+        """Fix common LLM response format issues."""
+        fixed_data = result_data.copy()
+
+        # Fix selectors if it's a list instead of dict
+        if "selectors" in fixed_data and isinstance(fixed_data["selectors"], list):
+            logger.log_html_analysis_failed(
+                error_message="Fixing selectors format: list -> dict",
+                details={
+                    "processor_type": self.processor_type,
+                    "original_selectors": fixed_data["selectors"],
+                },
+            )
+
+            # Convert list to dict with generic field names
+            selectors_list = fixed_data["selectors"]
+            fixed_data["selectors"] = {}
+
+            # Try to intelligently map list items to field names
+            field_names = ["item", "title", "price", "description", "image", "link"]
+            for i, selector in enumerate(selectors_list):
+                if i < len(field_names):
+                    field_name = field_names[i]
+                else:
+                    field_name = f"field_{i+1}"
+
+                # Convert single selector to list
+                if isinstance(selector, str):
+                    fixed_data["selectors"][field_name] = [selector]
+                elif isinstance(selector, list):
+                    fixed_data["selectors"][field_name] = selector
+                else:
+                    fixed_data["selectors"][field_name] = [str(selector)]
+
+        # Ensure all required fields exist
+        required_fields = [
+            "detected_item_type",
+            "extraction_strategy",
+            "confidence",
+            "selectors",
+            "documentation",
+        ]
+        for field in required_fields:
+            if field not in fixed_data:
+                if field == "detected_item_type":
+                    fixed_data[field] = "unknown"
+                elif field == "extraction_strategy":
+                    fixed_data[field] = "fallback_strategy"
+                elif field == "confidence":
+                    fixed_data[field] = 0.1
+                elif field == "selectors":
+                    fixed_data[field] = {}
+                elif field == "documentation":
+                    fixed_data[field] = (
+                        "Extraction completed with fallback processing due to format issues."
+                    )
+
+        return fixed_data
+
+    def _create_fallback_result(self, result_data: dict, error_message: str) -> dict:
+        """Create a minimal valid result when all else fails."""
+        return {
+            "detected_item_type": "unknown",
+            "extraction_strategy": "fallback_strategy",
+            "confidence": 0.1,
+            "selectors": {},
+            "documentation": f"Extraction failed due to validation error: {error_message}. Raw data: {str(result_data)[:500]}...",
+        }

unrealon_llm/src/modules/html_processor/details_processor.py

@@ -25,37 +25,61 @@ class DetailsProcessor(BaseHTMLProcessor):
     def get_extraction_prompt_template(self) -> str:
         """Return details-specific extraction prompt template"""
 
-        return """{schema}
-
-[__TASK_DESCRIPTION__]
-Analyze this DETAILS/PRODUCT/ITEM page and generate universal extraction patterns.
-PROCESSOR TYPE: {processor_type}
-THIS IS A DETAILS PAGE containing information about a single item/product/service/article.
-[/__TASK_DESCRIPTION__]
-
-[__INSTRUCTIONS__]
-YOUR TASK:
-Analyze this details page and generate extraction patterns for ANY type of item.
-This could be: product details, service info, article content, job description, real estate listing, person profile, etc.
-
-CRITICAL REQUIREMENTS:
-1. Return simple CSS selectors in the "selectors" object
-2. Include comprehensive markdown documentation
-3. Provide real examples from the actual HTML
-4. Explain the page structure and best extraction approach
-5. Include confidence scores and fallback strategies
-6. Document any special handling needed
-
-ANALYZE THE HTML AND DETERMINE:
-- What type of item this page describes
-- What information is available (specs, pricing, reviews, etc.)
-- How content is structured and organized
-- What actions are possible (buy, contact, etc.)
-- Best extraction strategy for this specific page
-[/__INSTRUCTIONS__]
-
-[__HTML_CONTENT__]
-HTML CONTENT (first 50KB):
-{html_content}
-[/__HTML_CONTENT__]
-"""
+        prompt = """{schema}
+        [__TASK_DESCRIPTION__]
+        Analyze this DETAILS/PRODUCT/ITEM page and generate universal extraction patterns.
+        PROCESSOR TYPE: {processor_type}
+        THIS IS A DETAILS PAGE containing information about a single item/product/service/article.
+        [/__TASK_DESCRIPTION__]
+
+        [__CRITICAL_FORMAT_REQUIREMENTS__]
+        🚨 SELECTORS FORMAT: The "selectors" field MUST be a DICTIONARY/OBJECT, NOT a list!
+        Example of CORRECT format:
+        "selectors": {{
+            "title": ["h1.product-title", "h1.page-title", ".item-name"],
+            "price": [".price", ".cost", "span[data-price]", ".product-price"],
+            "description": [".description", ".product-desc", ".item-details"],
+            "images": ["img.product-image", ".gallery img", "img[src*='product']"],
+            "specifications": [".specs", ".product-specs", ".item-specifications"],
+            "reviews": [".reviews", ".product-reviews", ".customer-reviews"]
+        }}
+
+        ❌ WRONG format (DO NOT USE):
+        "selectors": ["h1.title", ".price", ".description"]
+
+        ✅ CORRECT format (USE THIS):
+        "selectors": {{
+            "title": ["h1.title", ".product-name", "h1[itemprop='name']"],
+            "price": [".price", ".cost", "span[data-price]"],
+            "description": [".description", ".product-desc", ".item-details"]
+        }}
+        [/__CRITICAL_FORMAT_REQUIREMENTS__]
+
+        [__INSTRUCTIONS__]
+        YOUR TASK:
+        Analyze this details page and generate extraction patterns for ANY type of item.
+        This could be: product details, service info, article content, job description, real estate listing, person profile, etc.
+
+        CRITICAL REQUIREMENTS:
+        1. The "selectors" field MUST be a DICTIONARY with field names as keys and arrays of CSS selectors as values
+        2. Include comprehensive markdown documentation
+        3. Provide real examples from the actual HTML
+        4. Explain the page structure and best extraction approach
+        5. Include confidence scores and fallback strategies
+        6. Document any special handling needed
+
+        ANALYZE THE HTML AND DETERMINE:
+        - What type of item this page describes
+        - What information is available (specs, pricing, reviews, etc.)
+        - How content is structured and organized
+        - What actions are possible (buy, contact, etc.)
+        - Best extraction strategy for this specific page
+        [/__INSTRUCTIONS__]
+
+        [__HTML_CONTENT__]
+        HTML CONTENT (first 50KB):
+        {html_content}
+        [/__HTML_CONTENT__]
+        """
+
+        return self._trim_system_prompt(prompt)

unrealon_llm/src/modules/html_processor/listing_processor.py

@@ -25,43 +25,67 @@ class ListingProcessor(BaseHTMLProcessor):
     def get_extraction_prompt_template(self) -> str:
         """Return listing-specific extraction prompt template"""
 
-        return """{schema}
-
-[__TASK_DESCRIPTION__]
-Analyze this LISTING/CATALOG page and generate universal extraction patterns.
-PROCESSOR TYPE: {processor_type}
-THIS IS A LISTING PAGE containing multiple items arranged in a list or grid.
-[/__TASK_DESCRIPTION__]
-
-[__INSTRUCTIONS__]
-YOUR TASK:
-Analyze this listing page and generate extraction patterns for ANY type of items.
-This could be: products, services, articles, jobs, real estate, people, cars, etc.
-
-CRITICAL REQUIREMENTS:
-1. Return simple CSS selectors in the "selectors" object
-2. This is a LISTING PAGE with multiple items
-3. Focus on identifying item containers and individual item patterns
-4. Detect ANY type of items - not just products!
-5. Provide multiple fallback selectors for reliability
-6. Include pagination and navigation patterns
-7. Use realistic confidence scores (0.1-1.0)
-8. Auto-detect what type of content this listing contains
-9. Provide extraction strategy advice
-10. Look for structured data (JSON-LD, microdata)
-11. Generate patterns that work with BeautifulSoup4 .select() method
-12. RETURN JSON that EXACTLY matches the Pydantic schema above!
-
-ANALYZE THE HTML AND DETERMINE:
-- What type of items are listed (products, services, articles, etc.)
-- How items are structured and contained
-- What navigation elements exist
-- What metadata is available
-- Best extraction strategy for this specific page
-[/__INSTRUCTIONS__]
-
-[__HTML_CONTENT__]
-HTML CONTENT (first 50KB):
-{html_content}
-[/__HTML_CONTENT__]
-"""
+        prompt = """{schema}
+
+        [__TASK_DESCRIPTION__]
+        Analyze this LISTING/CATALOG page and generate universal extraction patterns.
+        PROCESSOR TYPE: {processor_type}
+        THIS IS A LISTING PAGE containing multiple items arranged in a list or grid.
+        [/__TASK_DESCRIPTION__]
+
+        [__CRITICAL_FORMAT_REQUIREMENTS__]
+        🚨 SELECTORS FORMAT: The "selectors" field MUST be a DICTIONARY/OBJECT, NOT a list!
+        Example of CORRECT format:
+        "selectors": {{
+            "items_container": ["div.product-grid", "ul.product-list", "div.items"],
+            "item_title": ["h3.product-title", "a.product-link", ".item-name"],
+            "item_price": [".price", ".cost", "span[data-price]"],
+            "item_image": ["img.product-image", ".item-img", "img[src*='product']"],
+            "pagination": [".pagination", ".page-nav", "nav[aria-label='pagination']"]
+        }}
+
+        ❌ WRONG format (DO NOT USE):
+        "selectors": ["div.product", "h3.title", ".price"]
+
+        ✅ CORRECT format (USE THIS):
+        "selectors": {{
+            "items": ["div.product", "li.item", ".product-card"],
+            "titles": ["h3.title", ".product-name", "a[title]"],
+            "prices": [".price", ".cost", "span[data-price]"]
+        }}
+        [/__CRITICAL_FORMAT_REQUIREMENTS__]
+
+        [__INSTRUCTIONS__]
+        YOUR TASK:
+        Analyze this listing page and generate extraction patterns for ANY type of items.
+        This could be: products, services, articles, jobs, real estate, people, cars, etc.
+
+        CRITICAL REQUIREMENTS:
+        1. The "selectors" field MUST be a DICTIONARY with field names as keys and arrays of CSS selectors as values
+        2. This is a LISTING PAGE with multiple items
+        3. Focus on identifying item containers and individual item patterns
+        4. Detect ANY type of items - not just products!
+        5. Provide multiple fallback selectors for reliability
+        6. Include pagination and navigation patterns
+        7. Use realistic confidence scores (0.1-1.0)
+        8. Auto-detect what type of content this listing contains
+        9. Provide extraction strategy advice
+        10. Look for structured data (JSON-LD, microdata)
+        11. Generate patterns that work with BeautifulSoup4 .select() method
+        12. RETURN JSON that EXACTLY matches the Pydantic schema above!
+
+        ANALYZE THE HTML AND DETERMINE:
+        - What type of items are listed (products, services, articles, etc.)
+        - How items are structured and contained
+        - What navigation elements exist
+        - What metadata is available
+        - Best extraction strategy for this specific page
+        [/__INSTRUCTIONS__]
+
+        [__HTML_CONTENT__]
+        HTML CONTENT (first 50KB):
+        {html_content}
+        [/__HTML_CONTENT__]
+        """
+
+        return self._trim_system_prompt(prompt)