unrealon 2.0.13__tar.gz → 2.0.15__tar.gz

{unrealon-2.0.13/unrealon.egg-info → unrealon-2.0.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unrealon
-Version: 2.0.13
+Version: 2.0.15
 Summary: Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities
 Author-email: UnrealOn Team <team@unrealon.com>
 License: MIT
@@ -33,6 +33,7 @@ Requires-Dist: aiohttp>=3.8.0
 Requires-Dist: httpx>=0.23.0
 Requires-Dist: beautifulsoup4>=4.13.5
 Requires-Dist: pydantic-yaml>=1.6.0
+Requires-Dist: jinja2>=3.1.0
 Requires-Dist: rich>=13.0.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: python-socketio>=5.0

{unrealon-2.0.13 → unrealon-2.0.15}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unrealon"
-version = "2.0.13"
+version = "2.0.15"
 description = "Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities"
 authors = [
     {name = "UnrealOn Team", email = "team@unrealon.com"}
@@ -45,6 +45,7 @@ dependencies = [
     "httpx>=0.23.0",
     "beautifulsoup4>=4.13.5",
     "pydantic-yaml>=1.6.0",
+    "jinja2>=3.1.0",
     
     # Additional dependencies for full functionality
     "rich>=13.0.0",
@@ -104,6 +105,7 @@ full = ["selenium>=4.15.0", "playwright>=1.40.0", "undetected-chromedriver>=3.5.
 [project.scripts]
 unrealon = "unrealon_driver.cli:main"
 unrealon-browser = "unrealon_browser.cli.main:main"
+unrealon-installer = "unrealon_installer.cli:main"
 
 [project.urls]
 Homepage = "https://unrealon.com"

{unrealon-2.0.13 → unrealon-2.0.15}/unrealon-driver/src/unrealon_driver/__init__.py

@@ -60,12 +60,8 @@ from .decorators import (
     timing
 )
 
-# Utilities - Cross-platform compatibility
-from .utils import (
-    PlatformCompatibility,
-    ensure_platform_compatibility,
-    get_platform_info
-)
+# Utilities - Time utilities only (platform compatibility moved to installer)
+from .utils import time
 
 __version__ = get_driver_version()
 __author__ = "UnrealOn Team"
@@ -108,8 +104,4 @@ __all__ = [
     "schedule", 
     "timing",
     
-    # Utilities
-    "PlatformCompatibility",
-    "ensure_platform_compatibility",
-    "get_platform_info",
 ]

unrealon-2.0.15/unrealon-driver/src/unrealon_driver/utils/__init__.py

@@ -0,0 +1,10 @@
+"""
+Utility modules for UnrealOn Driver.
+
+Contains cross-platform compatibility and other utility functions.
+"""
+from .time import utc_now
+
+__all__ = [
+    'utc_now',
+]

unrealon-2.0.15/unrealon-driver/src/unrealon_installer/__init__.py

@@ -0,0 +1,11 @@
+"""
+UnrealOn Universal Installer - Clean Edition
+
+One function installs everything. Simple and reliable.
+"""
+
+from .core import install_parser
+from .browser_fixes import fix_browsers, diagnose_browsers, get_browser_status
+
+__version__ = "2.0.0"
+__all__ = ["install_parser", "fix_browsers", "diagnose_browsers", "get_browser_status"]

unrealon-2.0.15/unrealon-driver/src/unrealon_installer/browser_fixes.py

@@ -0,0 +1,227 @@
+"""
+Browser troubleshooting and fixes for Windows.
+"""
+
+import os
+import sys
+import subprocess
+import platform
+from pathlib import Path
+from typing import Dict, List, Optional
+import shutil
+
+
+class BrowserFixer:
+    """Windows browser troubleshooting and fixes."""
+    
+    def __init__(self):
+        self.is_windows = platform.system() == "Windows"
+        self.playwright_cache = self._get_playwright_cache_path()
+    
+    def _get_playwright_cache_path(self) -> Optional[Path]:
+        """Get Playwright cache directory path."""
+        if self.is_windows:
+            return Path(os.environ.get("USERPROFILE", "")) / "AppData" / "Local" / "ms-playwright"
+        elif platform.system() == "Darwin":  # macOS
+            return Path.home() / "Library" / "Caches" / "ms-playwright"
+        else:  # Linux
+            return Path.home() / ".cache" / "ms-playwright"
+    
+    def diagnose_browser_issues(self) -> Dict[str, bool]:
+        """Diagnose common browser issues."""
+        issues = {}
+        
+        # Check if Playwright is installed
+        issues['playwright_installed'] = self._check_playwright_installed()
+        
+        # Check if browsers are installed
+        issues['browsers_installed'] = self._check_browsers_installed()
+        
+        # Check cache directory
+        issues['cache_exists'] = self._check_cache_directory()
+        
+        # Check if browsers can launch
+        issues['browser_launch_test'] = self._test_browser_launch()
+        
+        return issues
+    
+    def _check_playwright_installed(self) -> bool:
+        """Check if Playwright is installed."""
+        try:
+            subprocess.run([sys.executable, "-c", "import playwright"], 
+                         check=True, capture_output=True)
+            return True
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            return False
+    
+    def _check_browsers_installed(self) -> bool:
+        """Check if Playwright browsers are installed."""
+        try:
+            result = subprocess.run([sys.executable, "-m", "playwright", "--version"], 
+                                  check=True, capture_output=True, text=True)
+            return "version" in result.stdout.lower()
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            return False
+    
+    def _check_cache_directory(self) -> bool:
+        """Check if Playwright cache directory exists."""
+        if not self.playwright_cache:
+            return False
+        return self.playwright_cache.exists()
+    
+    def _test_browser_launch(self) -> bool:
+        """Test if browser can launch (quick test)."""
+        try:
+            test_code = """
+from playwright.sync_api import sync_playwright
+try:
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        browser.close()
+    print("OK")
+except Exception as e:
+    print(f"ERROR: {e}")
+"""
+            result = subprocess.run([sys.executable, "-c", test_code], 
+                                  capture_output=True, text=True, timeout=30)
+            return "OK" in result.stdout
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+            return False
+    
+    def fix_browser_issues(self, force_reinstall: bool = False) -> bool:
+        """Fix common browser issues."""
+        print("🔧 Fixing browser issues...")
+        
+        success = True
+        
+        # Step 1: Clear cache if it exists
+        if self._clear_browser_cache():
+            print("✅ Browser cache cleared")
+        else:
+            print("⚠️  Could not clear browser cache")
+        
+        # Step 2: Reinstall Playwright if needed or forced
+        if force_reinstall or not self._check_playwright_installed():
+            if self._reinstall_playwright():
+                print("✅ Playwright reinstalled")
+            else:
+                print("❌ Failed to reinstall Playwright")
+                success = False
+        
+        # Step 3: Install browsers
+        if self._install_browsers():
+            print("✅ Browsers installed")
+        else:
+            print("❌ Failed to install browsers")
+            success = False
+        
+        # Step 4: Test installation
+        if self._test_browser_launch():
+            print("✅ Browser test passed")
+        else:
+            print("❌ Browser test failed")
+            success = False
+        
+        return success
+    
+    def _clear_browser_cache(self) -> bool:
+        """Clear Playwright browser cache."""
+        try:
+            if self.playwright_cache and self.playwright_cache.exists():
+                shutil.rmtree(self.playwright_cache, ignore_errors=True)
+                print(f"Cleared cache: {self.playwright_cache}")
+            return True
+        except Exception as e:
+            print(f"Cache clear error: {e}")
+            return False
+    
+    def _reinstall_playwright(self) -> bool:
+        """Reinstall Playwright completely."""
+        try:
+            # Uninstall
+            subprocess.run([sys.executable, "-m", "pip", "uninstall", "-y", "playwright"], 
+                         check=False, capture_output=True)
+            
+            # Clear pip cache
+            subprocess.run([sys.executable, "-m", "pip", "cache", "purge"], 
+                         check=False, capture_output=True)
+            
+            # Reinstall
+            subprocess.run([sys.executable, "-m", "pip", "install", "playwright"], 
+                         check=True, capture_output=True)
+            
+            return True
+        except subprocess.CalledProcessError:
+            return False
+    
+    def _install_browsers(self) -> bool:
+        """Install Playwright browsers."""
+        try:
+            # Install chromium (most stable)
+            subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"], 
+                         check=True, capture_output=True)
+            
+            # Try to install firefox too (optional)
+            subprocess.run([sys.executable, "-m", "playwright", "install", "firefox"], 
+                         check=False, capture_output=True)
+            
+            return True
+        except subprocess.CalledProcessError:
+            return False
+    
+    def get_browser_info(self) -> Dict[str, str]:
+        """Get browser installation information."""
+        info = {}
+        
+        # Playwright version
+        try:
+            result = subprocess.run([sys.executable, "-m", "playwright", "--version"], 
+                                  capture_output=True, text=True)
+            info['playwright_version'] = result.stdout.strip()
+        except:
+            info['playwright_version'] = "Not installed"
+        
+        # Cache directory
+        info['cache_path'] = str(self.playwright_cache) if self.playwright_cache else "Unknown"
+        info['cache_exists'] = str(self._check_cache_directory())
+        
+        # Browser test
+        info['browser_test'] = "PASS" if self._test_browser_launch() else "FAIL"
+        
+        return info
+
+
+def fix_browsers(force_reinstall: bool = False) -> bool:
+    """
+    Quick browser fix function.
+    
+    Args:
+        force_reinstall: Force complete reinstall
+    
+    Returns:
+        True if successful
+    """
+    fixer = BrowserFixer()
+    return fixer.fix_browser_issues(force_reinstall)
+
+
+def diagnose_browsers() -> Dict[str, bool]:
+    """
+    Quick browser diagnosis.
+    
+    Returns:
+        Dictionary with diagnosis results
+    """
+    fixer = BrowserFixer()
+    return fixer.diagnose_browser_issues()
+
+
+def get_browser_status() -> Dict[str, str]:
+    """
+    Get browser status information.
+    
+    Returns:
+        Dictionary with browser info
+    """
+    fixer = BrowserFixer()
+    return fixer.get_browser_info()

unrealon-2.0.15/unrealon-driver/src/unrealon_installer/core.py

@@ -0,0 +1,143 @@
+"""
+Core installer logic. Clean and simple.
+"""
+
+import os
+import sys
+import subprocess
+import platform
+from pathlib import Path
+from typing import Dict, Any
+
+from .platform import apply_platform_fixes, check_system_requirements
+from .templates import TemplateEngine
+from .browser_fixes import fix_browsers, diagnose_browsers
+
+
+def install_parser(parser_name: str, parser_path: str = ".") -> bool:
+    """
+    Install everything for a UnrealOn parser.
+    
+    Args:
+        parser_name: Name like "upbit_concurrent"
+        parser_path: Path to parser directory
+    
+    Returns:
+        True if success, False if failed
+    """
+    print(f"🚀 Installing {parser_name}")
+    print("=" * 40)
+    
+    parser_dir = Path(parser_path).resolve()
+    os.chdir(parser_dir)
+    
+    # 1. Apply platform fixes
+    print("🔧 Applying platform fixes...")
+    apply_platform_fixes()
+    print("✅ Platform fixes applied")
+    
+    # 2. Check system
+    print("🔍 Checking system...")
+    requirements = check_system_requirements()
+    failed = [k for k, v in requirements.items() if not v]
+    if failed:
+        print(f"❌ Missing: {', '.join(failed)}")
+        return False
+    
+    print(f"✅ Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")
+    
+    # 3. Check if basic tools are available (optional)
+    _check_tools_availability()
+    
+    # 4. Create batch files (Windows only)
+    if platform.system() == "Windows":
+        _create_batch_files(parser_name)
+    
+    print(f"\n✅ {parser_name} setup completed!")
+    print("\n📋 Next steps:")
+    print("1. Install dependencies: pip install -r requirements.txt")
+    print("2. Install browsers: playwright install chromium")
+    print("3. Run parser: python main.py 5 2")
+    if platform.system() == "Windows":
+        print("   Or use: START.bat (includes setup menu)")
+    
+    return True
+
+
+def _check_tools_availability():
+    """Check if basic tools are available (non-blocking)."""
+    print("🔍 Checking available tools...")
+    
+    # Check pip
+    try:
+        subprocess.run([sys.executable, "-m", "pip", "--version"], 
+                     check=True, capture_output=True)
+        print("✅ pip available")
+    except:
+        print("⚠️  pip not available")
+    
+    # Check if requirements.txt exists
+    if Path("requirements.txt").exists():
+        print("✅ requirements.txt found")
+        print("💡 To install dependencies: pip install -r requirements.txt")
+    else:
+        print("⚠️  requirements.txt not found")
+    
+    # Check playwright
+    try:
+        subprocess.run([sys.executable, "-c", "import playwright"], 
+                     check=True, capture_output=True)
+        print("✅ playwright available")
+    except:
+        print("⚠️  playwright not available")
+        print("💡 To install: pip install playwright && playwright install chromium")
+
+
+def _create_batch_files(parser_name: str):
+    """Create Windows batch files using Jinja2 templates."""
+    print("🪟 Creating Windows batch files...")
+    
+    # Detect parser config
+    config = _detect_parser_config()
+    
+    # Create template engine
+    engine = TemplateEngine()
+    
+    # Generate batch files
+    start_content = engine.render_start_bat(parser_name, config)
+    Path("START.bat").write_text(start_content, encoding="utf-8")
+    
+    quick_content = engine.render_quick_run_bat(parser_name, config)
+    Path("QUICK_RUN.bat").write_text(quick_content, encoding="utf-8")
+    
+    test_content = engine.render_test_bat(parser_name, config)
+    Path("TEST.bat").write_text(test_content, encoding="utf-8")
+    
+    print("✅ Created START.bat, QUICK_RUN.bat, TEST.bat")
+
+
+def _detect_parser_config() -> Dict[str, Any]:
+    """Detect parser configuration from files."""
+    config = {
+        'browsers_needed': ['chromium'],
+        'proxy_support': False,
+        'supports_persistent': False
+    }
+    
+    # Check main.py for persistent mode support
+    if Path("main.py").exists():
+        main_content = Path("main.py").read_text()
+        if "--persistent" in main_content:
+            config['supports_persistent'] = True
+    
+    # Check for proxy config
+    if Path("src/proxy_config.py").exists() or Path("proxy_config.py").exists():
+        config['proxy_support'] = True
+    
+    # Check pyproject.toml for browser requirements
+    if Path("pyproject.toml").exists():
+        toml_content = Path("pyproject.toml").read_text()
+        if "firefox" in toml_content.lower():
+            config['browsers_needed'].append('firefox')
+    
+    return config

unrealon-2.0.15/unrealon-driver/src/unrealon_installer/platform.py

@@ -0,0 +1,122 @@
+"""
+Platform compatibility fixes. Clean and simple.
+"""
+
+import asyncio
+import sys
+import platform
+import warnings
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def apply_platform_fixes():
+    """Apply all necessary platform fixes automatically."""
+    system = platform.system()
+    fixes = []
+    
+    if system == "Windows":
+        fixes.extend(_apply_windows_fixes())
+    elif system == "Darwin":  # macOS
+        fixes.extend(_apply_macos_fixes())
+    elif system == "Linux":
+        fixes.extend(_apply_linux_fixes())
+    
+    if fixes:
+        logger.info(f"✅ Applied platform fixes: {', '.join(fixes)}")
+    
+    return fixes
+
+
+def _apply_windows_fixes():
+    """Windows-specific fixes."""
+    fixes = []
+    
+    # Asyncio event loop policy
+    if sys.version_info >= (3, 8):
+        try:
+            asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+            fixes.append("ProactorEventLoop")
+        except Exception:
+            pass
+    
+    # Console encoding
+    try:
+        if hasattr(sys.stdout, 'reconfigure'):
+            sys.stdout.reconfigure(encoding='utf-8')
+            sys.stderr.reconfigure(encoding='utf-8')
+            fixes.append("UTF-8 encoding")
+    except Exception:
+        pass
+    
+    # Suppress warnings
+    warnings.filterwarnings("ignore", category=ResourceWarning)
+    fixes.append("ResourceWarning suppression")
+    
+    return fixes
+
+
+def _apply_macos_fixes():
+    """macOS-specific fixes."""
+    fixes = []
+    
+    # SSL context fix
+    try:
+        import ssl
+        ssl._create_default_https_context = ssl._create_unverified_context
+        fixes.append("SSL context")
+    except Exception:
+        pass
+    
+    # File descriptor limits
+    try:
+        import resource
+        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if soft < 1024:
+            resource.setrlimit(resource.RLIMIT_NOFILE, (min(1024, hard), hard))
+            fixes.append("File descriptor limit")
+    except Exception:
+        pass
+    
+    return fixes
+
+
+def _apply_linux_fixes():
+    """Linux-specific fixes."""
+    fixes = []
+    
+    # File descriptor limits
+    try:
+        import resource
+        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if soft < 2048:
+            resource.setrlimit(resource.RLIMIT_NOFILE, (min(2048, hard), hard))
+            fixes.append("File descriptor limit")
+    except Exception:
+        pass
+    
+    # Headless display
+    import os
+    if not os.environ.get('DISPLAY'):
+        os.environ['DISPLAY'] = ':99'
+        fixes.append("Headless display")
+    
+    return fixes
+
+
+def get_platform_info():
+    """Get platform information."""
+    return {
+        'platform': platform.system(),
+        'architecture': platform.architecture()[0],
+        'python_version': f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+    }
+
+
+def check_system_requirements():
+    """Check if system meets requirements."""
+    return {
+        'python_version': sys.version_info >= (3, 9),
+        'platform_supported': platform.system() in ['Windows', 'Darwin', 'Linux']
+    }

unrealon-2.0.15/unrealon-driver/src/unrealon_installer/templates.py

@@ -0,0 +1,60 @@
+"""
+Jinja2 template engine for batch files.
+"""
+
+from pathlib import Path
+from typing import Dict, Any
+import json
+from jinja2 import Environment, FileSystemLoader
+
+
+class TemplateEngine:
+    """Jinja2-based template engine for Windows batch files."""
+    
+    def __init__(self):
+        # Get batch templates directory
+        templates_dir = Path(__file__).parent / "batch_templates"
+        
+        # Create Jinja2 environment
+        self.env = Environment(
+            loader=FileSystemLoader(str(templates_dir)),
+            trim_blocks=True,
+            lstrip_blocks=True
+        )
+    
+    def render_start_bat(self, parser_name: str, config: Dict[str, Any]) -> str:
+        """Render START.bat file."""
+        template = self.env.get_template('start.bat.j2')
+        
+        context = {
+            'parser_name': parser_name,
+            'config': config,
+            'config_json': json.dumps(config, indent=None),
+            'browsers_list': ', '.join(config.get('browsers_needed', ['chromium'])),
+            'has_proxy': config.get('proxy_support', False),
+            'has_persistent': config.get('supports_persistent', False)
+        }
+        
+        return template.render(**context)
+    
+    def render_quick_run_bat(self, parser_name: str, config: Dict[str, Any]) -> str:
+        """Render QUICK_RUN.bat file."""
+        template = self.env.get_template('quick_run.bat.j2')
+        
+        context = {
+            'parser_name': parser_name,
+            'config': config
+        }
+        
+        return template.render(**context)
+    
+    def render_test_bat(self, parser_name: str, config: Dict[str, Any]) -> str:
+        """Render TEST.bat file."""
+        template = self.env.get_template('test.bat.j2')
+        
+        context = {
+            'parser_name': parser_name,
+            'config': config
+        }
+        
+        return template.render(**context)

{unrealon-2.0.13 → unrealon-2.0.15/unrealon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unrealon
-Version: 2.0.13
+Version: 2.0.15
 Summary: Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities
 Author-email: UnrealOn Team <team@unrealon.com>
 License: MIT
@@ -33,6 +33,7 @@ Requires-Dist: aiohttp>=3.8.0
 Requires-Dist: httpx>=0.23.0
 Requires-Dist: beautifulsoup4>=4.13.5
 Requires-Dist: pydantic-yaml>=1.6.0
+Requires-Dist: jinja2>=3.1.0
 Requires-Dist: rich>=13.0.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: python-socketio>=5.0

{unrealon-2.0.13 → unrealon-2.0.15}/unrealon.egg-info/SOURCES.txt

@@ -128,8 +128,12 @@ unrealon-driver/src/unrealon_driver/managers/registry.py
 unrealon-driver/src/unrealon_driver/managers/threading.py
 unrealon-driver/src/unrealon_driver/managers/update.py
 unrealon-driver/src/unrealon_driver/utils/__init__.py
-unrealon-driver/src/unrealon_driver/utils/platform_compatibility.py
 unrealon-driver/src/unrealon_driver/utils/time.py
+unrealon-driver/src/unrealon_installer/__init__.py
+unrealon-driver/src/unrealon_installer/browser_fixes.py
+unrealon-driver/src/unrealon_installer/core.py
+unrealon-driver/src/unrealon_installer/platform.py
+unrealon-driver/src/unrealon_installer/templates.py
 unrealon.egg-info/PKG-INFO
 unrealon.egg-info/SOURCES.txt
 unrealon.egg-info/dependency_links.txt

{unrealon-2.0.13 → unrealon-2.0.15}/unrealon.egg-info/entry_points.txt

@@ -1,3 +1,4 @@
 [console_scripts]
 unrealon = unrealon_driver.cli:main
 unrealon-browser = unrealon_browser.cli.main:main
+unrealon-installer = unrealon_installer.cli:main

{unrealon-2.0.13 → unrealon-2.0.15}/unrealon.egg-info/requires.txt

@@ -10,6 +10,7 @@ aiohttp>=3.8.0
 httpx>=0.23.0
 beautifulsoup4>=4.13.5
 pydantic-yaml>=1.6.0
+jinja2>=3.1.0
 rich>=13.0.0
 pyyaml>=6.0
 python-socketio>=5.0