unrealon 2.0.31__py3-none-any.whl → 2.0.33__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: unrealon
3
- Version: 2.0.31
3
+ Version: 2.0.33
4
4
  Summary: Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities
5
5
  Author-email: UnrealOn Team <team@unrealon.com>
6
6
  License: MIT
@@ -33,7 +33,6 @@ Requires-Dist: aiohttp>=3.8.0
33
33
  Requires-Dist: httpx>=0.23.0
34
34
  Requires-Dist: beautifulsoup4>=4.13.5
35
35
  Requires-Dist: pydantic-yaml>=1.6.0
36
- Requires-Dist: jinja2>=3.1.0
37
36
  Requires-Dist: rich>=13.0.0
38
37
  Requires-Dist: pyyaml>=6.0
39
38
  Requires-Dist: python-socketio>=5.0
@@ -113,6 +113,8 @@ unrealon_driver/driver/monitoring/health.py,sha256=rOH0wChDBTnDbV7MstB21JsJJUMGM
113
113
  unrealon_driver/driver/utilities/__init__.py,sha256=YCTR7CXu3wlynKNDJh4YsE1uhZ_QrtoQjLNArleMChM,232
114
114
  unrealon_driver/driver/utilities/logging.py,sha256=2my2QnkAa6Hdw-TfO4oOQ94yGc-CjlRfnWsnbWIKb9o,1799
115
115
  unrealon_driver/driver/utilities/serialization.py,sha256=wTCSVrEloykiGN4K1JXbk2aqNKm7W90aWXmzhcLyAZc,2123
116
+ unrealon_driver/installer/__init__.py,sha256=PraOjOg-cN1zOtuhPSTE5vCGPSMzWtEBYU8A05GWEf8,227
117
+ unrealon_driver/installer/platform.py,sha256=U_8FJZk0C8M0ujpfzcpOPWEoUrT6asTNEIhsN0n2bCg,5081
116
118
  unrealon_driver/managers/__init__.py,sha256=zJJsOb6Oodg7l00v4ncKUytnyeaZM887pHY8-eSuWdU,981
117
119
  unrealon_driver/managers/base.py,sha256=GkuXillg9uqqnx6RL682fmKgK-7JyqYlH6DFUgyN4F8,5445
118
120
  unrealon_driver/managers/browser.py,sha256=bc6O2NyC4FV82mb9sat48_k8s1c3IGY4i90ddMVWRIo,5432
@@ -125,17 +127,9 @@ unrealon_driver/managers/threading.py,sha256=djw5cSC99dfBKmep3IJ_8IgxQceMXtNvCp5
125
127
  unrealon_driver/managers/update.py,sha256=-hohVxGXpj5bZ6ZTQN6NH1RK9Pd6GVzCMtu3GS2SdcQ,3582
126
128
  unrealon_driver/utils/__init__.py,sha256=2Sz3eats5q4O2fDmefDuJt8M_zkN6xrS-9xXntWZWFc,168
127
129
  unrealon_driver/utils/time.py,sha256=Oxk1eicKeZl8ZWbf7gu1Ll716k6CpXmVj67FHSnPIsA,184
128
- unrealon_installer/__init__.py,sha256=C07m5dnhmUTOj8DmMNvG7soD55WZvnXdV2YRAvd6ehI,456
129
- unrealon_installer/browser_fixes.py,sha256=f67T41e4vfOpDAu1hiDEZ5-BLJ663Bl7oWiHD83rR6E,8081
130
- unrealon_installer/core.py,sha256=VvAi7iNBFUrJFUkANQ0gv4tyYCaf-cgtUboXVnrjr50,3826
131
- unrealon_installer/platform.py,sha256=40XrpEN7lI1nALK06JF7wJ_fRhl0IQE70crW44KrC5Y,5595
132
- unrealon_installer/templates.py,sha256=PBGfKWlrhtd92h2c68RhtXgICATmHkGObRn6nT0e2Jg,1920
133
- unrealon_installer/batch_templates/quick_run.bat.j2,sha256=AQRh8qBbspeOr4rG1SWWar6FRXQVKyNTQ36sIvx0WlE,492
134
- unrealon_installer/batch_templates/start.bat.j2,sha256=flApkVbdqQ6qg0vyHsIME9GDLGmdg3nEbA-k1X-ixUE,6138
135
- unrealon_installer/batch_templates/test.bat.j2,sha256=eq_Bj-zN6jUZWLwgzPtYDCr4YbgjseMSIfDPYT5ODFw,1348
136
- unrealon-2.0.31.dist-info/LICENSE,sha256=eEH8mWZW49YMpl4Sh5MtKqkZ8aVTzKQXiNPEnvL14ns,1070
137
- unrealon-2.0.31.dist-info/METADATA,sha256=Bx_-kHnogbJ8Kh1YEl4ARoEl7nfgFp7gevdZfUtupac,15718
138
- unrealon-2.0.31.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
139
- unrealon-2.0.31.dist-info/entry_points.txt,sha256=tBJgozewpyuXznEYIsLwfE1s16VRjy3Wizhuyh26zb4,153
140
- unrealon-2.0.31.dist-info/top_level.txt,sha256=qN6Q72fe4_i8mTOhYcO3fhGa3g4dmBgvZOsqmK4j8D8,66
141
- unrealon-2.0.31.dist-info/RECORD,,
130
+ unrealon-2.0.33.dist-info/LICENSE,sha256=eEH8mWZW49YMpl4Sh5MtKqkZ8aVTzKQXiNPEnvL14ns,1070
131
+ unrealon-2.0.33.dist-info/METADATA,sha256=8wOfd43TlGdZ-0sB4AaXtmCPP1mX1y7wsTN9lqPHKvw,15689
132
+ unrealon-2.0.33.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
133
+ unrealon-2.0.33.dist-info/entry_points.txt,sha256=k0qM-eotpajkKUq-almJmxj9afhXprZ6IkvQkSdcKhI,104
134
+ unrealon-2.0.33.dist-info/top_level.txt,sha256=Gu8IeIfIVfUxdi-h-F0nKMQxo15pjhHZ0aTadXTpRE8,47
135
+ unrealon-2.0.33.dist-info/RECORD,,
@@ -1,4 +1,3 @@
1
1
  [console_scripts]
2
2
  unrealon = unrealon_driver.cli:main
3
3
  unrealon-browser = unrealon_browser.cli.main:main
4
- unrealon-installer = unrealon_installer.cli:main
@@ -1,4 +1,3 @@
1
1
  unrealon_browser
2
2
  unrealon_core
3
3
  unrealon_driver
4
- unrealon_installer
@@ -0,0 +1,8 @@
1
+ """
2
+ Simple installer utilities for UnrealOn parsers.
3
+ No bullshit, just platform fixes.
4
+ """
5
+
6
+ from .platform import apply_platform_fixes, cleanup_asyncio_resources
7
+
8
+ __all__ = ["apply_platform_fixes", "cleanup_asyncio_resources"]
@@ -139,23 +139,6 @@ def _apply_linux_fixes():
139
139
  return fixes
140
140
 
141
141
 
142
- def get_platform_info():
143
- """Get platform information."""
144
- return {
145
- 'platform': platform.system(),
146
- 'architecture': platform.architecture()[0],
147
- 'python_version': f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
148
- }
149
-
150
-
151
- def check_system_requirements():
152
- """Check if system meets requirements."""
153
- return {
154
- 'python_version': sys.version_info >= (3, 9),
155
- 'platform_supported': platform.system() in ['Windows', 'Darwin', 'Linux']
156
- }
157
-
158
-
159
142
  def cleanup_asyncio_resources():
160
143
  """Force cleanup of all asyncio resources. Call this at program exit on Windows."""
161
144
  if platform.system() != "Windows":
@@ -1,12 +0,0 @@
1
- """
2
- UnrealOn Universal Installer - Clean Edition
3
-
4
- One function installs everything. Simple and reliable.
5
- """
6
-
7
- from .core import install_parser
8
- from .browser_fixes import fix_browsers, diagnose_browsers, get_browser_status
9
- from .platform import apply_platform_fixes, cleanup_asyncio_resources
10
-
11
- __version__ = "2.0.0"
12
- __all__ = ["install_parser", "fix_browsers", "diagnose_browsers", "get_browser_status", "apply_platform_fixes", "cleanup_asyncio_resources"]
@@ -1,24 +0,0 @@
1
- @echo off
2
- REM {{ parser_name }} - Quick Run
3
- REM Fast execution without menus
4
-
5
- title {{ parser_name }} Quick Run
6
-
7
- echo.
8
- echo {{ parser_name }} - Quick Run Mode
9
- echo {{ "=" * (parser_name|length + 20) }}
10
- echo.
11
-
12
- REM Setup environment
13
- set PYTHONIOENCODING=utf-8
14
- set PLAYWRIGHT_BROWSERS_PATH=0
15
- set PYTHONUNBUFFERED=1
16
-
17
- REM Platform fixes are applied automatically by the parser itself
18
-
19
- echo Starting quick test: 5 requests, 2 concurrent...
20
- python main.py 5 2
21
-
22
- echo.
23
- echo Quick run completed!
24
- pause
@@ -1,250 +0,0 @@
1
- @echo off
2
- REM {{ parser_name }} - Universal Windows Installer
3
- REM Generated by UnrealOn Universal Installer v2.0.0
4
-
5
- title {{ parser_name }} Parser
6
-
7
- echo.
8
- echo {{ parser_name }} Parser - Windows Edition
9
- echo {{ "=" * (parser_name|length + 25) }}
10
- echo UnrealOn Driver-based parser with universal installer
11
- echo Browsers: {{ browsers_list }}
12
- echo Proxy Support: {% if has_proxy %}Yes{% else %}No{% endif %}
13
- echo.
14
-
15
- REM Setup environment variables
16
- set PYTHONIOENCODING=utf-8
17
- set PLAYWRIGHT_BROWSERS_PATH=0
18
- set PYTHONUNBUFFERED=1
19
- set UNREALON_PARSER_NAME={{ parser_name }}
20
-
21
- REM Platform fixes are applied automatically by the parser itself
22
-
23
- :main_menu
24
- echo Main Menu:
25
- echo.
26
- echo [1] RUN PARSER (Quick Start)
27
- echo [2] SETUP SYSTEM (First Time Setup)
28
- echo [3] MANAGE BROWSERS (Install/Fix Browsers)
29
- echo [4] MANAGE DEPENDENCIES (Install/Update Packages)
30
- echo [5] SYSTEM STATUS (Check Installation)
31
- echo [6] TROUBLESHOOTING (Fix Issues)
32
- echo [7] HELP and INFO
33
- echo [8] EXIT
34
- echo.
35
- set /p choice="Enter your choice (1-8): "
36
-
37
- if "%choice%"=="1" goto run_parser
38
- if "%choice%"=="2" goto setup_system
39
- if "%choice%"=="3" goto manage_browsers
40
- if "%choice%"=="4" goto manage_dependencies
41
- if "%choice%"=="5" goto system_status
42
- if "%choice%"=="6" goto troubleshooting
43
- if "%choice%"=="7" goto help_info
44
- if "%choice%"=="8" goto exit_script
45
- goto main_menu
46
-
47
- :run_parser
48
- cls
49
- echo.
50
- echo {{ parser_name }} Parser - Run Mode
51
- echo {{ "=" * (parser_name|length + 25) }}
52
- echo.
53
-
54
- echo Choose run mode:
55
- echo [1] QUICK RUN (default parameters)
56
- echo [2] TEST RUN (minimal test)
57
- {% if has_persistent %}echo [3] PERSISTENT mode (continuous){% endif %}
58
- echo [4] CUSTOM mode (specify arguments)
59
- echo.
60
- set /p mode="Enter mode (1-{% if has_persistent %}4{% else %}3{% endif %}): "
61
-
62
- if "%mode%"=="1" (
63
- echo Starting default run...
64
- python main.py
65
- )
66
- if "%mode%"=="2" (
67
- echo Starting test run...
68
- python main.py 1 1
69
- )
70
- {% if has_persistent %}
71
- if "%mode%"=="3" (
72
- echo Starting persistent mode...
73
- python main.py --persistent
74
- )
75
- if "%mode%"=="4" (
76
- set /p args="Enter arguments (e.g. 5 2): "
77
- python main.py %args%
78
- )
79
- {% else %}
80
- if "%mode%"=="3" (
81
- set /p args="Enter arguments (e.g. 5 2): "
82
- python main.py %args%
83
- )
84
- {% endif %}
85
-
86
- echo.
87
- echo Press any key to return to main menu...
88
- pause >nul
89
- goto main_menu
90
-
91
- :setup_system
92
- cls
93
- echo.
94
- echo System Setup
95
- echo =============
96
- echo.
97
- echo Installing Python dependencies...
98
- pip install -r requirements.txt
99
- echo.
100
- echo Installing browsers...
101
- playwright install chromium
102
- echo.
103
- echo Setup complete!
104
- echo.
105
- pause
106
- goto main_menu
107
-
108
- :manage_browsers
109
- cls
110
- echo.
111
- echo Browser Management
112
- echo ==================
113
- echo.
114
- echo [1] Install all browsers
115
- echo [2] Install Chromium only
116
- echo [3] Install Firefox only
117
- echo [4] Check browser status
118
- echo [5] Back to main menu
119
- echo.
120
- set /p browser_choice="Enter choice (1-5): "
121
-
122
- if "%browser_choice%"=="1" playwright install
123
- if "%browser_choice%"=="2" playwright install chromium
124
- if "%browser_choice%"=="3" playwright install firefox
125
- if "%browser_choice%"=="4" (
126
- echo Checking browser status...
127
- playwright --version
128
- python -c "from playwright.sync_api import sync_playwright; print('Browsers OK')" 2>nul || echo "Browser check failed"
129
- )
130
- if "%browser_choice%"=="5" goto main_menu
131
-
132
- echo.
133
- pause
134
- goto main_menu
135
-
136
- :manage_dependencies
137
- cls
138
- echo.
139
- echo Dependency Management
140
- echo =====================
141
- echo.
142
- echo [1] Install/Update all dependencies
143
- echo [2] Install production dependencies only
144
- echo [3] Check dependency status
145
- echo [4] Back to main menu
146
- echo.
147
- set /p dep_choice="Enter choice (1-4): "
148
-
149
- if "%dep_choice%"=="1" (
150
- pip install -r requirements.txt
151
- pip install --upgrade unrealon
152
- )
153
- if "%dep_choice%"=="2" pip install -r requirements.txt
154
- if "%dep_choice%"=="3" (
155
- echo Checking dependencies...
156
- pip list | findstr unrealon
157
- python -c "import unrealon_driver; print('UnrealOn Driver:', unrealon_driver.__version__)" 2>nul || echo "UnrealOn Driver not found"
158
- )
159
- if "%dep_choice%"=="4" goto main_menu
160
-
161
- echo.
162
- pause
163
- goto main_menu
164
-
165
- :system_status
166
- cls
167
- echo.
168
- echo System Status Check
169
- echo ===================
170
- echo.
171
-
172
- echo Python Version:
173
- python --version
174
-
175
- echo.
176
- echo Platform Information:
177
- python -c "from unrealon_installer.platform import get_platform_info; import json; print(json.dumps(get_platform_info(), indent=2))" 2>nul || echo "Platform info not available"
178
-
179
- echo.
180
- echo UnrealOn Driver Status:
181
- python -c "import unrealon_driver; print('Version:', unrealon_driver.__version__)" 2>nul || echo "UnrealOn Driver not installed"
182
-
183
- echo.
184
- echo Browser Status:
185
- playwright --version 2>nul || echo "Playwright not installed"
186
-
187
- echo.
188
- echo Dependencies:
189
- pip list | findstr -i "unrealon playwright requests"
190
-
191
- echo.
192
- pause
193
- goto main_menu
194
-
195
- :troubleshooting
196
- cls
197
- echo.
198
- echo Troubleshooting Guide
199
- echo =====================
200
- echo.
201
- echo Common issues and solutions:
202
- echo.
203
- echo 1. "ModuleNotFoundError: No module named 'unrealon_driver'"
204
- echo Solution: Run "pip install -r requirements.txt"
205
- echo.
206
- echo 2. "Browser not found" errors
207
- echo Solution: Run "playwright install chromium"
208
- echo.
209
- echo 3. "Permission denied" errors
210
- echo Solution: Run as Administrator
211
- echo.
212
- echo 4. Proxy connection issues
213
- echo Solution: Check proxy configuration in proxy_config.py
214
- echo.
215
- echo 5. SSL certificate errors
216
- echo Solution: Platform fixes are applied automatically
217
- echo.
218
- echo Press any key to return to main menu...
219
- pause >nul
220
- goto main_menu
221
-
222
- :help_info
223
- cls
224
- echo.
225
- echo {{ parser_name }} Parser - Help and Information
226
- echo {{ "=" * (parser_name|length + 25) }}
227
- echo.
228
- echo This is a UnrealOn Driver-based parser with the following features:
229
- echo.
230
- echo - Multi-concurrent API processing
231
- echo - Browser automation with Playwright
232
- echo - Proxy support (if configured)
233
- echo - Cross-platform compatibility
234
- {% if has_persistent %}echo - Persistent monitoring mode{% endif %}
235
- echo - Automatic dependency management
236
- echo.
237
- echo Configuration:
238
- echo {{ config_json }}
239
- echo.
240
- echo For more information, visit: https://github.com/unrealon
241
- echo.
242
- echo Press any key to return to main menu...
243
- pause >nul
244
- goto main_menu
245
-
246
- :exit_script
247
- echo.
248
- echo Thank you for using {{ parser_name }} Parser!
249
- echo.
250
- exit /b 0
@@ -1,56 +0,0 @@
1
- @echo off
2
- REM {{ parser_name }} - System Test
3
- REM Comprehensive system and functionality test
4
-
5
- title {{ parser_name }} System Test
6
-
7
- echo.
8
- echo {{ parser_name }} - System Test
9
- echo {{ "=" * (parser_name|length + 15) }}
10
- echo.
11
-
12
- REM Setup environment
13
- set PYTHONIOENCODING=utf-8
14
- set PLAYWRIGHT_BROWSERS_PATH=0
15
- set PYTHONUNBUFFERED=1
16
-
17
- echo [1/5] Testing Python environment...
18
- python --version || (
19
- echo ERROR: Python not found!
20
- pause
21
- exit /b 1
22
- )
23
-
24
- echo [2/5] Testing UnrealOn Driver...
25
- python -c "import unrealon_driver; print('UnrealOn Driver version:', unrealon_driver.__version__)" || (
26
- echo ERROR: UnrealOn Driver not installed!
27
- echo Run: pip install -r requirements.txt
28
- pause
29
- exit /b 1
30
- )
31
-
32
- echo [3/5] Testing platform compatibility...
33
- python -c "from unrealon_installer.platform import get_platform_info; print('Platform OK')" || (
34
- echo ERROR: Platform compatibility issues!
35
- pause
36
- exit /b 1
37
- )
38
-
39
- echo [4/5] Testing browser availability...
40
- playwright --version || (
41
- echo ERROR: Playwright not installed!
42
- echo Run: playwright install chromium
43
- pause
44
- exit /b 1
45
- )
46
-
47
- echo [5/5] Running parser test...
48
- python main.py 1 1
49
-
50
- echo.
51
- echo {{ "=" * (parser_name|length + 15) }}
52
- echo All tests completed successfully!
53
- echo {{ parser_name }} is ready to use.
54
- echo {{ "=" * (parser_name|length + 15) }}
55
- echo.
56
- pause
@@ -1,232 +0,0 @@
1
- """
2
- Browser troubleshooting and fixes for Windows.
3
- """
4
-
5
- import os
6
- import sys
7
- import subprocess
8
- import platform
9
- from pathlib import Path
10
- from typing import Dict, List, Optional
11
- import shutil
12
-
13
-
14
- class BrowserFixer:
15
- """Windows browser troubleshooting and fixes."""
16
-
17
- def __init__(self):
18
- self.is_windows = platform.system() == "Windows"
19
- self.playwright_cache = self._get_playwright_cache_path()
20
-
21
- def _get_playwright_cache_path(self) -> Optional[Path]:
22
- """Get Playwright cache directory path."""
23
- if self.is_windows:
24
- return Path(os.environ.get("USERPROFILE", "")) / "AppData" / "Local" / "ms-playwright"
25
- elif platform.system() == "Darwin": # macOS
26
- return Path.home() / "Library" / "Caches" / "ms-playwright"
27
- else: # Linux
28
- return Path.home() / ".cache" / "ms-playwright"
29
-
30
- def diagnose_browser_issues(self) -> Dict[str, bool]:
31
- """Diagnose common browser issues."""
32
- issues = {}
33
-
34
- # Check if Playwright is installed
35
- issues['playwright_installed'] = self._check_playwright_installed()
36
-
37
- # Check if browsers are installed
38
- issues['browsers_installed'] = self._check_browsers_installed()
39
-
40
- # Check cache directory
41
- issues['cache_exists'] = self._check_cache_directory()
42
-
43
- # Check if browsers can launch
44
- issues['browser_launch_test'] = self._test_browser_launch()
45
-
46
- return issues
47
-
48
- def _check_playwright_installed(self) -> bool:
49
- """Check if Playwright is installed."""
50
- try:
51
- subprocess.run([sys.executable, "-c", "import playwright"],
52
- check=True, capture_output=True)
53
- return True
54
- except (subprocess.CalledProcessError, FileNotFoundError):
55
- return False
56
-
57
- def _check_browsers_installed(self) -> bool:
58
- """Check if Playwright browsers are installed."""
59
- try:
60
- result = subprocess.run([sys.executable, "-m", "playwright", "--version"],
61
- check=True, capture_output=True, text=True)
62
- return "version" in result.stdout.lower()
63
- except (subprocess.CalledProcessError, FileNotFoundError):
64
- return False
65
-
66
- def _check_cache_directory(self) -> bool:
67
- """Check if Playwright cache directory exists."""
68
- if not self.playwright_cache:
69
- return False
70
- return self.playwright_cache.exists()
71
-
72
- def _test_browser_launch(self) -> bool:
73
- """Test if browser can launch (quick test)."""
74
- try:
75
- test_code = """
76
- from playwright.sync_api import sync_playwright
77
- try:
78
- with sync_playwright() as p:
79
- browser = p.chromium.launch(headless=True)
80
- browser.close()
81
- print("OK")
82
- except Exception as e:
83
- print(f"ERROR: {e}")
84
- """
85
- result = subprocess.run([sys.executable, "-c", test_code],
86
- capture_output=True, text=True, timeout=30)
87
- return "OK" in result.stdout
88
- except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
89
- return False
90
-
91
- def fix_browser_issues(self, force_reinstall: bool = False) -> bool:
92
- """Fix common browser issues."""
93
- print("🔧 Fixing browser issues...")
94
-
95
- success = True
96
-
97
- # Step 1: Clear cache if it exists
98
- if self._clear_browser_cache():
99
- print("✅ Browser cache cleared")
100
- else:
101
- print("⚠️ Could not clear browser cache")
102
-
103
- # Step 2: Reinstall Playwright if needed or forced
104
- if force_reinstall or not self._check_playwright_installed():
105
- if self._reinstall_playwright():
106
- print("✅ Playwright reinstalled")
107
- else:
108
- print("❌ Failed to reinstall Playwright")
109
- success = False
110
-
111
- # Step 3: Install browsers
112
- if self._install_browsers():
113
- print("✅ Browsers installed")
114
- else:
115
- print("❌ Failed to install browsers")
116
- success = False
117
-
118
- # Step 4: Test installation
119
- if self._test_browser_launch():
120
- print("✅ Browser test passed")
121
- else:
122
- print("❌ Browser test failed")
123
- success = False
124
-
125
- return success
126
-
127
- def _clear_browser_cache(self) -> bool:
128
- """Clear Playwright browser cache."""
129
- try:
130
- if self.playwright_cache and self.playwright_cache.exists():
131
- shutil.rmtree(self.playwright_cache, ignore_errors=True)
132
- print(f"Cleared cache: {self.playwright_cache}")
133
- return True
134
- except Exception as e:
135
- print(f"Cache clear error: {e}")
136
- return False
137
-
138
- def _reinstall_playwright(self) -> bool:
139
- """Reinstall Playwright completely."""
140
- try:
141
- # Uninstall
142
- subprocess.run([sys.executable, "-m", "pip", "uninstall", "-y", "playwright"],
143
- check=False, capture_output=True)
144
-
145
- # Clear pip cache
146
- subprocess.run([sys.executable, "-m", "pip", "cache", "purge"],
147
- check=False, capture_output=True)
148
-
149
- # Reinstall
150
- subprocess.run([sys.executable, "-m", "pip", "install", "playwright"],
151
- check=True, capture_output=True)
152
-
153
- return True
154
- except subprocess.CalledProcessError:
155
- return False
156
-
157
- def _install_browsers(self) -> bool:
158
- """Install Playwright browsers."""
159
- try:
160
- # Install chromium (most stable)
161
- subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"],
162
- check=True, capture_output=True)
163
-
164
- # Try to install firefox too (optional)
165
- subprocess.run([sys.executable, "-m", "playwright", "install", "firefox"],
166
- check=False, capture_output=True)
167
-
168
- return True
169
- except subprocess.CalledProcessError:
170
- return False
171
-
172
- def get_browser_info(self) -> Dict[str, str]:
173
- """Get browser installation information."""
174
- info = {}
175
-
176
- # Playwright version
177
- try:
178
- # Try new way first
179
- result = subprocess.run([sys.executable, "-c", "import playwright; print(getattr(playwright, '__version__', 'Unknown'))"],
180
- capture_output=True, text=True)
181
- if "Unknown" in result.stdout:
182
- # Fallback to CLI version
183
- result = subprocess.run([sys.executable, "-m", "playwright", "--version"],
184
- capture_output=True, text=True)
185
- info['playwright_version'] = result.stdout.strip()
186
- except:
187
- info['playwright_version'] = "Not installed"
188
-
189
- # Cache directory
190
- info['cache_path'] = str(self.playwright_cache) if self.playwright_cache else "Unknown"
191
- info['cache_exists'] = str(self._check_cache_directory())
192
-
193
- # Browser test
194
- info['browser_test'] = "PASS" if self._test_browser_launch() else "FAIL"
195
-
196
- return info
197
-
198
-
199
- def fix_browsers(force_reinstall: bool = False) -> bool:
200
- """
201
- Quick browser fix function.
202
-
203
- Args:
204
- force_reinstall: Force complete reinstall
205
-
206
- Returns:
207
- True if successful
208
- """
209
- fixer = BrowserFixer()
210
- return fixer.fix_browser_issues(force_reinstall)
211
-
212
-
213
- def diagnose_browsers() -> Dict[str, bool]:
214
- """
215
- Quick browser diagnosis.
216
-
217
- Returns:
218
- Dictionary with diagnosis results
219
- """
220
- fixer = BrowserFixer()
221
- return fixer.diagnose_browser_issues()
222
-
223
-
224
- def get_browser_status() -> Dict[str, str]:
225
- """
226
- Get browser status information.
227
-
228
- Returns:
229
- Dictionary with browser info
230
- """
231
- fixer = BrowserFixer()
232
- return fixer.get_browser_info()
@@ -1,122 +0,0 @@
1
- """
2
- Core installer logic. Clean and simple.
3
- """
4
-
5
- import os
6
- import sys
7
- import subprocess
8
- import platform
9
- from pathlib import Path
10
- from typing import Dict, Any
11
-
12
- from .platform import apply_platform_fixes, check_system_requirements
13
- from .templates import TemplateEngine
14
- from .browser_fixes import fix_browsers, diagnose_browsers
15
-
16
-
17
- def install_parser(parser_path: str = ".") -> bool:
18
- """
19
- Install everything for a UnrealOn parser.
20
-
21
- Args:
22
- parser_path: Path to parser directory
23
-
24
- Returns:
25
- True if success, False if failed
26
- """
27
- parser_dir = Path(parser_path).resolve()
28
- parser_name = parser_dir.name # Auto-detect from directory name
29
-
30
- print(f"🚀 Installing {parser_name}")
31
- print("=" * 40)
32
-
33
- os.chdir(parser_dir)
34
-
35
- # 1. Apply platform fixes
36
- print("🔧 Applying platform fixes...")
37
- apply_platform_fixes()
38
- print("✅ Platform fixes applied")
39
-
40
- # 2. Check system
41
- print("🔍 Checking system...")
42
- requirements = check_system_requirements()
43
- failed = [k for k, v in requirements.items() if not v]
44
- if failed:
45
- print(f"❌ Missing: {', '.join(failed)}")
46
- return False
47
-
48
- print(f"✅ Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")
49
-
50
- # 3. Check if basic tools are available (optional)
51
- _check_tools_availability()
52
-
53
- # 4. Create batch files (Windows only)
54
- if platform.system() == "Windows":
55
- _create_batch_files(parser_name)
56
-
57
- print(f"\n✅ {parser_name} setup completed!")
58
- print("\n📋 Next steps:")
59
- print("1. Install dependencies: pip install -r requirements.txt")
60
- print("2. Install browsers: playwright install chromium")
61
- print("3. Run parser: python main.py 5 2")
62
- if platform.system() == "Windows":
63
- print(" Or use: START.bat (includes setup menu)")
64
-
65
- return True
66
-
67
-
68
- def _check_tools_availability():
69
- """Check if basic tools are available (non-blocking)."""
70
- print("🔍 Checking available tools...")
71
-
72
- # Check pip
73
- try:
74
- subprocess.run([sys.executable, "-m", "pip", "--version"],
75
- check=True, capture_output=True)
76
- print("✅ pip available")
77
- except:
78
- print("⚠️ pip not available")
79
-
80
- # Check if requirements.txt exists
81
- if Path("requirements.txt").exists():
82
- print("✅ requirements.txt found")
83
- print("💡 To install dependencies: pip install -r requirements.txt")
84
- else:
85
- print("⚠️ requirements.txt not found")
86
-
87
- # Check playwright
88
- try:
89
- subprocess.run([sys.executable, "-c", "import playwright"],
90
- check=True, capture_output=True)
91
- print("✅ playwright available")
92
- except:
93
- print("⚠️ playwright not available")
94
- print("💡 To install: pip install playwright && playwright install chromium")
95
-
96
-
97
- def _create_batch_files(parser_name: str):
98
- """Create Windows batch files using Jinja2 templates."""
99
- print("🪟 Creating Windows batch files...")
100
-
101
- # Simple default config - no file reading needed
102
- config = {
103
- 'browsers_needed': ['chromium'],
104
- 'supports_persistent': True # Assume all parsers support persistent mode
105
- }
106
-
107
- # Create template engine
108
- engine = TemplateEngine()
109
-
110
- # Generate batch files
111
- start_content = engine.render_start_bat(parser_name, config)
112
- Path("START.bat").write_text(start_content, encoding="utf-8")
113
-
114
- quick_content = engine.render_quick_run_bat(parser_name, config)
115
- Path("QUICK_RUN.bat").write_text(quick_content, encoding="utf-8")
116
-
117
- test_content = engine.render_test_bat(parser_name, config)
118
- Path("TEST.bat").write_text(test_content, encoding="utf-8")
119
-
120
- print("✅ Created START.bat, QUICK_RUN.bat, TEST.bat")
121
-
122
-
@@ -1,60 +0,0 @@
1
- """
2
- Jinja2 template engine for batch files.
3
- """
4
-
5
- from pathlib import Path
6
- from typing import Dict, Any
7
- import json
8
- from jinja2 import Environment, FileSystemLoader
9
-
10
-
11
- class TemplateEngine:
12
- """Jinja2-based template engine for Windows batch files."""
13
-
14
- def __init__(self):
15
- # Get batch templates directory
16
- templates_dir = Path(__file__).parent / "batch_templates"
17
-
18
- # Create Jinja2 environment
19
- self.env = Environment(
20
- loader=FileSystemLoader(str(templates_dir)),
21
- trim_blocks=True,
22
- lstrip_blocks=True
23
- )
24
-
25
- def render_start_bat(self, parser_name: str, config: Dict[str, Any]) -> str:
26
- """Render START.bat file."""
27
- template = self.env.get_template('start.bat.j2')
28
-
29
- context = {
30
- 'parser_name': parser_name,
31
- 'config': config,
32
- 'config_json': json.dumps(config, indent=None),
33
- 'browsers_list': ', '.join(config.get('browsers_needed', ['chromium'])),
34
- 'has_proxy': False, # Removed proxy support detection
35
- 'has_persistent': config.get('supports_persistent', True) # Default to True
36
- }
37
-
38
- return template.render(**context)
39
-
40
- def render_quick_run_bat(self, parser_name: str, config: Dict[str, Any]) -> str:
41
- """Render QUICK_RUN.bat file."""
42
- template = self.env.get_template('quick_run.bat.j2')
43
-
44
- context = {
45
- 'parser_name': parser_name,
46
- 'config': config
47
- }
48
-
49
- return template.render(**context)
50
-
51
- def render_test_bat(self, parser_name: str, config: Dict[str, Any]) -> str:
52
- """Render TEST.bat file."""
53
- template = self.env.get_template('test.bat.j2')
54
-
55
- context = {
56
- 'parser_name': parser_name,
57
- 'config': config
58
- }
59
-
60
- return template.render(**context)