unrealon 2.0.32__py3-none-any.whl → 2.0.33__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {unrealon-2.0.32.dist-info → unrealon-2.0.33.dist-info}/METADATA +1 -2
- {unrealon-2.0.32.dist-info → unrealon-2.0.33.dist-info}/RECORD +8 -14
- {unrealon-2.0.32.dist-info → unrealon-2.0.33.dist-info}/entry_points.txt +0 -1
- {unrealon-2.0.32.dist-info → unrealon-2.0.33.dist-info}/top_level.txt +0 -1
- unrealon_driver/installer/__init__.py +8 -0
- {unrealon_installer → unrealon_driver/installer}/platform.py +0 -17
- unrealon_installer/__init__.py +0 -12
- unrealon_installer/batch_templates/quick_run.bat.j2 +0 -24
- unrealon_installer/batch_templates/start.bat.j2 +0 -254
- unrealon_installer/batch_templates/test.bat.j2 +0 -56
- unrealon_installer/browser_fixes.py +0 -232
- unrealon_installer/core.py +0 -122
- unrealon_installer/templates.py +0 -60
- {unrealon-2.0.32.dist-info → unrealon-2.0.33.dist-info}/LICENSE +0 -0
- {unrealon-2.0.32.dist-info → unrealon-2.0.33.dist-info}/WHEEL +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: unrealon
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.33
|
|
4
4
|
Summary: Enterprise-grade web scraping platform with AI-powered automation and real-time orchestration capabilities
|
|
5
5
|
Author-email: UnrealOn Team <team@unrealon.com>
|
|
6
6
|
License: MIT
|
|
@@ -33,7 +33,6 @@ Requires-Dist: aiohttp>=3.8.0
|
|
|
33
33
|
Requires-Dist: httpx>=0.23.0
|
|
34
34
|
Requires-Dist: beautifulsoup4>=4.13.5
|
|
35
35
|
Requires-Dist: pydantic-yaml>=1.6.0
|
|
36
|
-
Requires-Dist: jinja2>=3.1.0
|
|
37
36
|
Requires-Dist: rich>=13.0.0
|
|
38
37
|
Requires-Dist: pyyaml>=6.0
|
|
39
38
|
Requires-Dist: python-socketio>=5.0
|
|
@@ -113,6 +113,8 @@ unrealon_driver/driver/monitoring/health.py,sha256=rOH0wChDBTnDbV7MstB21JsJJUMGM
|
|
|
113
113
|
unrealon_driver/driver/utilities/__init__.py,sha256=YCTR7CXu3wlynKNDJh4YsE1uhZ_QrtoQjLNArleMChM,232
|
|
114
114
|
unrealon_driver/driver/utilities/logging.py,sha256=2my2QnkAa6Hdw-TfO4oOQ94yGc-CjlRfnWsnbWIKb9o,1799
|
|
115
115
|
unrealon_driver/driver/utilities/serialization.py,sha256=wTCSVrEloykiGN4K1JXbk2aqNKm7W90aWXmzhcLyAZc,2123
|
|
116
|
+
unrealon_driver/installer/__init__.py,sha256=PraOjOg-cN1zOtuhPSTE5vCGPSMzWtEBYU8A05GWEf8,227
|
|
117
|
+
unrealon_driver/installer/platform.py,sha256=U_8FJZk0C8M0ujpfzcpOPWEoUrT6asTNEIhsN0n2bCg,5081
|
|
116
118
|
unrealon_driver/managers/__init__.py,sha256=zJJsOb6Oodg7l00v4ncKUytnyeaZM887pHY8-eSuWdU,981
|
|
117
119
|
unrealon_driver/managers/base.py,sha256=GkuXillg9uqqnx6RL682fmKgK-7JyqYlH6DFUgyN4F8,5445
|
|
118
120
|
unrealon_driver/managers/browser.py,sha256=bc6O2NyC4FV82mb9sat48_k8s1c3IGY4i90ddMVWRIo,5432
|
|
@@ -125,17 +127,9 @@ unrealon_driver/managers/threading.py,sha256=djw5cSC99dfBKmep3IJ_8IgxQceMXtNvCp5
|
|
|
125
127
|
unrealon_driver/managers/update.py,sha256=-hohVxGXpj5bZ6ZTQN6NH1RK9Pd6GVzCMtu3GS2SdcQ,3582
|
|
126
128
|
unrealon_driver/utils/__init__.py,sha256=2Sz3eats5q4O2fDmefDuJt8M_zkN6xrS-9xXntWZWFc,168
|
|
127
129
|
unrealon_driver/utils/time.py,sha256=Oxk1eicKeZl8ZWbf7gu1Ll716k6CpXmVj67FHSnPIsA,184
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
unrealon_installer/batch_templates/start.bat.j2,sha256=MquGi-0RiWxPUIppdiCqejrLkQCgOsm74UbM3WHY2CQ,6282
|
|
135
|
-
unrealon_installer/batch_templates/test.bat.j2,sha256=eq_Bj-zN6jUZWLwgzPtYDCr4YbgjseMSIfDPYT5ODFw,1348
|
|
136
|
-
unrealon-2.0.32.dist-info/LICENSE,sha256=eEH8mWZW49YMpl4Sh5MtKqkZ8aVTzKQXiNPEnvL14ns,1070
|
|
137
|
-
unrealon-2.0.32.dist-info/METADATA,sha256=PmGeoss_5G-RK5QEc2KQPnUDOkSyE-73mhA9ooILsNM,15718
|
|
138
|
-
unrealon-2.0.32.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
|
|
139
|
-
unrealon-2.0.32.dist-info/entry_points.txt,sha256=tBJgozewpyuXznEYIsLwfE1s16VRjy3Wizhuyh26zb4,153
|
|
140
|
-
unrealon-2.0.32.dist-info/top_level.txt,sha256=qN6Q72fe4_i8mTOhYcO3fhGa3g4dmBgvZOsqmK4j8D8,66
|
|
141
|
-
unrealon-2.0.32.dist-info/RECORD,,
|
|
130
|
+
unrealon-2.0.33.dist-info/LICENSE,sha256=eEH8mWZW49YMpl4Sh5MtKqkZ8aVTzKQXiNPEnvL14ns,1070
|
|
131
|
+
unrealon-2.0.33.dist-info/METADATA,sha256=8wOfd43TlGdZ-0sB4AaXtmCPP1mX1y7wsTN9lqPHKvw,15689
|
|
132
|
+
unrealon-2.0.33.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
|
|
133
|
+
unrealon-2.0.33.dist-info/entry_points.txt,sha256=k0qM-eotpajkKUq-almJmxj9afhXprZ6IkvQkSdcKhI,104
|
|
134
|
+
unrealon-2.0.33.dist-info/top_level.txt,sha256=Gu8IeIfIVfUxdi-h-F0nKMQxo15pjhHZ0aTadXTpRE8,47
|
|
135
|
+
unrealon-2.0.33.dist-info/RECORD,,
|
|
@@ -139,23 +139,6 @@ def _apply_linux_fixes():
|
|
|
139
139
|
return fixes
|
|
140
140
|
|
|
141
141
|
|
|
142
|
-
def get_platform_info():
|
|
143
|
-
"""Get platform information."""
|
|
144
|
-
return {
|
|
145
|
-
'platform': platform.system(),
|
|
146
|
-
'architecture': platform.architecture()[0],
|
|
147
|
-
'python_version': f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
def check_system_requirements():
|
|
152
|
-
"""Check if system meets requirements."""
|
|
153
|
-
return {
|
|
154
|
-
'python_version': sys.version_info >= (3, 9),
|
|
155
|
-
'platform_supported': platform.system() in ['Windows', 'Darwin', 'Linux']
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
|
|
159
142
|
def cleanup_asyncio_resources():
|
|
160
143
|
"""Force cleanup of all asyncio resources. Call this at program exit on Windows."""
|
|
161
144
|
if platform.system() != "Windows":
|
unrealon_installer/__init__.py
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
UnrealOn Universal Installer - Clean Edition
|
|
3
|
-
|
|
4
|
-
One function installs everything. Simple and reliable.
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
from .core import install_parser
|
|
8
|
-
from .browser_fixes import fix_browsers, diagnose_browsers, get_browser_status
|
|
9
|
-
from .platform import apply_platform_fixes, cleanup_asyncio_resources
|
|
10
|
-
|
|
11
|
-
__version__ = "2.0.0"
|
|
12
|
-
__all__ = ["install_parser", "fix_browsers", "diagnose_browsers", "get_browser_status", "apply_platform_fixes", "cleanup_asyncio_resources"]
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
@echo off
|
|
2
|
-
REM {{ parser_name }} - Quick Run
|
|
3
|
-
REM Fast execution without menus
|
|
4
|
-
|
|
5
|
-
title {{ parser_name }} Quick Run
|
|
6
|
-
|
|
7
|
-
echo.
|
|
8
|
-
echo {{ parser_name }} - Quick Run Mode
|
|
9
|
-
echo {{ "=" * (parser_name|length + 20) }}
|
|
10
|
-
echo.
|
|
11
|
-
|
|
12
|
-
REM Setup environment
|
|
13
|
-
set PYTHONIOENCODING=utf-8
|
|
14
|
-
set PLAYWRIGHT_BROWSERS_PATH=0
|
|
15
|
-
set PYTHONUNBUFFERED=1
|
|
16
|
-
|
|
17
|
-
REM Platform fixes are applied automatically by the parser itself
|
|
18
|
-
|
|
19
|
-
echo Starting quick test: 5 requests, 2 concurrent...
|
|
20
|
-
python main.py 5 2
|
|
21
|
-
|
|
22
|
-
echo.
|
|
23
|
-
echo Quick run completed!
|
|
24
|
-
pause
|
|
@@ -1,254 +0,0 @@
|
|
|
1
|
-
@echo off
|
|
2
|
-
REM {{ parser_name }} - Universal Windows Installer
|
|
3
|
-
REM Generated by UnrealOn Universal Installer v2.0.0
|
|
4
|
-
|
|
5
|
-
title {{ parser_name }} Parser
|
|
6
|
-
|
|
7
|
-
echo.
|
|
8
|
-
echo {{ parser_name }} Parser - Windows Edition
|
|
9
|
-
echo {{ "=" * (parser_name|length + 25) }}
|
|
10
|
-
echo UnrealOn Driver-based parser with universal installer
|
|
11
|
-
echo Browsers: {{ browsers_list }}
|
|
12
|
-
echo Proxy Support: {% if has_proxy %}Yes{% else %}No{% endif %}
|
|
13
|
-
echo.
|
|
14
|
-
|
|
15
|
-
REM Setup environment variables
|
|
16
|
-
set PYTHONIOENCODING=utf-8
|
|
17
|
-
set PLAYWRIGHT_BROWSERS_PATH=0
|
|
18
|
-
set PYTHONUNBUFFERED=1
|
|
19
|
-
set UNREALON_PARSER_NAME={{ parser_name }}
|
|
20
|
-
|
|
21
|
-
REM Platform fixes are applied automatically by the parser itself
|
|
22
|
-
|
|
23
|
-
:main_menu
|
|
24
|
-
echo Main Menu:
|
|
25
|
-
echo.
|
|
26
|
-
echo [1] RUN PARSER (Quick Start)
|
|
27
|
-
echo [2] SETUP SYSTEM (First Time Setup)
|
|
28
|
-
echo [3] MANAGE BROWSERS (Install/Fix Browsers)
|
|
29
|
-
echo [4] MANAGE DEPENDENCIES (Install/Update Packages)
|
|
30
|
-
echo [5] SYSTEM STATUS (Check Installation)
|
|
31
|
-
echo [6] TROUBLESHOOTING (Fix Issues)
|
|
32
|
-
echo [7] HELP and INFO
|
|
33
|
-
echo [8] EXIT
|
|
34
|
-
echo.
|
|
35
|
-
set /p choice="Enter your choice (1-8): "
|
|
36
|
-
|
|
37
|
-
if "%choice%"=="1" goto run_parser
|
|
38
|
-
if "%choice%"=="2" goto setup_system
|
|
39
|
-
if "%choice%"=="3" goto manage_browsers
|
|
40
|
-
if "%choice%"=="4" goto manage_dependencies
|
|
41
|
-
if "%choice%"=="5" goto system_status
|
|
42
|
-
if "%choice%"=="6" goto troubleshooting
|
|
43
|
-
if "%choice%"=="7" goto help_info
|
|
44
|
-
if "%choice%"=="8" goto exit_script
|
|
45
|
-
goto main_menu
|
|
46
|
-
|
|
47
|
-
:run_parser
|
|
48
|
-
echo.
|
|
49
|
-
echo PARSER EXECUTION
|
|
50
|
-
echo ================
|
|
51
|
-
|
|
52
|
-
REM Check if system is ready
|
|
53
|
-
if not exist "venv" (
|
|
54
|
-
echo ERROR: System not set up! Please run setup first.
|
|
55
|
-
set /p setup_choice="Run setup now? (Y/N): "
|
|
56
|
-
if /i "%setup_choice%"=="Y" goto setup_system
|
|
57
|
-
goto main_menu
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
REM Activate environment
|
|
61
|
-
call venv\Scripts\activate.bat
|
|
62
|
-
if errorlevel 1 (
|
|
63
|
-
echo ERROR: Failed to activate virtual environment
|
|
64
|
-
pause
|
|
65
|
-
goto main_menu
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
echo.
|
|
69
|
-
echo Choose run mode:
|
|
70
|
-
echo [1] DEFAULT RUN (parser default settings)
|
|
71
|
-
echo [2] QUICK TEST (minimal test run)
|
|
72
|
-
echo [3] CUSTOM MODE (specify your arguments)
|
|
73
|
-
echo.
|
|
74
|
-
set /p mode="Enter mode (1-3): "
|
|
75
|
-
|
|
76
|
-
if "%mode%"=="1" (
|
|
77
|
-
echo Starting default run...
|
|
78
|
-
call venv\Scripts\activate.bat && python main.py
|
|
79
|
-
)
|
|
80
|
-
if "%mode%"=="2" (
|
|
81
|
-
echo Starting quick test...
|
|
82
|
-
call venv\Scripts\activate.bat && python main.py --test
|
|
83
|
-
)
|
|
84
|
-
if "%mode%"=="3" (
|
|
85
|
-
set /p args="Enter arguments: "
|
|
86
|
-
echo Starting with custom arguments: %args%
|
|
87
|
-
call venv\Scripts\activate.bat && python main.py %args%
|
|
88
|
-
)
|
|
89
|
-
|
|
90
|
-
echo.
|
|
91
|
-
echo Parser execution completed
|
|
92
|
-
pause
|
|
93
|
-
goto main_menu
|
|
94
|
-
|
|
95
|
-
:setup_system
|
|
96
|
-
cls
|
|
97
|
-
echo.
|
|
98
|
-
echo System Setup
|
|
99
|
-
echo =============
|
|
100
|
-
echo.
|
|
101
|
-
echo Installing Python dependencies...
|
|
102
|
-
pip install -r requirements.txt
|
|
103
|
-
echo.
|
|
104
|
-
echo Installing browsers...
|
|
105
|
-
playwright install chromium
|
|
106
|
-
echo.
|
|
107
|
-
echo Setup complete!
|
|
108
|
-
echo.
|
|
109
|
-
pause
|
|
110
|
-
goto main_menu
|
|
111
|
-
|
|
112
|
-
:manage_browsers
|
|
113
|
-
cls
|
|
114
|
-
echo.
|
|
115
|
-
echo Browser Management
|
|
116
|
-
echo ==================
|
|
117
|
-
echo.
|
|
118
|
-
echo [1] Install all browsers
|
|
119
|
-
echo [2] Install Chromium only
|
|
120
|
-
echo [3] Install Firefox only
|
|
121
|
-
echo [4] Check browser status
|
|
122
|
-
echo [5] Back to main menu
|
|
123
|
-
echo.
|
|
124
|
-
set /p browser_choice="Enter choice (1-5): "
|
|
125
|
-
|
|
126
|
-
if "%browser_choice%"=="1" playwright install
|
|
127
|
-
if "%browser_choice%"=="2" playwright install chromium
|
|
128
|
-
if "%browser_choice%"=="3" playwright install firefox
|
|
129
|
-
if "%browser_choice%"=="4" (
|
|
130
|
-
echo Checking browser status...
|
|
131
|
-
playwright --version
|
|
132
|
-
python -c "from playwright.sync_api import sync_playwright; print('Browsers OK')" 2>nul || echo "Browser check failed"
|
|
133
|
-
)
|
|
134
|
-
if "%browser_choice%"=="5" goto main_menu
|
|
135
|
-
|
|
136
|
-
echo.
|
|
137
|
-
pause
|
|
138
|
-
goto main_menu
|
|
139
|
-
|
|
140
|
-
:manage_dependencies
|
|
141
|
-
cls
|
|
142
|
-
echo.
|
|
143
|
-
echo Dependency Management
|
|
144
|
-
echo =====================
|
|
145
|
-
echo.
|
|
146
|
-
echo [1] Install/Update all dependencies
|
|
147
|
-
echo [2] Install production dependencies only
|
|
148
|
-
echo [3] Check dependency status
|
|
149
|
-
echo [4] Back to main menu
|
|
150
|
-
echo.
|
|
151
|
-
set /p dep_choice="Enter choice (1-4): "
|
|
152
|
-
|
|
153
|
-
if "%dep_choice%"=="1" (
|
|
154
|
-
pip install -r requirements.txt
|
|
155
|
-
pip install --upgrade unrealon
|
|
156
|
-
)
|
|
157
|
-
if "%dep_choice%"=="2" pip install -r requirements.txt
|
|
158
|
-
if "%dep_choice%"=="3" (
|
|
159
|
-
echo Checking dependencies...
|
|
160
|
-
pip list | findstr unrealon
|
|
161
|
-
python -c "import unrealon_driver; print('UnrealOn Driver:', unrealon_driver.__version__)" 2>nul || echo "UnrealOn Driver not found"
|
|
162
|
-
)
|
|
163
|
-
if "%dep_choice%"=="4" goto main_menu
|
|
164
|
-
|
|
165
|
-
echo.
|
|
166
|
-
pause
|
|
167
|
-
goto main_menu
|
|
168
|
-
|
|
169
|
-
:system_status
|
|
170
|
-
cls
|
|
171
|
-
echo.
|
|
172
|
-
echo System Status Check
|
|
173
|
-
echo ===================
|
|
174
|
-
echo.
|
|
175
|
-
|
|
176
|
-
echo Python Version:
|
|
177
|
-
python --version
|
|
178
|
-
|
|
179
|
-
echo.
|
|
180
|
-
echo Platform Information:
|
|
181
|
-
python -c "from unrealon_installer.platform import get_platform_info; import json; print(json.dumps(get_platform_info(), indent=2))" 2>nul || echo "Platform info not available"
|
|
182
|
-
|
|
183
|
-
echo.
|
|
184
|
-
echo UnrealOn Driver Status:
|
|
185
|
-
python -c "import unrealon_driver; print('Version:', unrealon_driver.__version__)" 2>nul || echo "UnrealOn Driver not installed"
|
|
186
|
-
|
|
187
|
-
echo.
|
|
188
|
-
echo Browser Status:
|
|
189
|
-
playwright --version 2>nul || echo "Playwright not installed"
|
|
190
|
-
|
|
191
|
-
echo.
|
|
192
|
-
echo Dependencies:
|
|
193
|
-
pip list | findstr -i "unrealon playwright requests"
|
|
194
|
-
|
|
195
|
-
echo.
|
|
196
|
-
pause
|
|
197
|
-
goto main_menu
|
|
198
|
-
|
|
199
|
-
:troubleshooting
|
|
200
|
-
cls
|
|
201
|
-
echo.
|
|
202
|
-
echo Troubleshooting Guide
|
|
203
|
-
echo =====================
|
|
204
|
-
echo.
|
|
205
|
-
echo Common issues and solutions:
|
|
206
|
-
echo.
|
|
207
|
-
echo 1. "ModuleNotFoundError: No module named 'unrealon_driver'"
|
|
208
|
-
echo Solution: Run "pip install -r requirements.txt"
|
|
209
|
-
echo.
|
|
210
|
-
echo 2. "Browser not found" errors
|
|
211
|
-
echo Solution: Run "playwright install chromium"
|
|
212
|
-
echo.
|
|
213
|
-
echo 3. "Permission denied" errors
|
|
214
|
-
echo Solution: Run as Administrator
|
|
215
|
-
echo.
|
|
216
|
-
echo 4. Proxy connection issues
|
|
217
|
-
echo Solution: Check proxy configuration in proxy_config.py
|
|
218
|
-
echo.
|
|
219
|
-
echo 5. SSL certificate errors
|
|
220
|
-
echo Solution: Platform fixes are applied automatically
|
|
221
|
-
echo.
|
|
222
|
-
echo Press any key to return to main menu...
|
|
223
|
-
pause >nul
|
|
224
|
-
goto main_menu
|
|
225
|
-
|
|
226
|
-
:help_info
|
|
227
|
-
cls
|
|
228
|
-
echo.
|
|
229
|
-
echo {{ parser_name }} Parser - Help and Information
|
|
230
|
-
echo {{ "=" * (parser_name|length + 25) }}
|
|
231
|
-
echo.
|
|
232
|
-
echo This is a UnrealOn Driver-based parser with the following features:
|
|
233
|
-
echo.
|
|
234
|
-
echo - Multi-concurrent API processing
|
|
235
|
-
echo - Browser automation with Playwright
|
|
236
|
-
echo - Proxy support (if configured)
|
|
237
|
-
echo - Cross-platform compatibility
|
|
238
|
-
{% if has_persistent %}echo - Persistent monitoring mode{% endif %}
|
|
239
|
-
echo - Automatic dependency management
|
|
240
|
-
echo.
|
|
241
|
-
echo Configuration:
|
|
242
|
-
echo {{ config_json }}
|
|
243
|
-
echo.
|
|
244
|
-
echo For more information, visit: https://github.com/unrealon
|
|
245
|
-
echo.
|
|
246
|
-
echo Press any key to return to main menu...
|
|
247
|
-
pause >nul
|
|
248
|
-
goto main_menu
|
|
249
|
-
|
|
250
|
-
:exit_script
|
|
251
|
-
echo.
|
|
252
|
-
echo Thank you for using {{ parser_name }} Parser!
|
|
253
|
-
echo.
|
|
254
|
-
exit /b 0
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
@echo off
|
|
2
|
-
REM {{ parser_name }} - System Test
|
|
3
|
-
REM Comprehensive system and functionality test
|
|
4
|
-
|
|
5
|
-
title {{ parser_name }} System Test
|
|
6
|
-
|
|
7
|
-
echo.
|
|
8
|
-
echo {{ parser_name }} - System Test
|
|
9
|
-
echo {{ "=" * (parser_name|length + 15) }}
|
|
10
|
-
echo.
|
|
11
|
-
|
|
12
|
-
REM Setup environment
|
|
13
|
-
set PYTHONIOENCODING=utf-8
|
|
14
|
-
set PLAYWRIGHT_BROWSERS_PATH=0
|
|
15
|
-
set PYTHONUNBUFFERED=1
|
|
16
|
-
|
|
17
|
-
echo [1/5] Testing Python environment...
|
|
18
|
-
python --version || (
|
|
19
|
-
echo ERROR: Python not found!
|
|
20
|
-
pause
|
|
21
|
-
exit /b 1
|
|
22
|
-
)
|
|
23
|
-
|
|
24
|
-
echo [2/5] Testing UnrealOn Driver...
|
|
25
|
-
python -c "import unrealon_driver; print('UnrealOn Driver version:', unrealon_driver.__version__)" || (
|
|
26
|
-
echo ERROR: UnrealOn Driver not installed!
|
|
27
|
-
echo Run: pip install -r requirements.txt
|
|
28
|
-
pause
|
|
29
|
-
exit /b 1
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
echo [3/5] Testing platform compatibility...
|
|
33
|
-
python -c "from unrealon_installer.platform import get_platform_info; print('Platform OK')" || (
|
|
34
|
-
echo ERROR: Platform compatibility issues!
|
|
35
|
-
pause
|
|
36
|
-
exit /b 1
|
|
37
|
-
)
|
|
38
|
-
|
|
39
|
-
echo [4/5] Testing browser availability...
|
|
40
|
-
playwright --version || (
|
|
41
|
-
echo ERROR: Playwright not installed!
|
|
42
|
-
echo Run: playwright install chromium
|
|
43
|
-
pause
|
|
44
|
-
exit /b 1
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
echo [5/5] Running parser test...
|
|
48
|
-
python main.py 1 1
|
|
49
|
-
|
|
50
|
-
echo.
|
|
51
|
-
echo {{ "=" * (parser_name|length + 15) }}
|
|
52
|
-
echo All tests completed successfully!
|
|
53
|
-
echo {{ parser_name }} is ready to use.
|
|
54
|
-
echo {{ "=" * (parser_name|length + 15) }}
|
|
55
|
-
echo.
|
|
56
|
-
pause
|
|
@@ -1,232 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Browser troubleshooting and fixes for Windows.
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
import os
|
|
6
|
-
import sys
|
|
7
|
-
import subprocess
|
|
8
|
-
import platform
|
|
9
|
-
from pathlib import Path
|
|
10
|
-
from typing import Dict, List, Optional
|
|
11
|
-
import shutil
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class BrowserFixer:
|
|
15
|
-
"""Windows browser troubleshooting and fixes."""
|
|
16
|
-
|
|
17
|
-
def __init__(self):
|
|
18
|
-
self.is_windows = platform.system() == "Windows"
|
|
19
|
-
self.playwright_cache = self._get_playwright_cache_path()
|
|
20
|
-
|
|
21
|
-
def _get_playwright_cache_path(self) -> Optional[Path]:
|
|
22
|
-
"""Get Playwright cache directory path."""
|
|
23
|
-
if self.is_windows:
|
|
24
|
-
return Path(os.environ.get("USERPROFILE", "")) / "AppData" / "Local" / "ms-playwright"
|
|
25
|
-
elif platform.system() == "Darwin": # macOS
|
|
26
|
-
return Path.home() / "Library" / "Caches" / "ms-playwright"
|
|
27
|
-
else: # Linux
|
|
28
|
-
return Path.home() / ".cache" / "ms-playwright"
|
|
29
|
-
|
|
30
|
-
def diagnose_browser_issues(self) -> Dict[str, bool]:
|
|
31
|
-
"""Diagnose common browser issues."""
|
|
32
|
-
issues = {}
|
|
33
|
-
|
|
34
|
-
# Check if Playwright is installed
|
|
35
|
-
issues['playwright_installed'] = self._check_playwright_installed()
|
|
36
|
-
|
|
37
|
-
# Check if browsers are installed
|
|
38
|
-
issues['browsers_installed'] = self._check_browsers_installed()
|
|
39
|
-
|
|
40
|
-
# Check cache directory
|
|
41
|
-
issues['cache_exists'] = self._check_cache_directory()
|
|
42
|
-
|
|
43
|
-
# Check if browsers can launch
|
|
44
|
-
issues['browser_launch_test'] = self._test_browser_launch()
|
|
45
|
-
|
|
46
|
-
return issues
|
|
47
|
-
|
|
48
|
-
def _check_playwright_installed(self) -> bool:
|
|
49
|
-
"""Check if Playwright is installed."""
|
|
50
|
-
try:
|
|
51
|
-
subprocess.run([sys.executable, "-c", "import playwright"],
|
|
52
|
-
check=True, capture_output=True)
|
|
53
|
-
return True
|
|
54
|
-
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
55
|
-
return False
|
|
56
|
-
|
|
57
|
-
def _check_browsers_installed(self) -> bool:
|
|
58
|
-
"""Check if Playwright browsers are installed."""
|
|
59
|
-
try:
|
|
60
|
-
result = subprocess.run([sys.executable, "-m", "playwright", "--version"],
|
|
61
|
-
check=True, capture_output=True, text=True)
|
|
62
|
-
return "version" in result.stdout.lower()
|
|
63
|
-
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
64
|
-
return False
|
|
65
|
-
|
|
66
|
-
def _check_cache_directory(self) -> bool:
|
|
67
|
-
"""Check if Playwright cache directory exists."""
|
|
68
|
-
if not self.playwright_cache:
|
|
69
|
-
return False
|
|
70
|
-
return self.playwright_cache.exists()
|
|
71
|
-
|
|
72
|
-
def _test_browser_launch(self) -> bool:
|
|
73
|
-
"""Test if browser can launch (quick test)."""
|
|
74
|
-
try:
|
|
75
|
-
test_code = """
|
|
76
|
-
from playwright.sync_api import sync_playwright
|
|
77
|
-
try:
|
|
78
|
-
with sync_playwright() as p:
|
|
79
|
-
browser = p.chromium.launch(headless=True)
|
|
80
|
-
browser.close()
|
|
81
|
-
print("OK")
|
|
82
|
-
except Exception as e:
|
|
83
|
-
print(f"ERROR: {e}")
|
|
84
|
-
"""
|
|
85
|
-
result = subprocess.run([sys.executable, "-c", test_code],
|
|
86
|
-
capture_output=True, text=True, timeout=30)
|
|
87
|
-
return "OK" in result.stdout
|
|
88
|
-
except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
|
|
89
|
-
return False
|
|
90
|
-
|
|
91
|
-
def fix_browser_issues(self, force_reinstall: bool = False) -> bool:
|
|
92
|
-
"""Fix common browser issues."""
|
|
93
|
-
print("🔧 Fixing browser issues...")
|
|
94
|
-
|
|
95
|
-
success = True
|
|
96
|
-
|
|
97
|
-
# Step 1: Clear cache if it exists
|
|
98
|
-
if self._clear_browser_cache():
|
|
99
|
-
print("✅ Browser cache cleared")
|
|
100
|
-
else:
|
|
101
|
-
print("⚠️ Could not clear browser cache")
|
|
102
|
-
|
|
103
|
-
# Step 2: Reinstall Playwright if needed or forced
|
|
104
|
-
if force_reinstall or not self._check_playwright_installed():
|
|
105
|
-
if self._reinstall_playwright():
|
|
106
|
-
print("✅ Playwright reinstalled")
|
|
107
|
-
else:
|
|
108
|
-
print("❌ Failed to reinstall Playwright")
|
|
109
|
-
success = False
|
|
110
|
-
|
|
111
|
-
# Step 3: Install browsers
|
|
112
|
-
if self._install_browsers():
|
|
113
|
-
print("✅ Browsers installed")
|
|
114
|
-
else:
|
|
115
|
-
print("❌ Failed to install browsers")
|
|
116
|
-
success = False
|
|
117
|
-
|
|
118
|
-
# Step 4: Test installation
|
|
119
|
-
if self._test_browser_launch():
|
|
120
|
-
print("✅ Browser test passed")
|
|
121
|
-
else:
|
|
122
|
-
print("❌ Browser test failed")
|
|
123
|
-
success = False
|
|
124
|
-
|
|
125
|
-
return success
|
|
126
|
-
|
|
127
|
-
def _clear_browser_cache(self) -> bool:
|
|
128
|
-
"""Clear Playwright browser cache."""
|
|
129
|
-
try:
|
|
130
|
-
if self.playwright_cache and self.playwright_cache.exists():
|
|
131
|
-
shutil.rmtree(self.playwright_cache, ignore_errors=True)
|
|
132
|
-
print(f"Cleared cache: {self.playwright_cache}")
|
|
133
|
-
return True
|
|
134
|
-
except Exception as e:
|
|
135
|
-
print(f"Cache clear error: {e}")
|
|
136
|
-
return False
|
|
137
|
-
|
|
138
|
-
def _reinstall_playwright(self) -> bool:
|
|
139
|
-
"""Reinstall Playwright completely."""
|
|
140
|
-
try:
|
|
141
|
-
# Uninstall
|
|
142
|
-
subprocess.run([sys.executable, "-m", "pip", "uninstall", "-y", "playwright"],
|
|
143
|
-
check=False, capture_output=True)
|
|
144
|
-
|
|
145
|
-
# Clear pip cache
|
|
146
|
-
subprocess.run([sys.executable, "-m", "pip", "cache", "purge"],
|
|
147
|
-
check=False, capture_output=True)
|
|
148
|
-
|
|
149
|
-
# Reinstall
|
|
150
|
-
subprocess.run([sys.executable, "-m", "pip", "install", "playwright"],
|
|
151
|
-
check=True, capture_output=True)
|
|
152
|
-
|
|
153
|
-
return True
|
|
154
|
-
except subprocess.CalledProcessError:
|
|
155
|
-
return False
|
|
156
|
-
|
|
157
|
-
def _install_browsers(self) -> bool:
|
|
158
|
-
"""Install Playwright browsers."""
|
|
159
|
-
try:
|
|
160
|
-
# Install chromium (most stable)
|
|
161
|
-
subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"],
|
|
162
|
-
check=True, capture_output=True)
|
|
163
|
-
|
|
164
|
-
# Try to install firefox too (optional)
|
|
165
|
-
subprocess.run([sys.executable, "-m", "playwright", "install", "firefox"],
|
|
166
|
-
check=False, capture_output=True)
|
|
167
|
-
|
|
168
|
-
return True
|
|
169
|
-
except subprocess.CalledProcessError:
|
|
170
|
-
return False
|
|
171
|
-
|
|
172
|
-
def get_browser_info(self) -> Dict[str, str]:
|
|
173
|
-
"""Get browser installation information."""
|
|
174
|
-
info = {}
|
|
175
|
-
|
|
176
|
-
# Playwright version
|
|
177
|
-
try:
|
|
178
|
-
# Try new way first
|
|
179
|
-
result = subprocess.run([sys.executable, "-c", "import playwright; print(getattr(playwright, '__version__', 'Unknown'))"],
|
|
180
|
-
capture_output=True, text=True)
|
|
181
|
-
if "Unknown" in result.stdout:
|
|
182
|
-
# Fallback to CLI version
|
|
183
|
-
result = subprocess.run([sys.executable, "-m", "playwright", "--version"],
|
|
184
|
-
capture_output=True, text=True)
|
|
185
|
-
info['playwright_version'] = result.stdout.strip()
|
|
186
|
-
except:
|
|
187
|
-
info['playwright_version'] = "Not installed"
|
|
188
|
-
|
|
189
|
-
# Cache directory
|
|
190
|
-
info['cache_path'] = str(self.playwright_cache) if self.playwright_cache else "Unknown"
|
|
191
|
-
info['cache_exists'] = str(self._check_cache_directory())
|
|
192
|
-
|
|
193
|
-
# Browser test
|
|
194
|
-
info['browser_test'] = "PASS" if self._test_browser_launch() else "FAIL"
|
|
195
|
-
|
|
196
|
-
return info
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
def fix_browsers(force_reinstall: bool = False) -> bool:
|
|
200
|
-
"""
|
|
201
|
-
Quick browser fix function.
|
|
202
|
-
|
|
203
|
-
Args:
|
|
204
|
-
force_reinstall: Force complete reinstall
|
|
205
|
-
|
|
206
|
-
Returns:
|
|
207
|
-
True if successful
|
|
208
|
-
"""
|
|
209
|
-
fixer = BrowserFixer()
|
|
210
|
-
return fixer.fix_browser_issues(force_reinstall)
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
def diagnose_browsers() -> Dict[str, bool]:
|
|
214
|
-
"""
|
|
215
|
-
Quick browser diagnosis.
|
|
216
|
-
|
|
217
|
-
Returns:
|
|
218
|
-
Dictionary with diagnosis results
|
|
219
|
-
"""
|
|
220
|
-
fixer = BrowserFixer()
|
|
221
|
-
return fixer.diagnose_browser_issues()
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
def get_browser_status() -> Dict[str, str]:
|
|
225
|
-
"""
|
|
226
|
-
Get browser status information.
|
|
227
|
-
|
|
228
|
-
Returns:
|
|
229
|
-
Dictionary with browser info
|
|
230
|
-
"""
|
|
231
|
-
fixer = BrowserFixer()
|
|
232
|
-
return fixer.get_browser_info()
|
unrealon_installer/core.py
DELETED
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Core installer logic. Clean and simple.
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
import os
|
|
6
|
-
import sys
|
|
7
|
-
import subprocess
|
|
8
|
-
import platform
|
|
9
|
-
from pathlib import Path
|
|
10
|
-
from typing import Dict, Any
|
|
11
|
-
|
|
12
|
-
from .platform import apply_platform_fixes, check_system_requirements
|
|
13
|
-
from .templates import TemplateEngine
|
|
14
|
-
from .browser_fixes import fix_browsers, diagnose_browsers
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def install_parser(parser_path: str = ".") -> bool:
|
|
18
|
-
"""
|
|
19
|
-
Install everything for a UnrealOn parser.
|
|
20
|
-
|
|
21
|
-
Args:
|
|
22
|
-
parser_path: Path to parser directory
|
|
23
|
-
|
|
24
|
-
Returns:
|
|
25
|
-
True if success, False if failed
|
|
26
|
-
"""
|
|
27
|
-
parser_dir = Path(parser_path).resolve()
|
|
28
|
-
parser_name = parser_dir.name # Auto-detect from directory name
|
|
29
|
-
|
|
30
|
-
print(f"🚀 Installing {parser_name}")
|
|
31
|
-
print("=" * 40)
|
|
32
|
-
|
|
33
|
-
os.chdir(parser_dir)
|
|
34
|
-
|
|
35
|
-
# 1. Apply platform fixes
|
|
36
|
-
print("🔧 Applying platform fixes...")
|
|
37
|
-
apply_platform_fixes()
|
|
38
|
-
print("✅ Platform fixes applied")
|
|
39
|
-
|
|
40
|
-
# 2. Check system
|
|
41
|
-
print("🔍 Checking system...")
|
|
42
|
-
requirements = check_system_requirements()
|
|
43
|
-
failed = [k for k, v in requirements.items() if not v]
|
|
44
|
-
if failed:
|
|
45
|
-
print(f"❌ Missing: {', '.join(failed)}")
|
|
46
|
-
return False
|
|
47
|
-
|
|
48
|
-
print(f"✅ Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")
|
|
49
|
-
|
|
50
|
-
# 3. Check if basic tools are available (optional)
|
|
51
|
-
_check_tools_availability()
|
|
52
|
-
|
|
53
|
-
# 4. Create batch files (Windows only)
|
|
54
|
-
if platform.system() == "Windows":
|
|
55
|
-
_create_batch_files(parser_name)
|
|
56
|
-
|
|
57
|
-
print(f"\n✅ {parser_name} setup completed!")
|
|
58
|
-
print("\n📋 Next steps:")
|
|
59
|
-
print("1. Install dependencies: pip install -r requirements.txt")
|
|
60
|
-
print("2. Install browsers: playwright install chromium")
|
|
61
|
-
print("3. Run parser: python main.py 5 2")
|
|
62
|
-
if platform.system() == "Windows":
|
|
63
|
-
print(" Or use: START.bat (includes setup menu)")
|
|
64
|
-
|
|
65
|
-
return True
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def _check_tools_availability():
|
|
69
|
-
"""Check if basic tools are available (non-blocking)."""
|
|
70
|
-
print("🔍 Checking available tools...")
|
|
71
|
-
|
|
72
|
-
# Check pip
|
|
73
|
-
try:
|
|
74
|
-
subprocess.run([sys.executable, "-m", "pip", "--version"],
|
|
75
|
-
check=True, capture_output=True)
|
|
76
|
-
print("✅ pip available")
|
|
77
|
-
except:
|
|
78
|
-
print("⚠️ pip not available")
|
|
79
|
-
|
|
80
|
-
# Check if requirements.txt exists
|
|
81
|
-
if Path("requirements.txt").exists():
|
|
82
|
-
print("✅ requirements.txt found")
|
|
83
|
-
print("💡 To install dependencies: pip install -r requirements.txt")
|
|
84
|
-
else:
|
|
85
|
-
print("⚠️ requirements.txt not found")
|
|
86
|
-
|
|
87
|
-
# Check playwright
|
|
88
|
-
try:
|
|
89
|
-
subprocess.run([sys.executable, "-c", "import playwright"],
|
|
90
|
-
check=True, capture_output=True)
|
|
91
|
-
print("✅ playwright available")
|
|
92
|
-
except:
|
|
93
|
-
print("⚠️ playwright not available")
|
|
94
|
-
print("💡 To install: pip install playwright && playwright install chromium")
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
def _create_batch_files(parser_name: str):
|
|
98
|
-
"""Create Windows batch files using Jinja2 templates."""
|
|
99
|
-
print("🪟 Creating Windows batch files...")
|
|
100
|
-
|
|
101
|
-
# Simple default config - no file reading needed
|
|
102
|
-
config = {
|
|
103
|
-
'browsers_needed': ['chromium'],
|
|
104
|
-
'supports_persistent': True # Assume all parsers support persistent mode
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
# Create template engine
|
|
108
|
-
engine = TemplateEngine()
|
|
109
|
-
|
|
110
|
-
# Generate batch files
|
|
111
|
-
start_content = engine.render_start_bat(parser_name, config)
|
|
112
|
-
Path("START.bat").write_text(start_content, encoding="utf-8")
|
|
113
|
-
|
|
114
|
-
quick_content = engine.render_quick_run_bat(parser_name, config)
|
|
115
|
-
Path("QUICK_RUN.bat").write_text(quick_content, encoding="utf-8")
|
|
116
|
-
|
|
117
|
-
test_content = engine.render_test_bat(parser_name, config)
|
|
118
|
-
Path("TEST.bat").write_text(test_content, encoding="utf-8")
|
|
119
|
-
|
|
120
|
-
print("✅ Created START.bat, QUICK_RUN.bat, TEST.bat")
|
|
121
|
-
|
|
122
|
-
|
unrealon_installer/templates.py
DELETED
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Jinja2 template engine for batch files.
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from typing import Dict, Any
|
|
7
|
-
import json
|
|
8
|
-
from jinja2 import Environment, FileSystemLoader
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class TemplateEngine:
|
|
12
|
-
"""Jinja2-based template engine for Windows batch files."""
|
|
13
|
-
|
|
14
|
-
def __init__(self):
|
|
15
|
-
# Get batch templates directory
|
|
16
|
-
templates_dir = Path(__file__).parent / "batch_templates"
|
|
17
|
-
|
|
18
|
-
# Create Jinja2 environment
|
|
19
|
-
self.env = Environment(
|
|
20
|
-
loader=FileSystemLoader(str(templates_dir)),
|
|
21
|
-
trim_blocks=True,
|
|
22
|
-
lstrip_blocks=True
|
|
23
|
-
)
|
|
24
|
-
|
|
25
|
-
def render_start_bat(self, parser_name: str, config: Dict[str, Any]) -> str:
|
|
26
|
-
"""Render START.bat file."""
|
|
27
|
-
template = self.env.get_template('start.bat.j2')
|
|
28
|
-
|
|
29
|
-
context = {
|
|
30
|
-
'parser_name': parser_name,
|
|
31
|
-
'config': config,
|
|
32
|
-
'config_json': json.dumps(config, indent=None),
|
|
33
|
-
'browsers_list': ', '.join(config.get('browsers_needed', ['chromium'])),
|
|
34
|
-
'has_proxy': False, # Removed proxy support detection
|
|
35
|
-
'has_persistent': config.get('supports_persistent', True) # Default to True
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
return template.render(**context)
|
|
39
|
-
|
|
40
|
-
def render_quick_run_bat(self, parser_name: str, config: Dict[str, Any]) -> str:
|
|
41
|
-
"""Render QUICK_RUN.bat file."""
|
|
42
|
-
template = self.env.get_template('quick_run.bat.j2')
|
|
43
|
-
|
|
44
|
-
context = {
|
|
45
|
-
'parser_name': parser_name,
|
|
46
|
-
'config': config
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
return template.render(**context)
|
|
50
|
-
|
|
51
|
-
def render_test_bat(self, parser_name: str, config: Dict[str, Any]) -> str:
|
|
52
|
-
"""Render TEST.bat file."""
|
|
53
|
-
template = self.env.get_template('test.bat.j2')
|
|
54
|
-
|
|
55
|
-
context = {
|
|
56
|
-
'parser_name': parser_name,
|
|
57
|
-
'config': config
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
return template.render(**context)
|
|
File without changes
|
|
File without changes
|