unrealon 1.1.6__py3-none-any.whl → 2.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. {unrealon-1.1.6.dist-info/licenses → unrealon-2.0.4.dist-info}/LICENSE +1 -1
  2. unrealon-2.0.4.dist-info/METADATA +491 -0
  3. unrealon-2.0.4.dist-info/RECORD +129 -0
  4. {unrealon-1.1.6.dist-info → unrealon-2.0.4.dist-info}/WHEEL +2 -1
  5. unrealon-2.0.4.dist-info/entry_points.txt +3 -0
  6. unrealon-2.0.4.dist-info/top_level.txt +3 -0
  7. unrealon_browser/__init__.py +5 -6
  8. unrealon_browser/cli/browser_cli.py +18 -9
  9. unrealon_browser/cli/interactive_mode.py +13 -4
  10. unrealon_browser/core/browser_manager.py +29 -16
  11. unrealon_browser/dto/__init__.py +21 -0
  12. unrealon_browser/dto/bot_detection.py +175 -0
  13. unrealon_browser/dto/models/config.py +9 -3
  14. unrealon_browser/managers/__init__.py +1 -1
  15. unrealon_browser/managers/logger_bridge.py +1 -4
  16. unrealon_browser/stealth/__init__.py +27 -0
  17. unrealon_browser/stealth/bypass_techniques.pyc +0 -0
  18. unrealon_browser/stealth/manager.pyc +0 -0
  19. unrealon_browser/stealth/nodriver_stealth.pyc +0 -0
  20. unrealon_browser/stealth/playwright_stealth.pyc +0 -0
  21. unrealon_browser/stealth/scanner_tester.pyc +0 -0
  22. unrealon_browser/stealth/undetected_chrome.pyc +0 -0
  23. unrealon_core/__init__.py +160 -0
  24. unrealon_core/config/__init__.py +16 -0
  25. unrealon_core/config/environment.py +98 -0
  26. unrealon_core/config/urls.py +93 -0
  27. unrealon_core/enums/__init__.py +24 -0
  28. unrealon_core/enums/status.py +216 -0
  29. unrealon_core/enums/types.py +240 -0
  30. unrealon_core/error_handling/__init__.py +45 -0
  31. unrealon_core/error_handling/circuit_breaker.py +292 -0
  32. unrealon_core/error_handling/error_context.py +324 -0
  33. unrealon_core/error_handling/recovery.py +371 -0
  34. unrealon_core/error_handling/retry.py +268 -0
  35. unrealon_core/exceptions/__init__.py +46 -0
  36. unrealon_core/exceptions/base.py +292 -0
  37. unrealon_core/exceptions/communication.py +22 -0
  38. unrealon_core/exceptions/driver.py +11 -0
  39. unrealon_core/exceptions/proxy.py +11 -0
  40. unrealon_core/exceptions/task.py +12 -0
  41. unrealon_core/exceptions/validation.py +17 -0
  42. unrealon_core/models/__init__.py +98 -0
  43. unrealon_core/models/arq_context.py +252 -0
  44. unrealon_core/models/arq_responses.py +125 -0
  45. unrealon_core/models/base.py +291 -0
  46. unrealon_core/models/bridge_stats.py +58 -0
  47. unrealon_core/models/communication.py +39 -0
  48. unrealon_core/models/config.py +47 -0
  49. unrealon_core/models/connection_stats.py +47 -0
  50. unrealon_core/models/driver.py +30 -0
  51. unrealon_core/models/driver_details.py +98 -0
  52. unrealon_core/models/logging.py +28 -0
  53. unrealon_core/models/task.py +21 -0
  54. unrealon_core/models/typed_responses.py +210 -0
  55. unrealon_core/models/websocket/__init__.py +91 -0
  56. unrealon_core/models/websocket/base.py +49 -0
  57. unrealon_core/models/websocket/config.py +200 -0
  58. unrealon_core/models/websocket/driver.py +215 -0
  59. unrealon_core/models/websocket/errors.py +138 -0
  60. unrealon_core/models/websocket/heartbeat.py +100 -0
  61. unrealon_core/models/websocket/logging.py +261 -0
  62. unrealon_core/models/websocket/proxy.py +496 -0
  63. unrealon_core/models/websocket/tasks.py +275 -0
  64. unrealon_core/models/websocket/utils.py +153 -0
  65. unrealon_core/models/websocket_session.py +144 -0
  66. unrealon_core/monitoring/__init__.py +43 -0
  67. unrealon_core/monitoring/alerts.py +398 -0
  68. unrealon_core/monitoring/dashboard.py +307 -0
  69. unrealon_core/monitoring/health_check.py +354 -0
  70. unrealon_core/monitoring/metrics.py +352 -0
  71. unrealon_core/utils/__init__.py +11 -0
  72. unrealon_core/utils/time.py +61 -0
  73. unrealon_core/version.py +219 -0
  74. unrealon_driver/__init__.py +90 -51
  75. unrealon_driver/core_module/__init__.py +34 -0
  76. unrealon_driver/core_module/base.py +184 -0
  77. unrealon_driver/core_module/config.py +30 -0
  78. unrealon_driver/core_module/event_manager.py +127 -0
  79. unrealon_driver/core_module/protocols.py +98 -0
  80. unrealon_driver/core_module/registry.py +146 -0
  81. unrealon_driver/decorators/__init__.py +15 -0
  82. unrealon_driver/decorators/retry.py +117 -0
  83. unrealon_driver/decorators/schedule.py +137 -0
  84. unrealon_driver/decorators/task.py +61 -0
  85. unrealon_driver/decorators/timing.py +132 -0
  86. unrealon_driver/driver/__init__.py +20 -0
  87. unrealon_driver/driver/communication/__init__.py +10 -0
  88. unrealon_driver/driver/communication/session.py +203 -0
  89. unrealon_driver/driver/communication/websocket_client.py +197 -0
  90. unrealon_driver/driver/core/__init__.py +10 -0
  91. unrealon_driver/driver/core/config.py +85 -0
  92. unrealon_driver/driver/core/driver.py +221 -0
  93. unrealon_driver/driver/factory/__init__.py +9 -0
  94. unrealon_driver/driver/factory/manager_factory.py +130 -0
  95. unrealon_driver/driver/lifecycle/__init__.py +11 -0
  96. unrealon_driver/driver/lifecycle/daemon.py +76 -0
  97. unrealon_driver/driver/lifecycle/initialization.py +97 -0
  98. unrealon_driver/driver/lifecycle/shutdown.py +48 -0
  99. unrealon_driver/driver/monitoring/__init__.py +9 -0
  100. unrealon_driver/driver/monitoring/health.py +63 -0
  101. unrealon_driver/driver/utilities/__init__.py +10 -0
  102. unrealon_driver/driver/utilities/logging.py +51 -0
  103. unrealon_driver/driver/utilities/serialization.py +61 -0
  104. unrealon_driver/managers/__init__.py +32 -0
  105. unrealon_driver/managers/base.py +174 -0
  106. unrealon_driver/managers/browser.py +98 -0
  107. unrealon_driver/managers/cache.py +116 -0
  108. unrealon_driver/managers/http.py +107 -0
  109. unrealon_driver/managers/logger.py +286 -0
  110. unrealon_driver/managers/proxy.py +99 -0
  111. unrealon_driver/managers/registry.py +87 -0
  112. unrealon_driver/managers/threading.py +54 -0
  113. unrealon_driver/managers/update.py +107 -0
  114. unrealon_driver/utils/__init__.py +9 -0
  115. unrealon_driver/utils/time.py +10 -0
  116. unrealon-1.1.6.dist-info/METADATA +0 -625
  117. unrealon-1.1.6.dist-info/RECORD +0 -55
  118. unrealon-1.1.6.dist-info/entry_points.txt +0 -9
  119. unrealon_browser/managers/stealth.py +0 -388
  120. unrealon_driver/README.md +0 -0
  121. unrealon_driver/exceptions.py +0 -33
  122. unrealon_driver/html_analyzer/__init__.py +0 -32
  123. unrealon_driver/html_analyzer/cleaner.py +0 -657
  124. unrealon_driver/html_analyzer/config.py +0 -64
  125. unrealon_driver/html_analyzer/manager.py +0 -247
  126. unrealon_driver/html_analyzer/models.py +0 -115
  127. unrealon_driver/html_analyzer/websocket_analyzer.py +0 -157
  128. unrealon_driver/models/__init__.py +0 -31
  129. unrealon_driver/models/websocket.py +0 -98
  130. unrealon_driver/parser/__init__.py +0 -36
  131. unrealon_driver/parser/cli_manager.py +0 -142
  132. unrealon_driver/parser/daemon_manager.py +0 -403
  133. unrealon_driver/parser/managers/__init__.py +0 -25
  134. unrealon_driver/parser/managers/config.py +0 -293
  135. unrealon_driver/parser/managers/error.py +0 -412
  136. unrealon_driver/parser/managers/result.py +0 -321
  137. unrealon_driver/parser/parser_manager.py +0 -458
  138. unrealon_driver/smart_logging/__init__.py +0 -24
  139. unrealon_driver/smart_logging/models.py +0 -44
  140. unrealon_driver/smart_logging/smart_logger.py +0 -406
  141. unrealon_driver/smart_logging/unified_logger.py +0 -525
  142. unrealon_driver/websocket/__init__.py +0 -31
  143. unrealon_driver/websocket/client.py +0 -249
  144. unrealon_driver/websocket/config.py +0 -188
  145. unrealon_driver/websocket/manager.py +0 -90
@@ -1,625 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: unrealon
3
- Version: 1.1.6
4
- Summary: 🚀 Revolutionary web scraping platform with unbreakable stealth, AI-powered extraction, and zero-config setup. Build enterprise parsers in minutes, not months!
5
- Project-URL: Homepage, https://github.com/unrealos/unrealon-rpc
6
- Project-URL: Documentation, https://unrealon-rpc.readthedocs.io
7
- Project-URL: Repository, https://github.com/unrealos/unrealon-rpc.git
8
- Project-URL: Issues, https://github.com/unrealos/unrealon-rpc/issues
9
- Project-URL: Changelog, https://github.com/unrealos/unrealon-rpc/blob/main/CHANGELOG.md
10
- Author-email: UnrealOS Team <dev@unrealos.com>
11
- Maintainer-email: UnrealOS Team <dev@unrealos.com>
12
- License: MIT
13
- License-File: LICENSE
14
- Keywords: ai-parsing,anti-detection,bot-protection,browser-automation,captcha-bypass,data-mining,distributed-parsing,enterprise-scraping,html-extraction,playwright,proxy-rotation,scalable-scraping,stealth-scraping,web-scraping,websocket-bridge,zero-config
15
- Classifier: Development Status :: 4 - Beta
16
- Classifier: Environment :: Web Environment
17
- Classifier: Framework :: AsyncIO
18
- Classifier: Intended Audience :: Developers
19
- Classifier: Intended Audience :: Information Technology
20
- Classifier: Intended Audience :: System Administrators
21
- Classifier: License :: OSI Approved :: MIT License
22
- Classifier: Operating System :: OS Independent
23
- Classifier: Programming Language :: Python
24
- Classifier: Programming Language :: Python :: 3
25
- Classifier: Programming Language :: Python :: 3.10
26
- Classifier: Programming Language :: Python :: 3.11
27
- Classifier: Programming Language :: Python :: 3.12
28
- Classifier: Topic :: Communications
29
- Classifier: Topic :: Database
30
- Classifier: Topic :: Internet :: WWW/HTTP
31
- Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
32
- Classifier: Topic :: Office/Business
33
- Classifier: Topic :: Scientific/Engineering :: Information Analysis
34
- Classifier: Topic :: Software Development :: Libraries :: Python Modules
35
- Classifier: Topic :: System :: Distributed Computing
36
- Classifier: Typing :: Typed
37
- Requires-Python: <4.0,>=3.10
38
- Requires-Dist: aiohttp>=3.9.0
39
- Requires-Dist: aioipfs<0.8.0,>=0.7.1
40
- Requires-Dist: asyncio-mqtt>=0.16.0
41
- Requires-Dist: beautifulsoup4>=4.13.4
42
- Requires-Dist: click>=8.2.0
43
- Requires-Dist: httpx>=0.26.0
44
- Requires-Dist: ipfshttpclient>=0.8.0a2
45
- Requires-Dist: lxml>=6.0.0
46
- Requires-Dist: msgpack<2.0.0,>=1.1.1
47
- Requires-Dist: playwright-stealth>=2.0.0
48
- Requires-Dist: playwright>=1.54.0
49
- Requires-Dist: pydantic-yaml<2.0.0,>=1.6.0
50
- Requires-Dist: pydantic<3.0,>=2.11
51
- Requires-Dist: python-dateutil>=2.8
52
- Requires-Dist: python-dotenv>=1.0.0
53
- Requires-Dist: pyyaml>=6.0
54
- Requires-Dist: questionary<3.0.0,>=2.1.0
55
- Requires-Dist: redis>=5.0.0
56
- Requires-Dist: rich>=13.0.0
57
- Requires-Dist: tomlkit>=0.13.0
58
- Requires-Dist: watchdog<7.0.0,>=6.0.0
59
- Requires-Dist: websockets>=12.0
60
- Provides-Extra: dev
61
- Requires-Dist: bandit>=1.7.0; extra == 'dev'
62
- Requires-Dist: black>=23.0.0; extra == 'dev'
63
- Requires-Dist: build>=1.0.0; extra == 'dev'
64
- Requires-Dist: flake8>=6.0.0; extra == 'dev'
65
- Requires-Dist: isort>=5.12.0; extra == 'dev'
66
- Requires-Dist: mkdocs-material>=9.0.0; extra == 'dev'
67
- Requires-Dist: mkdocs>=1.5.0; extra == 'dev'
68
- Requires-Dist: mkdocstrings[python]>=0.22.0; extra == 'dev'
69
- Requires-Dist: mypy>=1.5.0; extra == 'dev'
70
- Requires-Dist: pre-commit>=3.0.0; extra == 'dev'
71
- Requires-Dist: pydocstyle>=6.3.0; extra == 'dev'
72
- Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
73
- Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
74
- Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
75
- Requires-Dist: pytest-xdist>=3.0.0; extra == 'dev'
76
- Requires-Dist: pytest>=7.0; extra == 'dev'
77
- Requires-Dist: questionary>=2.1.0; extra == 'dev'
78
- Requires-Dist: twine>=4.0.0; extra == 'dev'
79
- Provides-Extra: docs
80
- Requires-Dist: mkdocs-material>=9.0.0; extra == 'docs'
81
- Requires-Dist: mkdocs>=1.5.0; extra == 'docs'
82
- Requires-Dist: mkdocstrings[python]>=0.22.0; extra == 'docs'
83
- Requires-Dist: pymdown-extensions>=10.0.0; extra == 'docs'
84
- Provides-Extra: test
85
- Requires-Dist: factory-boy>=3.2.0; extra == 'test'
86
- Requires-Dist: pytest-asyncio>=0.21.0; extra == 'test'
87
- Requires-Dist: pytest-cov>=4.0.0; extra == 'test'
88
- Requires-Dist: pytest-mock>=3.10.0; extra == 'test'
89
- Requires-Dist: pytest-xdist>=3.0.0; extra == 'test'
90
- Requires-Dist: pytest>=7.0; extra == 'test'
91
- Description-Content-Type: text/markdown
92
-
93
- # 🚀 UnrealOn - Next-Generation Web Scraping Platform
94
-
95
- > **Enterprise-grade browser automation framework that makes web scraping simple, reliable, and scalable**
96
-
97
- UnrealOn is a revolutionary web scraping platform that **solves all developer problems** once and for all. Forget about CAPTCHAs, blocks, browser setup, and infrastructure - **just write business logic!**
98
-
99
- [![PyPI version](https://badge.fury.io/py/unrealon.svg)](https://badge.fury.io/py/unrealon)
100
- [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
101
- [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
102
-
103
- ---
104
-
105
- ## ✨ Why UnrealOn?
106
-
107
- ### 🛡️ **Unbreakable Stealth Mode**
108
- - **100% bot detection bypass** - enterprise-level anti-detection
109
- - Automatic User-Agent, fingerprint, and TLS parameter rotation
110
- - Human-like behavior simulation at browser level
111
- - **No CAPTCHAs or blocks** - the system handles everything
112
-
113
- ### 🧠 **AI-Powered Parsing**
114
- - **Smart parsing by URL** - just provide a link, get structured data
115
- - Automatic content recognition using LLM
116
- - Adapts to website structure changes
117
- - **Zero selector configuration**
118
-
119
- ### 🎯 **Zero-Configuration Approach**
120
- - **Works out of the box** - no complex setup required
121
- - Automatic browser and proxy management
122
- - Built-in logging and monitoring system
123
- - **Just run and it works**
124
-
125
- ### 📊 **UnrealOn Cloud Platform**
126
- - Real-time monitoring of all parsers
127
- - Centralized logging and analytics
128
- - Task management through web interface
129
- - **Complete control over your parsing farm**
130
-
131
- ---
132
-
133
- ## 🎮 Quick Start
134
-
135
- ### 1️⃣ Installation (30 seconds)
136
- ```bash
137
- pip install unrealon
138
- ```
139
-
140
- ### 2️⃣ Your First Parser (2 minutes)
141
- ```python
142
- from unrealon import ParserManager
143
- import asyncio
144
-
145
- class MyParser(ParserManager):
146
- async def parse_products(self, url: str):
147
- # Navigate with built-in stealth
148
- await self.browser.navigate(url)
149
-
150
- # AI-powered extraction - no selectors needed!
151
- result = await self.extract_with_ai(
152
- url,
153
- "Extract all products with title, price, and image"
154
- )
155
-
156
- return result.data
157
-
158
- # Usage
159
- async def main():
160
- parser = MyParser()
161
- await parser.setup()
162
-
163
- products = await parser.parse_products("https://example.com/products")
164
- print(f"Found {len(products)} products!")
165
-
166
- await parser.cleanup()
167
-
168
- asyncio.run(main())
169
- ```
170
-
171
- ### 3️⃣ Daemon Mode with Cloud Platform
172
- ```python
173
- # Run as daemon with real-time dashboard
174
- await parser.start_daemon()
175
-
176
- # Now control via web interface at https://cloud.unrealon.com
177
- ```
178
-
179
- **That's it! You have a production-ready parser in 3 steps!**
180
-
181
- ---
182
-
183
- ## 🏗️ Architecture Overview
184
-
185
- ### 🎯 **Developer's Perspective - Simple & Clean**
186
-
187
- **Architecture Overview - Developer's Perspective:**
188
-
189
- - **💻 Your Parser Code (Python Script)**
190
- - Simple class extending ParserManager
191
- - Focus on business logic only
192
- - Example: `async def parse_products(url): return await self.extract_with_ai(url)`
193
-
194
- - **🚀 Built-in Browser (Playwright + Stealth)**
195
- - ✅ Anti-Detection
196
- - ✅ Proxy Rotation
197
- - ✅ CAPTCHA Solving
198
-
199
- - **🌐 Target Websites**
200
- - 🛒 E-commerce Sites
201
- - 📰 News Portals
202
- - 📱 Social Media
203
- - 🌍 Any Website
204
-
205
- - **📊 UnrealOn Dashboard**
206
- - 📈 Real-time Monitoring
207
- - 📋 Logs & Analytics
208
- - ⚙️ Task Management
209
- - 💾 Data Storage
210
-
211
- **Flow:** Your code → Built-in Browser → Target Websites
212
- **Automatic Sync:** Your code ⟷ UnrealOn Dashboard (metrics, logs, parsed data)
213
-
214
- ### 🔄 **Two Operation Modes**
215
-
216
- #### 🔧 **Standalone Mode** (Local Development)
217
- **Standalone Mode Flow:**
218
-
219
- - 💻 Your Parser (Local Python Script)
220
- - 🚀 Built-in Browser with Stealth Enabled
221
- - 🌐 Target Website (E-commerce/News)
222
- - 💾 Local Results (JSON/CSV/Database)
223
-
224
- **Process:** Your Parser → Browser → Target Website → Local Results
225
-
226
- #### 🚀 **Dashboard Mode** (Production)
227
- **Dashboard Mode Flow:**
228
-
229
- - 💻 Your Parser (Production Script)
230
- - 🚀 Built-in Browser with Enterprise Stealth
231
- - 🌐 Target Website (E-commerce/News)
232
- - 📊 UnrealOn Dashboard (Cloud Platform)
233
- - 👥 Team Collaboration & Role Management
234
- - 📈 Analytics & Business Intelligence Reports
235
- - 📤 Data Export via API/Webhooks
236
-
237
- **Process:**
238
- - Parser → Browser → Target Website
239
- - Parser → Dashboard → Team/Analytics/Export
240
-
241
- ### 🎯 **What You Focus On vs What UnrealOn Handles**
242
-
243
- **What You Focus On vs What UnrealOn Handles:**
244
-
245
- **🎯 Your Focus - Business Logic Only:**
246
- 1. 🎯 Define Target URLs
247
- - Example: `urls = ['amazon.com', 'ebay.com']`
248
- 2. 🔍 Specify Data to Extract
249
- - Example: `'Extract title, price, rating'`
250
- 3. 📊 Handle Results
251
- - Save to database/API
252
- 4. ⏰ Schedule Tasks
253
- - Run every hour/daily
254
-
255
- **🚀 UnrealOn Handles All Infrastructure:**
256
- 1. 🌐 Browser Management (Playwright + Chrome)
257
- 2. 🛡️ Stealth & Anti-Detection (Fingerprint Spoofing)
258
- 3. 🔄 Proxy Rotation (Global IP Pool)
259
- 4. 🤖 CAPTCHA Solving (Automatic Resolution)
260
- 5. ⚠️ Error Handling (Retry Logic)
261
- 6. 📈 Logging & Monitoring (Real-time Metrics)
262
- 7. 💾 Data Storage (Cloud Database)
263
- 8. ⚡ Performance Optimization (Auto-scaling)
264
-
265
- **Each of your actions automatically triggers the corresponding infrastructure components.**
266
-
267
- **🎉 Result: You write 10 lines of business logic, UnrealOn handles 1000+ lines of infrastructure!**
268
-
269
- ---
270
-
271
- ## 🎛️ Multiple Operation Modes
272
-
273
- ### 🔧 **Standalone Mode** (Simplest)
274
- Perfect for quick tasks and development:
275
-
276
- ```python
277
- from unrealon import quick_parse
278
-
279
- # One-liner magic - AI does everything
280
- products = await quick_parse("https://shop.com/products")
281
- ```
282
-
283
- ### 🤖 **Traditional Mode** (Full Control)
284
- For developers who prefer CSS selectors:
285
-
286
- ```python
287
- from unrealon import ParserManager
288
- from bs4 import BeautifulSoup
289
-
290
- class TraditionalParser(ParserManager):
291
- async def parse_products(self, url: str):
292
- html = await self.browser.get_html(url)
293
- soup = BeautifulSoup(html, "html.parser")
294
-
295
- products = []
296
- for item in soup.select(".product"):
297
- products.append({
298
- "title": item.select_one(".title").text,
299
- "price": item.select_one(".price").text
300
- })
301
-
302
- return products
303
- ```
304
-
305
- ### 🚀 **Daemon Mode** (Production)
306
- For enterprise deployments with dashboard:
307
-
308
- ```python
309
- class ProductionParser(ParserManager):
310
- async def handle_parse_command(self, command):
311
- """Handle remote commands from dashboard"""
312
- url = command.data.get("url")
313
- return await self.parse_products(url)
314
-
315
- # Start daemon
316
- await parser.start_daemon(
317
- api_key="your_api_key"
318
- )
319
- ```
320
-
321
- ### ⏰ **Scheduled Mode** (Automation)
322
- For regular data collection:
323
-
324
- ```python
325
- class ScheduledParser(ParserManager):
326
- async def run_scheduled(self):
327
- """Called automatically by scheduler"""
328
- urls = self.get_target_urls()
329
- results = []
330
-
331
- for url in urls:
332
- data = await self.parse_products(url)
333
- results.extend(data)
334
-
335
- return results
336
-
337
- # Run every hour
338
- await parser.start_daemon(schedule="1h")
339
- ```
340
-
341
- ---
342
-
343
- ## 🛡️ Advanced Stealth Technologies
344
-
345
- ### Built-in Anti-Detection Features:
346
- - **Playwright Stealth** - Browser fingerprint modification
347
- - **Proxy Rotation** - Automatic IP address switching
348
- - **User-Agent Spoofing** - Mimicking different browsers
349
- - **Request Timing** - Human-like delays
350
- - **Cookie Management** - Session persistence
351
- - **CAPTCHA Solving** - Automatic CAPTCHA resolution
352
- - **Behavioral Patterns** - User action simulation
353
-
354
- ### Stealth Features:
355
- ```python
356
- # Stealth is always enabled by default
357
- parser = ParserManager() # 🔥 STEALTH ALWAYS ON!
358
- ```
359
-
360
- - **Webdriver Detection Prevention** - Hides automation signals
361
- - **Browser Fingerprint Randomization** - Unique fingerprints
362
- - **JavaScript API Modifications** - Prevents detection
363
-
364
- ---
365
-
366
- ## 🧠 AI-Powered Features
367
-
368
- ```python
369
- # Smart content extraction - AI understands page structure
370
- result = await parser.extract_with_ai(
371
- url="https://ecommerce.com/products",
372
- instruction="Extract product name, price, rating"
373
- )
374
-
375
- print(f"Extracted {len(result.data)} products")
376
- print(f"Confidence: {result.confidence}")
377
-
378
- # AI adapts to website changes automatically
379
- result = await parser.adaptive_parse(
380
- url="https://news.com",
381
- data_type="articles",
382
- fields=["title", "author", "date"]
383
- )
384
- ```
385
-
386
- ---
387
-
388
- ## 📊 Enterprise Dashboard Features
389
-
390
- - 📈 **Live Metrics** - RPS, success rate, errors
391
- - 📋 **Task Management** - Create, stop, schedule tasks
392
- - 🔍 **Log Search** - Instant search across all events
393
- - 🚨 **Alerts** - Slack, Email, Telegram notifications
394
- - 👥 **Team Collaboration** - Roles and permissions
395
-
396
- **Access:** [https://cloud.unrealon.com](https://cloud.unrealon.com)
397
-
398
- ```python
399
- # Control parsers via API
400
- response = requests.post("https://api.unrealon.com/parsers/start", {
401
- "parser_id": "my_parser", "config": {"max_pages": 10}
402
- })
403
- ```
404
-
405
- ---
406
-
407
- ## 🎯 Working Examples
408
-
409
- ### E-commerce Parser
410
- ```python
411
- class EcommerceParser(ParserManager):
412
- async def parse_products(self, url: str):
413
- await self.browser.navigate(url)
414
-
415
- # AI extracts all product data automatically
416
- products = await self.extract_with_ai(
417
- url, "Extract products with title, price, rating"
418
- )
419
-
420
- return products.data
421
-
422
- # Usage - Parse multiple sites
423
- parser = EcommerceParser()
424
- await parser.setup()
425
-
426
- amazon_products = await parser.parse_products("https://amazon.com/s?k=laptop")
427
- ebay_products = await parser.parse_products("https://ebay.com/sch/laptop")
428
-
429
- await parser.cleanup()
430
- ```
431
-
432
- ### News & Social Media
433
- ```python
434
- class NewsParser(ParserManager):
435
- async def parse_articles(self, url: str):
436
- await self.browser.navigate(url)
437
- return await self.extract_with_ai(url, "Extract articles with title, author, date")
438
-
439
- # Parse multiple sources
440
- sources = ["https://news.ycombinator.com", "https://techcrunch.com"]
441
- all_articles = []
442
- for source in sources:
443
- articles = await parser.parse_articles(source)
444
- all_articles.extend(articles)
445
- ```
446
-
447
- ---
448
-
449
- ## 🔧 Configuration
450
-
451
- ```yaml
452
- # config.yaml
453
- parser:
454
- name: "My Parser"
455
- target_urls:
456
- - https://example.com/products
457
-
458
- browser:
459
- headless: true
460
-
461
- bridge:
462
- enabled: true
463
- api_key: "your_api_key"
464
-
465
- processing:
466
- delay_between_requests: 1.0
467
- max_pages: 1
468
-
469
- logging:
470
- level: INFO
471
- to_bridge: true
472
- ```
473
-
474
- ---
475
-
476
- ## 🚀 CLI Tools
477
-
478
- ```bash
479
- # Quick parsing
480
- unrealon parse --url https://example.com --ai-instruction "Extract products"
481
-
482
- # Start daemon
483
- unrealon daemon --config config.yaml
484
-
485
- # Test stealth
486
- unrealon browser test-stealth --url https://bot.sannysoft.com
487
-
488
- # Export results
489
- unrealon export --format csv --output results.csv
490
- ```
491
-
492
- ---
493
-
494
- ## 🎉 Real-World Success Stories
495
-
496
- ### 🚗 **CarAPIs** - Automotive Data Platform
497
- **Platform**: [carapis.com](https://carapis.com)
498
- **Challenge**: Extract vehicle data from 500+ dealership websites
499
- **Solution**: UnrealOn with AI-powered extraction
500
- **Results**: 95% accuracy, 10M+ vehicles processed monthly
501
-
502
- ### 🛒 **ShopAPIs** - E-commerce Intelligence
503
- **Platform**: [shopapis.com](https://shopapis.com)
504
- **Challenge**: Monitor prices across 50+ e-commerce platforms
505
- **Solution**: UnrealOn cluster with real-time monitoring
506
- **Results**: 99.9% uptime, 1M+ products tracked daily
507
-
508
- ### 📊 **StockAPIs** - Financial Data Platform
509
- **Platform**: [stockapis.com](https://stockapis.com)
510
- **Challenge**: High-frequency financial data collection
511
- **Solution**: UnrealOn with millisecond precision
512
- **Results**: 100K+ data points per second, 99.99% accuracy
513
-
514
- ### 🏠 **PropAPIs** - Real Estate Intelligence
515
- **Platform**: [propapis.com](https://propapis.com)
516
- **Challenge**: Aggregate listings from 200+ real estate sites
517
- **Solution**: UnrealOn with geographic clustering
518
- **Results**: 5M+ properties indexed, real-time updates
519
-
520
- **All platforms built with UnrealOn - proving enterprise reliability!**
521
-
522
- ---
523
-
524
- ## 💎 Enterprise Features
525
-
526
- Need **enterprise capabilities**?
527
-
528
- ### 🏢 **Enterprise Edition Includes:**
529
- - 🛡️ **Dedicated Infrastructure** - Private cloud deployment
530
- - 🔒 **Advanced Security** - SOC2/GDPR compliance
531
- - 🤝 **24/7 Support** - Dedicated success manager
532
- - 📊 **Custom Analytics** - Tailored reporting and insights
533
- - 🚀 **Priority Features** - Early access to new capabilities
534
- - 🔧 **Custom Integrations** - Bespoke API development
535
-
536
- ### 📞 **Contact Enterprise Sales:**
537
- - **Email**: [enterprise@unrealon.com](mailto:enterprise@unrealon.com)
538
- - **Phone**: +1 (555) 123-4567
539
- - **Schedule Demo**: [calendly.com/unrealon-demo](https://calendly.com/unrealon-demo)
540
-
541
- ---
542
-
543
- ## 📚 Documentation & Support
544
-
545
- ### 📖 **Resources:**
546
- - [📘 Complete Documentation](https://docs.unrealon.com)
547
- - [🎥 Video Tutorials](https://youtube.com/unrealon)
548
- - [💬 Discord Community](https://discord.gg/unrealon)
549
- - [📧 Technical Support](mailto:support@unrealon.com)
550
-
551
- ### 🎓 **Learning Path:**
552
- 1. [🚀 Quick Start (5 minutes)](https://docs.unrealon.com/quickstart)
553
- 2. [🏗️ Platform Architecture](https://docs.unrealon.com/architecture)
554
- 3. [🛡️ Advanced Stealth Guide](https://docs.unrealon.com/stealth)
555
- 4. [🤖 AI Parsing Tutorial](https://docs.unrealon.com/ai-parsing)
556
- 5. [📊 Dashboard Management](https://docs.unrealon.com/dashboard)
557
-
558
- ### 🆘 **Getting Help:**
559
- - **GitHub Issues**: [Report bugs](https://github.com/unrealon/unrealon-rpc/issues)
560
- - **GitHub Discussions**: [Ask questions](https://github.com/unrealon/unrealon-rpc/discussions)
561
- - **Stack Overflow**: Tag your questions with `unrealon`
562
- - **Email Support**: [support@unrealon.com](mailto:support@unrealon.com)
563
-
564
- ---
565
-
566
- ## 🤝 Contributing
567
-
568
- We welcome contributions! Here's how to get started:
569
-
570
- ### Development Setup
571
- ```bash
572
- # Clone repository
573
- git clone https://github.com/unrealon/unrealon-rpc.git
574
- cd unrealon-rpc
575
-
576
- # Install development dependencies
577
- poetry install
578
-
579
- # Install pre-commit hooks
580
- pre-commit install
581
-
582
- # Run tests
583
- pytest
584
-
585
- # Run linting
586
- black src/
587
- isort src/
588
- mypy src/
589
- ```
590
-
591
- ### Contribution Guidelines
592
- - Follow PEP 8 style guide
593
- - Add type hints to all functions
594
- - Write comprehensive docstrings
595
- - Include tests for new features
596
- - Update documentation as needed
597
-
598
- ---
599
-
600
- ## 📄 License
601
-
602
- This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
603
-
604
- ---
605
-
606
- <div align="center">
607
-
608
- ## 🚀 Start Building Amazing Parsers Today!
609
-
610
- ```bash
611
- pip install unrealon
612
- ```
613
-
614
- **UnrealOn Platform** - The Future of Web Scraping is Here! 🌟
615
-
616
- [![GitHub](https://img.shields.io/badge/GitHub-unrealon-blue?logo=github)](https://github.com/unrealon)
617
- [![Discord](https://img.shields.io/badge/Discord-Join-7289da?logo=discord)](https://discord.gg/unrealon)
618
- [![Documentation](https://img.shields.io/badge/Docs-Read-green?logo=gitbook)](https://docs.unrealon.com)
619
- [![Twitter](https://img.shields.io/badge/Twitter-Follow-1da1f2?logo=twitter)](https://twitter.com/unrealon)
620
-
621
- *Built with ❤️ by the UnrealOn Team*
622
-
623
- **Ready to revolutionize your web scraping?** [Get Started Now!](https://docs.unrealon.com/quickstart)
624
-
625
- </div>
@@ -1,55 +0,0 @@
1
- unrealon_browser/README.md,sha256=9pP6RrfMGHtdT5uDLFAUB1e4nNGzZudXViEo1940gKw,396
2
- unrealon_browser/__init__.py,sha256=Bk1jERQP3CpwfExd3UmydNRIZcEIylQaEfP430p9Vjg,1502
3
- unrealon_browser/cli/__init__.py,sha256=b3r88oeCYsqZF8EU8EZXP9v54Q8cIimN7UmxJsXcB84,264
4
- unrealon_browser/cli/browser_cli.py,sha256=SRRCGbNXaEg1ZN04-jPo9GOWcP2b6Bkalu6PYVOOt5k,8342
5
- unrealon_browser/cli/cookies_cli.py,sha256=yhZvGrg8bknlH4zlySdi8ue-25Ue-1rI_u1G06OIMg4,13304
6
- unrealon_browser/cli/interactive_mode.py,sha256=KgdPPa5rZEEApc92JQz346WMESAFDBHERpz_dxeZgKk,11885
7
- unrealon_browser/cli/main.py,sha256=XCYcTxJUqaz320KCU_JPKizYMk6bdljb8Boyok3uO-4,1353
8
- unrealon_browser/core/__init__.py,sha256=uVL_t4sZelUzflWPdgrwoXGnAkSV1WNQ98-eu0QB2eM,151
9
- unrealon_browser/core/browser_manager.py,sha256=-m6ZeuRTUvI7t7vZ20xK5klR6px9eeswSv1z6eGkD2k,29056
10
- unrealon_browser/dto/__init__.py,sha256=p9mG2QwnXEdHUHYK67vGD6aameM8RkiVATzz8y0u5EE,1206
11
- unrealon_browser/dto/models/config.py,sha256=3O5nk6vFw6Kxgu941lJIg9dGAUzXacQn0yvxC1mGEa4,1585
12
- unrealon_browser/dto/models/core.py,sha256=HvbwYG27rmmWtp401uws7lfalN_9QPad0M6ceCiN5iQ,2741
13
- unrealon_browser/dto/models/dataclasses.py,sha256=zqhJVyzp4CvtuTBsZwm6n6TodVWrZf9gkdDG-0_tgeA,2571
14
- unrealon_browser/dto/models/detection.py,sha256=ma9ZNIjPR7HnjqZaAj6ZoskiewPFiSn_FgFXSkgiQc8,2715
15
- unrealon_browser/dto/models/enums.py,sha256=Q4WzHdfSKf7dhKyX00i_Pvl2U8w3lBsxOYfSIoaQY3Q,1219
16
- unrealon_browser/dto/models/statistics.py,sha256=aIzJNV5r23VBxjhEoja4tXwI1Z7_UCw5zOaxuPya2E8,2728
17
- unrealon_browser/managers/__init__.py,sha256=ec5gvXFv2iLJD9rVDZkKF_yszqOSxVnl9m8AWEVmI8c,527
18
- unrealon_browser/managers/captcha.py,sha256=KGBO7sfq9XusAlcPByUFdIg-v6rlruzS2oHx-Zx28wo,21453
19
- unrealon_browser/managers/cookies.py,sha256=r4VVnKLXH82vhU7qgtY-dF7KPf0Ie3QxGD3FEi6geFA,15085
20
- unrealon_browser/managers/logger_bridge.py,sha256=wWf13oizI3fBXbTvE_ghjhWbb2q8xp9gsTTUm9UNmJg,11098
21
- unrealon_browser/managers/page_wait_manager.py,sha256=UyZqiSfkjzahrxp9x1odXFIT_sFhZGvdECxWuIMCVBY,7876
22
- unrealon_browser/managers/profile.py,sha256=HjddlSeUry_65WPtF8CMkT7cfJ6X3Jap9kJaaZpwtAA,18956
23
- unrealon_browser/managers/stealth.py,sha256=-ge67IAueO6zRa0SSffi8Fpd2mhG_2IyRvZADzxJbfk,14651
24
- unrealon_driver/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
- unrealon_driver/__init__.py,sha256=GZq15ye175N4MchCxMklfDzbIokLcmIZC92fMPgx4ls,1600
26
- unrealon_driver/exceptions.py,sha256=5b7ndK-UROfVvON3qf9YVXK4tq8PZzgtBSHTkiA1Oos,437
27
- unrealon_driver/html_analyzer/__init__.py,sha256=sDrhKYJ9I3OdgbL60w92WxfiHD_Jhcw0vgV2XYkKQwg,1007
28
- unrealon_driver/html_analyzer/cleaner.py,sha256=77umjQojIl9BMSFpe4j6x3_5TMQh7Jk5GXUhuempu3k,25198
29
- unrealon_driver/html_analyzer/config.py,sha256=BnAUKIr2sLX8XtJ0CC0PxTQieR2eA4SJJ_DM22IhwrY,3021
30
- unrealon_driver/html_analyzer/manager.py,sha256=0y8EBEJeW3LuWWBLt--W0czs_-UZ-4j5btLS8cigb6E,10734
31
- unrealon_driver/html_analyzer/models.py,sha256=EXiGJC3i8O-rjqGH4YKVuyS0vqp0s8BmvNZ78qevt50,4854
32
- unrealon_driver/html_analyzer/websocket_analyzer.py,sha256=MXqtEdV2K4CF4I1Z0zKexf1uqDdwmuguzuAkjV2tyAA,5768
33
- unrealon_driver/models/__init__.py,sha256=ugsaGIJWH5QU7FduoCoV7S9AxKzYCPisAqow6oV89JQ,620
34
- unrealon_driver/models/websocket.py,sha256=zmz_J8vER-Y4BCwDXCUbnc-eoUpbFcRLXJRsp_8xFbs,4271
35
- unrealon_driver/parser/__init__.py,sha256=QW1u5zju71o7mtvINpdz7dO8ZltylrtXUkSm4xzWDwo,949
36
- unrealon_driver/parser/cli_manager.py,sha256=l0Kk7zm1KGvnjjyydtXxcULmwFerEE03_ZZp4dkN7nc,5124
37
- unrealon_driver/parser/daemon_manager.py,sha256=0GQ4_ZffJQKcEpw3kyPLXu8up1LGptLbly0BQxwXz5M,15945
38
- unrealon_driver/parser/parser_manager.py,sha256=URq1xZ7Uyw5CM4TXYWVc1n3IzylyLUfmK5iO2HDYLL8,17923
39
- unrealon_driver/parser/managers/__init__.py,sha256=DChR17AOEabrIuSNElEzA8kmjmc33rcB1n_RTo3QOaU,616
40
- unrealon_driver/parser/managers/config.py,sha256=H0MM8FOx_2JQESY9AujrTvsslC9NCTs7sayDIGq3TBs,9897
41
- unrealon_driver/parser/managers/error.py,sha256=EnJkZLlZihXeKZdHauvnqaSby05caxHOYYIuiLoelkw,14555
42
- unrealon_driver/parser/managers/result.py,sha256=yIoDTx6e1YzhKmJ2yJPD8_eAAkrjtm7ofN3DFEOLdUU,10236
43
- unrealon_driver/smart_logging/__init__.py,sha256=sC0Bzzvx73V2iAj7-AH3-H_WFKGRH04eajodREIuwgg,544
44
- unrealon_driver/smart_logging/models.py,sha256=lGqCxjBp18mA35-VMn7GRUCJvISMw-p1cGUtD8Im3Qo,1079
45
- unrealon_driver/smart_logging/smart_logger.py,sha256=BVLjA2FTZfFiY97AE3mZh0Ds8rtCj97yIKFcHP56afU,13039
46
- unrealon_driver/smart_logging/unified_logger.py,sha256=4DfDFNQE1I555Gk1HAxsrd5MFyLID_FyYiqgVRslNvE,18284
47
- unrealon_driver/websocket/__init__.py,sha256=qwOOIKEcwvP_0GYW6fTAmNdiUMHaotG_IL56g3wi_aU,1008
48
- unrealon_driver/websocket/client.py,sha256=NvfbeDe_uDfLniS2q6Qzd5p4UnhSqvdSzUaUXNMtzLw,9179
49
- unrealon_driver/websocket/config.py,sha256=ez6EjAOPjDMoQeXHEkulT90o2vpeC34eShgNFLX2g1E,6500
50
- unrealon_driver/websocket/manager.py,sha256=8RAq3NuBFfV2y0ThAZVD7IhO_t7OAJV9phvnZooaLlc,2779
51
- unrealon-1.1.6.dist-info/METADATA,sha256=ZR5NN3JCehzeweWQhnVYteEhVeJdrviGCQ_7SzpbFOc,19458
52
- unrealon-1.1.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
53
- unrealon-1.1.6.dist-info/entry_points.txt,sha256=s-tuFkQDg3TQ3U5k0yzt6oWGUhTLum0BLdCQEIDnpiU,431
54
- unrealon-1.1.6.dist-info/licenses/LICENSE,sha256=uTZpktXKUsE0IzS5RdSV398HHI74ssbGKTdCbv7U9l0,1070
55
- unrealon-1.1.6.dist-info/RECORD,,
@@ -1,9 +0,0 @@
1
- [console_scripts]
2
- browser-cookies = unrealon_browser.cli.cookies_cli:main
3
- browser-interactive = unrealon_browser.cli.interactive_mode:main
4
- browser-profiles = unrealon_browser.cli.main:profiles_command
5
- browser-stealth = unrealon_browser.cli.main:stealth_command
6
- unrealon = unrealon_driver.cli:main
7
- unrealon-bridge = unrealon_server.cli:main
8
- unrealon-browser = unrealon_browser.cli.main:cli
9
- unrealon-rpc = unrealon_rpc.cli.main:main