unrealon 1.1.1__py3-none-any.whl → 1.1.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. unrealon/__init__.py +16 -6
  2. unrealon-1.1.5.dist-info/METADATA +621 -0
  3. unrealon-1.1.5.dist-info/RECORD +54 -0
  4. {unrealon-1.1.1.dist-info → unrealon-1.1.5.dist-info}/entry_points.txt +1 -1
  5. unrealon_browser/__init__.py +3 -6
  6. unrealon_browser/core/browser_manager.py +86 -84
  7. unrealon_browser/dto/models/config.py +2 -0
  8. unrealon_browser/managers/captcha.py +165 -185
  9. unrealon_browser/managers/cookies.py +57 -28
  10. unrealon_browser/managers/logger_bridge.py +94 -34
  11. unrealon_browser/managers/profile.py +186 -158
  12. unrealon_browser/managers/stealth.py +58 -47
  13. unrealon_driver/__init__.py +8 -21
  14. unrealon_driver/exceptions.py +5 -0
  15. unrealon_driver/html_analyzer/__init__.py +32 -0
  16. unrealon_driver/{parser/managers/html.py → html_analyzer/cleaner.py} +330 -405
  17. unrealon_driver/html_analyzer/config.py +64 -0
  18. unrealon_driver/html_analyzer/manager.py +247 -0
  19. unrealon_driver/html_analyzer/models.py +115 -0
  20. unrealon_driver/html_analyzer/websocket_analyzer.py +157 -0
  21. unrealon_driver/models/__init__.py +31 -0
  22. unrealon_driver/models/websocket.py +98 -0
  23. unrealon_driver/parser/__init__.py +4 -23
  24. unrealon_driver/parser/cli_manager.py +6 -5
  25. unrealon_driver/parser/daemon_manager.py +242 -66
  26. unrealon_driver/parser/managers/__init__.py +0 -21
  27. unrealon_driver/parser/managers/config.py +15 -3
  28. unrealon_driver/parser/parser_manager.py +225 -395
  29. unrealon_driver/smart_logging/__init__.py +24 -0
  30. unrealon_driver/smart_logging/models.py +44 -0
  31. unrealon_driver/smart_logging/smart_logger.py +406 -0
  32. unrealon_driver/smart_logging/unified_logger.py +525 -0
  33. unrealon_driver/websocket/__init__.py +31 -0
  34. unrealon_driver/websocket/client.py +249 -0
  35. unrealon_driver/websocket/config.py +188 -0
  36. unrealon_driver/websocket/manager.py +90 -0
  37. unrealon-1.1.1.dist-info/METADATA +0 -722
  38. unrealon-1.1.1.dist-info/RECORD +0 -82
  39. unrealon_bridge/__init__.py +0 -114
  40. unrealon_bridge/cli.py +0 -316
  41. unrealon_bridge/client/__init__.py +0 -93
  42. unrealon_bridge/client/base.py +0 -78
  43. unrealon_bridge/client/commands.py +0 -89
  44. unrealon_bridge/client/connection.py +0 -90
  45. unrealon_bridge/client/events.py +0 -65
  46. unrealon_bridge/client/health.py +0 -38
  47. unrealon_bridge/client/html_parser.py +0 -146
  48. unrealon_bridge/client/logging.py +0 -139
  49. unrealon_bridge/client/proxy.py +0 -70
  50. unrealon_bridge/client/scheduler.py +0 -450
  51. unrealon_bridge/client/session.py +0 -70
  52. unrealon_bridge/configs/__init__.py +0 -14
  53. unrealon_bridge/configs/bridge_config.py +0 -212
  54. unrealon_bridge/configs/bridge_config.yaml +0 -39
  55. unrealon_bridge/models/__init__.py +0 -138
  56. unrealon_bridge/models/base.py +0 -28
  57. unrealon_bridge/models/command.py +0 -41
  58. unrealon_bridge/models/events.py +0 -40
  59. unrealon_bridge/models/html_parser.py +0 -79
  60. unrealon_bridge/models/logging.py +0 -55
  61. unrealon_bridge/models/parser.py +0 -63
  62. unrealon_bridge/models/proxy.py +0 -41
  63. unrealon_bridge/models/requests.py +0 -95
  64. unrealon_bridge/models/responses.py +0 -88
  65. unrealon_bridge/models/scheduler.py +0 -592
  66. unrealon_bridge/models/session.py +0 -28
  67. unrealon_bridge/server/__init__.py +0 -91
  68. unrealon_bridge/server/base.py +0 -171
  69. unrealon_bridge/server/handlers/__init__.py +0 -23
  70. unrealon_bridge/server/handlers/command.py +0 -110
  71. unrealon_bridge/server/handlers/html_parser.py +0 -139
  72. unrealon_bridge/server/handlers/logging.py +0 -95
  73. unrealon_bridge/server/handlers/parser.py +0 -95
  74. unrealon_bridge/server/handlers/proxy.py +0 -75
  75. unrealon_bridge/server/handlers/scheduler.py +0 -545
  76. unrealon_bridge/server/handlers/session.py +0 -66
  77. unrealon_driver/browser/__init__.py +0 -8
  78. unrealon_driver/browser/config.py +0 -74
  79. unrealon_driver/browser/manager.py +0 -416
  80. unrealon_driver/parser/managers/browser.py +0 -51
  81. unrealon_driver/parser/managers/logging.py +0 -609
  82. {unrealon-1.1.1.dist-info → unrealon-1.1.5.dist-info}/WHEEL +0 -0
  83. {unrealon-1.1.1.dist-info → unrealon-1.1.5.dist-info}/licenses/LICENSE +0 -0
@@ -1,82 +0,0 @@
1
- unrealon/__init__.py,sha256=ZQ0UqKF7eEu3Sy1uzLrRNq57Kyv11XzQAn2FF6AkOU4,695
2
- unrealon_bridge/__init__.py,sha256=7IMIDQ4T8vu-J5D1_PaRDrwfANExUPwjSOH_1B_8WjI,2675
3
- unrealon_bridge/cli.py,sha256=wItb_FqHGAScEqfP0ipC-vaj0stOiO4hjgpfYLF0INs,11827
4
- unrealon_bridge/client/__init__.py,sha256=M33ADDrexYlSx7bQ6z0MQF_kv0W_AM8S7cfL0C18nUU,2993
5
- unrealon_bridge/client/base.py,sha256=iuBBKAadpacg4LL-kuPJ-atm8CKXMEdELh6UKh68JRo,2605
6
- unrealon_bridge/client/commands.py,sha256=SwjRM154BWgQLLKYq9ay1kC2uTmYsJHh2YwDzItmMUc,2844
7
- unrealon_bridge/client/connection.py,sha256=bCS-D7zAFM-er4nREZqKwKP1gFNDjAaW3l1ngbnXmJo,2864
8
- unrealon_bridge/client/events.py,sha256=gFguotwRPU_dXZ0iXkE3d97N-Vd935KU91OscEhuzuY,1842
9
- unrealon_bridge/client/health.py,sha256=sHgl93IyXSzlIUfgfKKnVoRQIeqozz7X-3jW61YbJ6c,946
10
- unrealon_bridge/client/html_parser.py,sha256=Pij8GWS76C7stQmr8V5S2iYuZSe0J5ACOPDFLdsZ93w,5138
11
- unrealon_bridge/client/logging.py,sha256=RP8d2xZJxjOziHrQOYX56oCBxTGCTDSIxJRjQZxMokY,4847
12
- unrealon_bridge/client/proxy.py,sha256=fJUBAnqe_BGCDYRW9ZhgWwJ_V30U1Tv1JPxmmtKp4LA,1975
13
- unrealon_bridge/client/scheduler.py,sha256=ZwewB-ZRO6XVIULx96wYI3OTvuft2S9jLTLqHStkO6A,14132
14
- unrealon_bridge/client/session.py,sha256=YOkJL60guDg9T0ftpQl-yHEAvtKFunmhYGcTSGt9A5M,1988
15
- unrealon_bridge/configs/__init__.py,sha256=Oj9OLuu9RtqexhdpOjVSqIRI_Gh5CA-5vb9zta177Nw,323
16
- unrealon_bridge/configs/bridge_config.py,sha256=1fwEFiViHdCzjP_yTl2QcfAYuNFof6ryhUH0mx3OBIA,7223
17
- unrealon_bridge/configs/bridge_config.yaml,sha256=QrKe87Zb2o4b6zeSNOaRP1EvArENxSTIrxEfn5kBUcw,710
18
- unrealon_bridge/models/__init__.py,sha256=EnB4OPwa6Og7p5pAzDKX4tf_B7c2jXpKdy02ngI0rU0,3559
19
- unrealon_bridge/models/base.py,sha256=TyEzJ2I80rQQzINIyWlIcGdC7EdW-cwE9uO5MO3sb3o,864
20
- unrealon_bridge/models/command.py,sha256=22T_30SyKugj7cW4vvmy4UCsmmn19LSRk36B2qEBsLw,1798
21
- unrealon_bridge/models/events.py,sha256=i9-TpUZeM_yCzBe9_YhURoOP0Q8ENn8gvl9F6Mm6Pa8,1704
22
- unrealon_bridge/models/html_parser.py,sha256=9Vat1I01QjMQAE8NhumIquCU6XjBd3iapSwWZPc8l8Y,2733
23
- unrealon_bridge/models/logging.py,sha256=beh9EFoQ7T7UEYRKHHxf2xnAbMTJBzUl9vsDBrS_1J4,1841
24
- unrealon_bridge/models/parser.py,sha256=6EGw2nGT_tS6c7BKlKRP-bRyxv76ee5bPOGp35tMdaQ,2694
25
- unrealon_bridge/models/proxy.py,sha256=Wgpq5fS8MSjeWl_j6liqh3ejBPEEZMbQOgjbEAHl9yo,1732
26
- unrealon_bridge/models/requests.py,sha256=YpR1SpGH4XqhJQRxvL47FhPhjTvHPsOG1ov5EMbm35c,3012
27
- unrealon_bridge/models/responses.py,sha256=Ao6su6x7-xBVX5Ju0IVM6xTnC_s4UxqnC5R7oX4UCNg,2670
28
- unrealon_bridge/models/scheduler.py,sha256=P1KkEU7u1LostR8TTaTQ-vrnADhyWH4rHefHYcf8IqI,18035
29
- unrealon_bridge/models/session.py,sha256=-vzaXlj3P9ZeqZiv0ZwH7SZ0bD94AG_qkm054BaOGzw,1146
30
- unrealon_bridge/server/__init__.py,sha256=07MhDxLbeFn6qBqUttph-aHbN1Sjra-xpaAI7VcjZpw,3902
31
- unrealon_bridge/server/base.py,sha256=JvAp24TIbW9Y8253kdKwq7N-7HaMIF8VBx8ZB4MJPmc,5792
32
- unrealon_bridge/server/handlers/__init__.py,sha256=w6hh0Mcjtm5h3D_-Qs-ACNCp6d9DqVjYwtij3HURLag,547
33
- unrealon_bridge/server/handlers/command.py,sha256=pc_EMKBMl9R59QJURRD2Fle-_leGMoPFKoN_eJrL0NM,4436
34
- unrealon_bridge/server/handlers/html_parser.py,sha256=TANWEgok1OL2gUNdsk9L7VdL8GbJOUoiSXQUGV4Wt4M,5150
35
- unrealon_bridge/server/handlers/logging.py,sha256=dZ2CCe1t5aue0IgR_LXo4dDq-mZ6VilkW5wBcOIBkdg,3001
36
- unrealon_bridge/server/handlers/parser.py,sha256=1XznKO9pTmhsc-FamApS_2eoIxfrwmZQniQgFQVACPs,4984
37
- unrealon_bridge/server/handlers/proxy.py,sha256=rXU9W-5DZ2daXVjVFra1V_7NhGCQ6cLJB9lMi4eV-qk,2801
38
- unrealon_bridge/server/handlers/scheduler.py,sha256=eYOAHE9kpQYaJM8G_FQWmlfSdjseYbxoLinuXmS4U-g,19559
39
- unrealon_bridge/server/handlers/session.py,sha256=YHyKDOjIJICRJ1HjtMhEikMYJWt8q3AEEuWPx4OOjjw,2409
40
- unrealon_browser/README.md,sha256=9pP6RrfMGHtdT5uDLFAUB1e4nNGzZudXViEo1940gKw,396
41
- unrealon_browser/__init__.py,sha256=s46-gFZQgW-89xcG_1a2k-GGNwIboMQVy7HkFrOVk90,1467
42
- unrealon_browser/cli/__init__.py,sha256=b3r88oeCYsqZF8EU8EZXP9v54Q8cIimN7UmxJsXcB84,264
43
- unrealon_browser/cli/browser_cli.py,sha256=SRRCGbNXaEg1ZN04-jPo9GOWcP2b6Bkalu6PYVOOt5k,8342
44
- unrealon_browser/cli/cookies_cli.py,sha256=yhZvGrg8bknlH4zlySdi8ue-25Ue-1rI_u1G06OIMg4,13304
45
- unrealon_browser/cli/interactive_mode.py,sha256=iYt9PNaIBhNjZ9aUa1ZxeeneV2u1VTSkYQ6rsyio-o8,11730
46
- unrealon_browser/cli/main.py,sha256=XCYcTxJUqaz320KCU_JPKizYMk6bdljb8Boyok3uO-4,1353
47
- unrealon_browser/core/__init__.py,sha256=uVL_t4sZelUzflWPdgrwoXGnAkSV1WNQ98-eu0QB2eM,151
48
- unrealon_browser/core/browser_manager.py,sha256=4x3wWEMJbENc0msCG3Mw8cn4ie2O-i77a7PcW7xF_3o,24439
49
- unrealon_browser/dto/__init__.py,sha256=p9mG2QwnXEdHUHYK67vGD6aameM8RkiVATzz8y0u5EE,1206
50
- unrealon_browser/dto/models/config.py,sha256=gMKOc1TpmjNSXxHqimHKGhGA71PZOBF5zkeAo3wSFZs,894
51
- unrealon_browser/dto/models/core.py,sha256=HvbwYG27rmmWtp401uws7lfalN_9QPad0M6ceCiN5iQ,2741
52
- unrealon_browser/dto/models/dataclasses.py,sha256=zqhJVyzp4CvtuTBsZwm6n6TodVWrZf9gkdDG-0_tgeA,2571
53
- unrealon_browser/dto/models/detection.py,sha256=ma9ZNIjPR7HnjqZaAj6ZoskiewPFiSn_FgFXSkgiQc8,2715
54
- unrealon_browser/dto/models/enums.py,sha256=Q4WzHdfSKf7dhKyX00i_Pvl2U8w3lBsxOYfSIoaQY3Q,1219
55
- unrealon_browser/dto/models/statistics.py,sha256=aIzJNV5r23VBxjhEoja4tXwI1Z7_UCw5zOaxuPya2E8,2728
56
- unrealon_browser/managers/__init__.py,sha256=JuH9FW_kTzVv71jCDp6wOT4SXT6HGSBpyNAb4tD7-ck,456
57
- unrealon_browser/managers/captcha.py,sha256=v5OKpQHCyNwsGBpQdnilcOPHxXZ8jZSHAjZ3DbtuaJ4,21432
58
- unrealon_browser/managers/cookies.py,sha256=p7Yl1GRGhi0JQw0KjH6m1HFVqP9Ap-eYFALvQ0hCiDs,13824
59
- unrealon_browser/managers/logger_bridge.py,sha256=6kWKDZG3wbHBoml6iQE2xKwN01tVztOVYvQ9BU6UB8o,8700
60
- unrealon_browser/managers/profile.py,sha256=vELLiTpMTglKFGAfRVyDqV8qnjrhG5IygHFsdtVmGbc,18175
61
- unrealon_browser/managers/stealth.py,sha256=eSLAqpCHyyntUD1RzZC0jpNpYFuHpKl4J9WxmICx3Ww,13890
62
- unrealon_driver/__init__.py,sha256=OQA93fHeIG5LhwP7MenaP0WA9i40Vnr0baHW4R1NOUA,1910
63
- unrealon_driver/exceptions.py,sha256=buCxZ-16AG9OTiqIgfNA_Z3QSxXiqM2kXzQTvq3XKAY,432
64
- unrealon_driver/browser/__init__.py,sha256=Rq0YuvRlrlSCy_uOQPVSaT2cHc5OE2ivc2gXnFKP2SU,152
65
- unrealon_driver/browser/config.py,sha256=z2U6DP5Gv5D0pZH2lB2I0pYL4dCwN4M-pjLNgUk3fHY,3079
66
- unrealon_driver/browser/manager.py,sha256=OijjVNWwJ0oyHY93tmd9vuqhvdAZnx0_URp9R53Q1bg,14248
67
- unrealon_driver/parser/__init__.py,sha256=L-rsthz3S_GEd_glNvdHazYJotcRfRPwm-7Bin_LCzM,1352
68
- unrealon_driver/parser/cli_manager.py,sha256=uSQLRwgogbPq4KBTHC1Q3ToGGDMKqezNWlpvO273_Zw,5102
69
- unrealon_driver/parser/daemon_manager.py,sha256=RYqQE1Dpbkh_0bBAtbdUqPbQuwahpERi_QkibZoqVLo,8698
70
- unrealon_driver/parser/parser_manager.py,sha256=HVobmUpkVwHpWH1xltyazYBCKW2haUsDfoOg9dLrvw4,21985
71
- unrealon_driver/parser/managers/__init__.py,sha256=fmOu0JMWmCcmAKvDTE77bTKzWuu-Wihtg029yrg02Pw,1120
72
- unrealon_driver/parser/managers/browser.py,sha256=JLsBjusNBH-37e7xwY0YP6dXEHKynVBRaUj97BYfUeY,1642
73
- unrealon_driver/parser/managers/config.py,sha256=-_juYfH0Br3q1Y8xdlU1vbJqRyzhU-Fx4sZsRzNwTBY,9424
74
- unrealon_driver/parser/managers/error.py,sha256=EnJkZLlZihXeKZdHauvnqaSby05caxHOYYIuiLoelkw,14555
75
- unrealon_driver/parser/managers/html.py,sha256=dLToyv_KJ-LXvsB2Drmyc4bIbrMDsj9opD7iGGqspHo,26272
76
- unrealon_driver/parser/managers/logging.py,sha256=DjiJdI1hhIesi0p04BNCekN7bh_nOxPZKYqyG9td2Xs,21663
77
- unrealon_driver/parser/managers/result.py,sha256=yIoDTx6e1YzhKmJ2yJPD8_eAAkrjtm7ofN3DFEOLdUU,10236
78
- unrealon-1.1.1.dist-info/METADATA,sha256=El88IEjD0e1fAUQCKeIXrYmvRwxo0-if4y82dYAXV5E,20865
79
- unrealon-1.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
80
- unrealon-1.1.1.dist-info/entry_points.txt,sha256=425kGFy7wjMYuNhyHRGKNciSQTVSw2IsHKJRRIHc5jo,431
81
- unrealon-1.1.1.dist-info/licenses/LICENSE,sha256=uTZpktXKUsE0IzS5RdSV398HHI74ssbGKTdCbv7U9l0,1070
82
- unrealon-1.1.1.dist-info/RECORD,,
@@ -1,114 +0,0 @@
1
- """
2
- Bridge Parsers - Parser-specific wrapper over generic bridge.
3
-
4
- Extends the generic WebSocket bridge with parser orchestration capabilities:
5
- - Parser registration and management
6
- - Command execution and tracking
7
- - Proxy allocation and management
8
- - Parser session logging
9
- - Typed data models for all parser operations
10
-
11
- This is a wrapper that adds parser domain knowledge to the generic bridge.
12
- """
13
-
14
- from importlib.metadata import version
15
-
16
- try:
17
- __version__ = version("unrealon")
18
- except Exception:
19
- __version__ = "1.1.1"
20
-
21
- from .models import (
22
- # Base models
23
- BaseParserModel,
24
- # Core models
25
- ParserInfo,
26
- ParserCommand,
27
- CommandResult,
28
- ProxyRequest,
29
- ProxyInfo,
30
- ParserSession,
31
- ParserEvent,
32
- ParserStats,
33
- ParserHealth,
34
- WebhookConfig,
35
- # RPC Response models
36
- BaseRPCResponse,
37
- ParserRegisterResponse,
38
- ParserStatusResponse,
39
- ParserListResponse,
40
- ParserHealthResponse,
41
- SessionStartResponse,
42
- SessionEndResponse,
43
- CommandExecuteResponse,
44
- CommandCreateResponse,
45
- CommandStatusResponse,
46
- ProxyAllocateResponse,
47
- ProxyReleaseResponse,
48
- ProxyCheckResponse,
49
- # RPC Request models
50
- BaseRPCRequest,
51
- ParserRegisterRequest,
52
- ParserStatusRequest,
53
- ParserListRequest,
54
- ParserHealthRequest,
55
- SessionStartRequest,
56
- SessionEndRequest,
57
- CommandExecuteRequest,
58
- CommandCreateRequest,
59
- CommandStatusRequest,
60
- ProxyAllocateRequest,
61
- ProxyReleaseRequest,
62
- ProxyCheckRequest,
63
- )
64
-
65
- from .client import ParserBridgeClient
66
- from .server import ParserBridgeServer
67
-
68
-
69
- __all__ = [
70
- # Base Models
71
- "BaseParserModel",
72
- # Core Models
73
- "ParserInfo",
74
- "ParserCommand",
75
- "CommandResult",
76
- "ProxyRequest",
77
- "ProxyInfo",
78
- "ParserSession",
79
- "ParserEvent",
80
- "ParserStats",
81
- "ParserHealth",
82
- "WebhookConfig",
83
- # RPC Response Models
84
- "BaseRPCResponse",
85
- "ParserRegisterResponse",
86
- "ParserStatusResponse",
87
- "ParserListResponse",
88
- "ParserHealthResponse",
89
- "SessionStartResponse",
90
- "SessionEndResponse",
91
- "CommandExecuteResponse",
92
- "CommandCreateResponse",
93
- "CommandStatusResponse",
94
- "ProxyAllocateResponse",
95
- "ProxyReleaseResponse",
96
- "ProxyCheckResponse",
97
- # RPC Request Models
98
- "BaseRPCRequest",
99
- "ParserRegisterRequest",
100
- "ParserStatusRequest",
101
- "ParserListRequest",
102
- "ParserHealthRequest",
103
- "SessionStartRequest",
104
- "SessionEndRequest",
105
- "CommandExecuteRequest",
106
- "CommandCreateRequest",
107
- "CommandStatusRequest",
108
- "ProxyAllocateRequest",
109
- "ProxyReleaseRequest",
110
- "ProxyCheckRequest",
111
- # Client/Server
112
- "ParserBridgeClient",
113
- "ParserBridgeServer",
114
- ]
unrealon_bridge/cli.py DELETED
@@ -1,316 +0,0 @@
1
- """
2
- CLI commands for unrealon_bridge.
3
-
4
- Provides command-line interfaces for running parser bridge servers.
5
- """
6
-
7
- import asyncio
8
- import sys
9
- import os
10
- import time
11
- from pathlib import Path
12
- from typing import Optional
13
- from watchdog.observers import Observer
14
- from watchdog.events import FileSystemEventHandler
15
- import websockets
16
- import click
17
-
18
- from unrealon_bridge import ParserBridgeServer
19
- from unrealon_bridge.configs import load_bridge_config, create_sample_config, save_bridge_config
20
- from unrealon_rpc.logging import get_logger, setup_logging_with_clear, clear_specific_log_file, resolve_log_path
21
- from unrealon_rpc.logging.models import LogConfig
22
-
23
- logger = get_logger(__name__)
24
-
25
-
26
- async def run_parser_bridge_server_async(
27
- redis_url: str = "redis://localhost:6379/0",
28
- rpc_channel: str = "parser_rpc",
29
- pubsub_prefix: str = "parser",
30
- websocket_host: str = "localhost",
31
- websocket_port: int = 8001,
32
- clear_logs: bool = True
33
- ) -> None:
34
- """
35
- Run Parser Bridge Server asynchronously.
36
-
37
- Args:
38
- redis_url: Redis connection URL
39
- rpc_channel: RPC channel name
40
- pubsub_prefix: PubSub channel prefix
41
- websocket_host: WebSocket host
42
- websocket_port: WebSocket port
43
- """
44
- # Setup logging with file output and clear only this server's log
45
- log_file_path = resolve_log_path("logs/parser_bridge.log")
46
- log_config = LogConfig(
47
- file_enabled=True,
48
- file_path=log_file_path,
49
- log_level="INFO"
50
- )
51
- if clear_logs:
52
- clear_specific_log_file(log_file_path)
53
- setup_logging_with_clear(log_config, clear_logs=False)
54
-
55
- logger.info("Starting Parser Bridge Server")
56
- logger.info(f"Redis URL: {redis_url}")
57
- logger.info(f"RPC Channel: {rpc_channel}")
58
- logger.info(f"PubSub Prefix: {pubsub_prefix}")
59
- logger.info(f"WebSocket: {websocket_host}:{websocket_port}")
60
-
61
- server = ParserBridgeServer(
62
- redis_url=redis_url,
63
- rpc_channel=rpc_channel,
64
- pubsub_prefix=pubsub_prefix
65
- )
66
-
67
- # Register example command handlers
68
- async def example_scrape_handler(command):
69
- """Example handler for scrape commands."""
70
- logger.info(f"Processing scrape command: {command.command_id}")
71
- return {
72
- "success": "true",
73
- "message": f"Scrape completed for command {command.command_id}",
74
- "items_found": "42",
75
- "timestamp": command.created_at.isoformat()
76
- }
77
-
78
- async def example_parse_handler(command):
79
- """Example handler for parse commands."""
80
- logger.info(f"Processing parse command: {command.command_id}")
81
- return {
82
- "success": "true",
83
- "message": f"Parse completed for command {command.command_id}",
84
- "parsed_items": "15",
85
- "timestamp": command.created_at.isoformat()
86
- }
87
-
88
- async def health_handler(command):
89
- """Handler for health commands."""
90
- logger.info(f"Processing health command: {command.command_id} for parser {command.parser_id}")
91
- return {
92
- "command_type": "health",
93
- "status": "healthy",
94
- "parser_ready": "true",
95
- "bridge_connected": "true",
96
- "timestamp": command.created_at.isoformat()
97
- }
98
-
99
- async def status_handler(command):
100
- """Handler for status commands."""
101
- logger.info(f"Processing status command: {command.command_id} for parser {command.parser_id}")
102
- return {
103
- "command_type": "status",
104
- "running": "true",
105
- "uptime_seconds": "60",
106
- "total_runs": "0",
107
- "successful_runs": "0",
108
- "failed_runs": "0",
109
- "timestamp": command.created_at.isoformat()
110
- }
111
-
112
- # Register handlers
113
- server.register_command_handler("scrape", example_scrape_handler)
114
- server.register_command_handler("parse", example_parse_handler)
115
- server.register_command_handler("test_command", example_scrape_handler)
116
- server.register_command_handler("benchmark_command", example_scrape_handler)
117
- server.register_command_handler("throughput_test", example_parse_handler)
118
- server.register_command_handler("concurrent_test", example_scrape_handler)
119
- server.register_command_handler("health", health_handler)
120
- server.register_command_handler("status", status_handler)
121
-
122
- # Start server
123
- await server.start()
124
- logger.info("Parser Bridge Server started successfully")
125
-
126
- # WebSocket handler
127
- async def websocket_handler(websocket):
128
- """Handle WebSocket connections."""
129
- logger.info(f"New WebSocket connection from {websocket.remote_address}")
130
- try:
131
- await server.bridge.handle_websocket(websocket)
132
- except Exception as e:
133
- logger.error(f"WebSocket connection error: {e}")
134
- finally:
135
- logger.info("WebSocket connection closed")
136
-
137
- try:
138
- logger.info(f"Starting WebSocket server on {websocket_host}:{websocket_port}")
139
-
140
- # Start WebSocket server
141
- async with websockets.serve(websocket_handler, websocket_host, websocket_port):
142
- logger.info(f"WebSocket server started on {websocket_host}:{websocket_port}")
143
-
144
- # Keep server running
145
- while True:
146
- await asyncio.sleep(1)
147
-
148
- except KeyboardInterrupt:
149
- logger.info("Received shutdown signal")
150
- except Exception as e:
151
- logger.error(f"Server error: {e}")
152
- raise
153
- finally:
154
- logger.info("Stopping Parser Bridge Server...")
155
- await server.stop()
156
- logger.info("Parser Bridge Server stopped")
157
-
158
-
159
- @click.command()
160
- @click.option("--redis-url", default="redis://localhost:6379/0", help="Redis connection URL")
161
- @click.option("--rpc-channel", default="parser_rpc", help="RPC channel name")
162
- @click.option("--pubsub-prefix", default="parser", help="PubSub channel prefix")
163
- @click.option("--websocket-host", default="localhost", help="WebSocket host")
164
- @click.option("--websocket-port", type=int, default=8001, help="WebSocket port")
165
- def run_parser_bridge_server(redis_url: str, rpc_channel: str, pubsub_prefix: str, websocket_host: str, websocket_port: int) -> None:
166
- """Run unrealon_bridge Parser Bridge Server."""
167
- try:
168
- asyncio.run(run_parser_bridge_server_async(
169
- redis_url=redis_url,
170
- rpc_channel=rpc_channel,
171
- pubsub_prefix=pubsub_prefix,
172
- websocket_host=websocket_host,
173
- websocket_port=websocket_port
174
- ))
175
- except KeyboardInterrupt:
176
- logger.info("Server shutdown completed")
177
- sys.exit(0)
178
- except Exception as e:
179
- logger.error(f"Failed to start server: {e}")
180
- sys.exit(1)
181
-
182
-
183
- def _run_parser_bridge_server_with_reload_impl(
184
- redis_url: str = "redis://localhost:6379/0",
185
- rpc_channel: str = "parser_rpc",
186
- pubsub_prefix: str = "parser",
187
- websocket_host: str = "localhost",
188
- websocket_port: int = 8002,
189
- watch_dirs: Optional[list] = None
190
- ) -> None:
191
- """
192
- Run Parser Bridge Server with auto-reload on file changes.
193
- """
194
-
195
- if watch_dirs is None:
196
- # Watch current directory and src directory
197
- current_dir = Path.cwd()
198
- src_dir = current_dir / "src"
199
- watch_dirs = [str(current_dir), str(src_dir)] if src_dir.exists() else [str(current_dir)]
200
-
201
- class ReloadHandler(FileSystemEventHandler):
202
- def __init__(self):
203
- self.last_reload = 0
204
- self.reload_delay = 1.0 # Delay to avoid multiple reloads
205
-
206
- def on_modified(self, event):
207
- if event.is_directory:
208
- return
209
-
210
- # Only reload for Python files
211
- if not event.src_path.endswith('.py'):
212
- return
213
-
214
- # Avoid too frequent reloads
215
- current_time = time.time()
216
- if current_time - self.last_reload < self.reload_delay:
217
- return
218
-
219
- self.last_reload = current_time
220
- logger.info(f"File changed: {event.src_path}")
221
- logger.info("Restarting Parser Bridge Server...")
222
-
223
- # Restart the process
224
- os.execv(sys.executable, [sys.executable] + sys.argv)
225
-
226
- # Setup file watcher
227
- event_handler = ReloadHandler()
228
- observer = Observer()
229
-
230
- for watch_dir in watch_dirs:
231
- if os.path.exists(watch_dir):
232
- observer.schedule(event_handler, watch_dir, recursive=True)
233
- logger.info(f"Watching directory for changes: {watch_dir}")
234
-
235
- observer.start()
236
-
237
- try:
238
- logger.info("🔄 Auto-reload enabled - server will restart on file changes")
239
- # Don't clear logs on auto-reload, only on first start
240
- clear_logs_on_start = not os.environ.get('PARSER_BRIDGE_RELOADING', False)
241
- os.environ['PARSER_BRIDGE_RELOADING'] = 'true'
242
- asyncio.run(run_parser_bridge_server_async(
243
- redis_url, rpc_channel, pubsub_prefix, websocket_host, websocket_port, clear_logs_on_start
244
- ))
245
- except KeyboardInterrupt:
246
- logger.info("Server shutdown completed")
247
- finally:
248
- observer.stop()
249
- observer.join()
250
-
251
-
252
- @click.command()
253
- @click.option("--redis-url", default="redis://localhost:6379/0", help="Redis connection URL")
254
- @click.option("--rpc-channel", default="parser_rpc", help="RPC channel name")
255
- @click.option("--pubsub-prefix", default="parser", help="PubSub channel prefix")
256
- @click.option("--websocket-host", default="localhost", help="WebSocket host")
257
- @click.option("--websocket-port", type=int, default=8002, help="WebSocket port (dev mode)")
258
- @click.option("--watch-dirs", multiple=True, help="Additional directories to watch for changes")
259
- def run_parser_bridge_server_with_reload(redis_url: str, rpc_channel: str, pubsub_prefix: str, websocket_host: str, websocket_port: int, watch_dirs: tuple) -> None:
260
- """Run unrealon_bridge Parser Bridge Server with auto-reload (development mode)."""
261
- try:
262
- _run_parser_bridge_server_with_reload_impl(
263
- redis_url=redis_url,
264
- rpc_channel=rpc_channel,
265
- pubsub_prefix=pubsub_prefix,
266
- websocket_host=websocket_host,
267
- websocket_port=websocket_port,
268
- watch_dirs=list(watch_dirs) if watch_dirs else None
269
- )
270
- except Exception as e:
271
- logger.error(f"Failed to start server: {e}")
272
- sys.exit(1)
273
-
274
-
275
- @click.command()
276
- @click.option('--output', '-o', default='bridge_config.yaml', help='Output config file path')
277
- @click.option('--force', is_flag=True, help='Overwrite existing config file')
278
- def create_config(output: str, force: bool):
279
- """Create a sample bridge configuration file."""
280
- config_path = Path(output)
281
-
282
- if config_path.exists() and not force:
283
- click.echo(f"❌ Config file already exists: {config_path}")
284
- click.echo("Use --force to overwrite")
285
- sys.exit(1)
286
-
287
- try:
288
- # Create sample configuration
289
- config = create_sample_config()
290
- saved_path = save_bridge_config(config, config_path)
291
-
292
- click.echo(f"✅ Created sample config: {saved_path}")
293
- click.echo("\n📋 Configuration includes:")
294
- click.echo(" • Test API keys for debugging")
295
- click.echo(" • Development environment settings")
296
- click.echo(" • WebSocket on port 8002")
297
- click.echo(" • Redis connection settings")
298
- click.echo("\n🔧 Edit the config file to customize settings")
299
-
300
- except Exception as e:
301
- click.echo(f"❌ Failed to create config: {e}")
302
- sys.exit(1)
303
-
304
-
305
- @click.group()
306
- def cli():
307
- """Unrealon Bridge CLI tools."""
308
- pass
309
-
310
-
311
- cli.add_command(run_parser_bridge_server, name="server")
312
- cli.add_command(create_config, name="create-config")
313
-
314
-
315
- if __name__ == "__main__":
316
- cli()
@@ -1,93 +0,0 @@
1
- """
2
- Parser Bridge Client - Modular implementation with composition.
3
-
4
- Clean architecture without multiple inheritance or hasattr checks.
5
- """
6
-
7
- from typing import Optional, List
8
- from unrealon_rpc.logging import get_logger
9
-
10
- from .base import ParserBridgeClientBase
11
- from .connection import ConnectionMixin
12
- from .session import SessionMixin
13
- from .commands import CommandsMixin
14
- from .proxy import ProxyMixin
15
- from .events import EventsMixin
16
- from .health import HealthMixin
17
- from .logging import LoggingMixin
18
- from .html_parser import HTMLParserMixin
19
- from .scheduler import SchedulerMixin
20
-
21
- from ..models import (
22
- ParserInfo, ParserHealth, ParserStats, CommandResult, ProxyInfo
23
- )
24
-
25
- logger = get_logger(__name__)
26
-
27
-
28
- class ParserBridgeClient(
29
- ParserBridgeClientBase,
30
- ConnectionMixin,
31
- SessionMixin,
32
- CommandsMixin,
33
- ProxyMixin,
34
- EventsMixin,
35
- HealthMixin,
36
- LoggingMixin,
37
- HTMLParserMixin,
38
- SchedulerMixin
39
- ):
40
- """
41
- Complete Parser Bridge Client with all functionality.
42
-
43
- Combines all mixins to provide full parser client capabilities:
44
- - Connection and registration management
45
- - Session lifecycle
46
- - Command execution
47
- - Proxy management
48
- - Event logging and heartbeat
49
- - Health monitoring
50
- - Parser logging to Django
51
- - HTML parsing via AI/LLM
52
- - Task scheduling and management
53
- """
54
-
55
- def __init__(self, websocket_url: str, parser_type: str, parser_version: str = "1.0.0", capabilities: List[str] = None, api_key: str = None, **kwargs):
56
- """
57
- Initialize complete parser bridge client.
58
-
59
- Args:
60
- websocket_url: WebSocket server URL
61
- parser_type: Type of parser (encar, autotrader, etc.)
62
- parser_version: Parser version
63
- capabilities: List of parser capabilities
64
- api_key: API key for authentication
65
- **kwargs: Additional arguments for BridgeClient
66
- """
67
- super().__init__(websocket_url, parser_type, parser_version, capabilities, api_key, **kwargs)
68
-
69
- async def disconnect(self) -> None:
70
- """Enhanced disconnect with proper session cleanup."""
71
- if self.session_id:
72
- await self.end_session()
73
-
74
- # Call parent disconnect
75
- await super().disconnect()
76
-
77
- async def _log_command_event(self, event_type: str, message: str, level: str = "info", data: Optional[dict[str, str]] = None) -> None:
78
- """Override to provide actual event logging."""
79
- await self.log_event(event_type=event_type, message=message, level=level, data=data)
80
-
81
- def set_command_handler(self, handler) -> None:
82
- """
83
- Set command handler for daemon compatibility.
84
-
85
- Args:
86
- handler: Command handler function
87
- """
88
- # For now, just store the handler - can be extended later for actual command processing
89
- self._command_handler = handler
90
- logger.info("Command handler set for daemon compatibility")
91
-
92
-
93
- __all__ = ["ParserBridgeClient"]
@@ -1,78 +0,0 @@
1
- """
2
- Base Parser Bridge Client.
3
-
4
- Core client functionality and state management.
5
- """
6
-
7
- from typing import Optional, List
8
- from unrealon_rpc.bridge import BridgeClient
9
- from unrealon_rpc.logging import get_logger
10
-
11
- logger = get_logger(__name__)
12
-
13
-
14
- class ParserBridgeClientBase:
15
- """
16
- Base parser bridge client with core functionality.
17
-
18
- Manages connection state and provides foundation for specialized components.
19
- """
20
-
21
- def __init__(self, websocket_url: str, parser_type: str, parser_version: str = "1.0.0", capabilities: List[str] = None, api_key: str = None, **kwargs):
22
- """
23
- Initialize parser bridge client.
24
-
25
- Args:
26
- websocket_url: WebSocket server URL
27
- parser_type: Type of parser (encar, autotrader, etc.)
28
- parser_version: Parser version
29
- capabilities: List of parser capabilities
30
- api_key: API key for authentication
31
- **kwargs: Additional arguments for BridgeClient
32
- """
33
- self.parser_type = parser_type
34
- self.parser_version = parser_version
35
- self.capabilities = capabilities or []
36
- self.api_key = api_key
37
-
38
- # Initialize generic bridge client
39
- self.bridge_client = BridgeClient(
40
- websocket_url=websocket_url,
41
- client_type=f"parser_{parser_type}",
42
- client_version=parser_version,
43
- api_key=api_key,
44
- **kwargs
45
- )
46
-
47
- # Parser state
48
- self.parser_id: Optional[str] = None
49
- self.session_id: Optional[str] = None
50
- self.registered = False
51
-
52
- @property
53
- def is_connected(self) -> bool:
54
- """Check if client is connected."""
55
- return self.bridge_client.is_connected
56
-
57
- @property
58
- def is_registered(self) -> bool:
59
- """Check if parser is registered."""
60
- return self.registered
61
-
62
- async def send_message(self, message: dict) -> None:
63
- """Send message through WebSocket bridge."""
64
- await self.bridge_client._send_message(message)
65
-
66
- def add_message_handler(self, message_type: str, handler) -> None:
67
- """Add message handler for specific message type."""
68
- self.bridge_client.message_handlers[message_type] = handler
69
-
70
- def _ensure_registered(self) -> None:
71
- """Ensure parser is registered, raise error if not."""
72
- if not self.registered:
73
- raise RuntimeError("Parser not registered")
74
-
75
- def _ensure_connected(self) -> None:
76
- """Ensure client is connected, raise error if not."""
77
- if not self.is_connected:
78
- raise RuntimeError("Client not connected")