zenx 0.9.3__py3-none-any.whl → 0.9.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenx/pipelines/discord.py +55 -35
- {zenx-0.9.3.dist-info → zenx-0.9.4.dist-info}/METADATA +6 -4
- {zenx-0.9.3.dist-info → zenx-0.9.4.dist-info}/RECORD +6 -6
- {zenx-0.9.3.dist-info → zenx-0.9.4.dist-info}/WHEEL +0 -0
- {zenx-0.9.3.dist-info → zenx-0.9.4.dist-info}/entry_points.txt +0 -0
- {zenx-0.9.3.dist-info → zenx-0.9.4.dist-info}/top_level.txt +0 -0
zenx/pipelines/discord.py
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
from httpx import AsyncClient
|
2
1
|
from typing import Dict
|
3
2
|
from structlog import BoundLogger
|
4
3
|
|
@@ -8,38 +7,59 @@ from zenx.settings import Settings
|
|
8
7
|
from zenx.utils import log_processing_time
|
9
8
|
|
10
9
|
|
10
|
+
try:
|
11
|
+
from httpx import AsyncClient
|
11
12
|
|
12
|
-
class SynopticDiscordPipeline(Pipeline):
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
13
|
+
class SynopticDiscordPipeline(Pipeline): # type: ignore[reportRedeclaration]
|
14
|
+
name = "syncoptic_discord"
|
15
|
+
required_settings = ["SYNOPTIC_DISCORD_WEBHOOK"]
|
16
|
+
|
17
|
+
|
18
|
+
def __init__(self, logger: BoundLogger, db: DBClient, settings: Settings) -> None:
|
19
|
+
super().__init__(logger, db, settings)
|
20
|
+
self._uri = settings.SYNOPTIC_DISCORD_WEBHOOK
|
21
|
+
self._client = AsyncClient(headers={"Content-Type": "application/json"})
|
22
|
+
|
23
|
+
|
24
|
+
async def start(self) -> None:
|
25
|
+
for setting in self.required_settings:
|
26
|
+
if not getattr(self.settings, setting):
|
27
|
+
raise ValueError(f"Missing required setting: {setting}")
|
28
|
+
|
29
|
+
|
30
|
+
@log_processing_time
|
31
|
+
async def process_item(self, item: Dict, spider: str) -> Dict:
|
32
|
+
await self._process(item)
|
33
|
+
return item
|
34
|
+
|
35
|
+
|
36
|
+
async def _process(self, item: Dict) -> None:
|
37
|
+
try:
|
38
|
+
_item = {k: v for k, v in item.items() if not k.startswith("_")}
|
39
|
+
await self._client.post(self._uri, json=_item)
|
40
|
+
except Exception as e:
|
41
|
+
self.logger.error("processing", exception=str(e), id=item.get("_id"), pipeline=self.name)
|
42
|
+
|
43
|
+
|
44
|
+
async def close(self) -> None:
|
45
|
+
if hasattr(self, "_client") and self._client:
|
46
|
+
await self._client.aclose()
|
47
|
+
except ModuleNotFoundError:
|
48
|
+
# proxy pattern
|
49
|
+
class SynopticDiscordPipeline(Pipeline):
|
50
|
+
name = "synoptic_discord"
|
51
|
+
required_settings = []
|
52
|
+
|
53
|
+
_ERROR_MESSAGE = (
|
54
|
+
f"The '{name}' pipeline is disabled because the required dependencies are not installed. "
|
55
|
+
"Please install it to enable this feature:\n\n"
|
56
|
+
" pip install 'zenx[discord]'"
|
57
|
+
)
|
58
|
+
|
59
|
+
def __init__(self, *args, **kwargs):
|
60
|
+
super().__init__(*args, **kwargs)
|
61
|
+
raise ImportError(self._ERROR_MESSAGE)
|
62
|
+
|
63
|
+
async def start(self) -> None: pass
|
64
|
+
async def process_item(self, item: Dict, spider: str) -> Dict: return {}
|
65
|
+
async def close(self) -> None: pass
|
@@ -1,10 +1,9 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: zenx
|
3
|
-
Version: 0.9.
|
3
|
+
Version: 0.9.4
|
4
4
|
Summary: mini-framework
|
5
5
|
Requires-Python: >=3.12
|
6
6
|
Requires-Dist: curl-cffi>=0.12.0
|
7
|
-
Requires-Dist: httpx>=0.28.1
|
8
7
|
Requires-Dist: parsel>=1.10.0
|
9
8
|
Requires-Dist: pebble>=5.1.1
|
10
9
|
Requires-Dist: pydantic>=2.11.7
|
@@ -18,6 +17,10 @@ Requires-Dist: redis[hiredis]>=6.2.0; extra == "redis"
|
|
18
17
|
Provides-Extra: grpc
|
19
18
|
Requires-Dist: grpcio>=1.73.1; extra == "grpc"
|
20
19
|
Requires-Dist: protobuf>=6.31.1; extra == "grpc"
|
20
|
+
Provides-Extra: websocket
|
21
|
+
Requires-Dist: websockets>=15.0.1; extra == "websocket"
|
22
|
+
Provides-Extra: discord
|
23
|
+
Requires-Dist: httpx>=0.28.1; extra == "discord"
|
21
24
|
Provides-Extra: all
|
22
25
|
Requires-Dist: curl-cffi>=0.12.0; extra == "all"
|
23
26
|
Requires-Dist: grpcio>=1.73.1; extra == "all"
|
@@ -31,5 +34,4 @@ Requires-Dist: typer>=0.16.0; extra == "all"
|
|
31
34
|
Requires-Dist: uvloop>=0.21.0; extra == "all"
|
32
35
|
Requires-Dist: redis[hiredis]>=6.2.0; extra == "all"
|
33
36
|
Requires-Dist: websockets>=15.0.1; extra == "all"
|
34
|
-
|
35
|
-
Requires-Dist: websockets>=15.0.1; extra == "websocket"
|
37
|
+
Requires-Dist: httpx>=0.28.1; extra == "all"
|
@@ -11,7 +11,7 @@ zenx/clients/database.py,sha256=AF-L7iYrWRNzUZKn7taveiihpu--mXXC6eWOrMNlqzQ,4806
|
|
11
11
|
zenx/clients/http.py,sha256=C2VLpxqvlRcKTbESDGXqjSi7LNrdt8Jr9pogXwrsQ8o,6252
|
12
12
|
zenx/pipelines/__init__.py,sha256=IxkZ0UpEJdYjLdd-PMcC9PzzzArTBNNcpgKc7NiOe5Y,131
|
13
13
|
zenx/pipelines/base.py,sha256=N_388z5DFMeaU6wMwcClZAbQFWKh4kpAF7eUJhpQevs,1863
|
14
|
-
zenx/pipelines/discord.py,sha256=
|
14
|
+
zenx/pipelines/discord.py,sha256=VCLWaDQveiRj2f9q8gOXv8hr7SESxv23L8mDvrJXRWk,2378
|
15
15
|
zenx/pipelines/google_rpc.py,sha256=nR7hNCqAcg1YGWYm3xoj8G8vOTs74lAeYc0dYk842n8,4490
|
16
16
|
zenx/pipelines/manager.py,sha256=lm2A_4h8NNdurFfwrrLwx5Z1tqJ9yZp2qgYvwGWd1lc,2017
|
17
17
|
zenx/pipelines/preprocess.py,sha256=PTXDYo-B_T3YAY5AeUx5ExTDKZovQUwqmB8pyHo6i6o,958
|
@@ -21,8 +21,8 @@ zenx/resources/proto/feed_pb2.py,sha256=ZyICOLnyuXekkvV4bAHZ1nE1-wwzcYYRRrmRJCMr
|
|
21
21
|
zenx/resources/proto/feed_pb2_grpc.py,sha256=Mim6FfBgIMj0PmTqHk036nVUMJH3A6I3ts6r1j3bQF8,7441
|
22
22
|
zenx/spiders/__init__.py,sha256=rs5LuqdM2MQlUYiTGJrzkYhzN8_SSLTrR7wGjSRrrSo,25
|
23
23
|
zenx/spiders/base.py,sha256=MeZ3wZOPOyOX4V2ufFXtYGCDtXHZO_mNfnXdKMkisuQ,1951
|
24
|
-
zenx-0.9.
|
25
|
-
zenx-0.9.
|
26
|
-
zenx-0.9.
|
27
|
-
zenx-0.9.
|
28
|
-
zenx-0.9.
|
24
|
+
zenx-0.9.4.dist-info/METADATA,sha256=vRckAJxLnSnXaQ9nf5_2_3SYlHImPkk_mT7j7kpXOOk,1391
|
25
|
+
zenx-0.9.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
26
|
+
zenx-0.9.4.dist-info/entry_points.txt,sha256=8JXob2f1VtvzGFris-e9Usqywg7oca-cChDlH9moOZU,38
|
27
|
+
zenx-0.9.4.dist-info/top_level.txt,sha256=JeXwvK86d7sB-2x-avugFnZIZa33zaHWKI8RHWJR6KY,5
|
28
|
+
zenx-0.9.4.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|