zenx 0.7.7__tar.gz → 0.7.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zenx-0.7.7 → zenx-0.7.8}/PKG-INFO +1 -1
- {zenx-0.7.7 → zenx-0.7.8}/pyproject.toml +1 -1
- {zenx-0.7.7 → zenx-0.7.8}/zenx/spiders/base.py +10 -2
- {zenx-0.7.7 → zenx-0.7.8}/zenx.egg-info/PKG-INFO +1 -1
- {zenx-0.7.7 → zenx-0.7.8}/setup.cfg +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/cli.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/clients/__init__.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/clients/database.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/clients/http.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/debug_runner.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/discovery.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/engine.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/exceptions.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/logger.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/pipelines/__init__.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/pipelines/base.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/pipelines/google_rpc.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/pipelines/manager.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/pipelines/preprocess.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/pipelines/websocket.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/resources/proto/__init__.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/resources/proto/feed_pb2.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/resources/proto/feed_pb2_grpc.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/settings.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/spiders/__init__.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx/utils.py +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx.egg-info/SOURCES.txt +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx.egg-info/dependency_links.txt +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx.egg-info/entry_points.txt +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx.egg-info/requires.txt +0 -0
- {zenx-0.7.7 → zenx-0.7.8}/zenx.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
import asyncio
|
|
2
3
|
from abc import ABC, abstractmethod
|
|
3
|
-
from typing import ClassVar, Dict, List, Literal, Type
|
|
4
|
+
from typing import ClassVar, Coroutine, Dict, List, Literal, Type
|
|
4
5
|
from structlog import BoundLogger
|
|
5
6
|
|
|
6
7
|
from zenx.clients.http import HttpClient, Response
|
|
@@ -44,7 +45,14 @@ class Spider(ABC):
|
|
|
44
45
|
self.pm = pm
|
|
45
46
|
self.logger = logger
|
|
46
47
|
self.settings = settings
|
|
47
|
-
|
|
48
|
+
self.background_tasks = set()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def create_task(self, coro: Coroutine) -> None:
|
|
52
|
+
t = asyncio.create_task(coro)
|
|
53
|
+
self.background_tasks.add(t)
|
|
54
|
+
t.add_done_callback(self.background_tasks.discard)
|
|
55
|
+
|
|
48
56
|
|
|
49
57
|
@abstractmethod
|
|
50
58
|
async def crawl(self) -> None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|