aioscrapper 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aioscrapper/scrapper/executor.py +1 -3
- {aioscrapper-0.1.1.dist-info → aioscrapper-0.1.2.dist-info}/METADATA +16 -6
- {aioscrapper-0.1.1.dist-info → aioscrapper-0.1.2.dist-info}/RECORD +6 -6
- {aioscrapper-0.1.1.dist-info → aioscrapper-0.1.2.dist-info}/WHEEL +1 -1
- {aioscrapper-0.1.1.dist-info → aioscrapper-0.1.2.dist-info}/licenses/LICENSE +0 -0
- {aioscrapper-0.1.1.dist-info → aioscrapper-0.1.2.dist-info}/top_level.txt +0 -0
aioscrapper/scrapper/executor.py
CHANGED
|
@@ -32,9 +32,7 @@ class AIOScrapper:
|
|
|
32
32
|
self._response_middlewares = []
|
|
33
33
|
|
|
34
34
|
self._pipelines: dict[str, list[BasePipeline]] = {}
|
|
35
|
-
self._pipeline_dispatcher = PipelineDispatcher(
|
|
36
|
-
logger=self._logger.getChild("pipeline"), pipelines=self._pipelines
|
|
37
|
-
)
|
|
35
|
+
self._pipeline_dispatcher = PipelineDispatcher(self._logger.getChild("pipeline"), pipelines=self._pipelines)
|
|
38
36
|
|
|
39
37
|
def _exception_handler(_, context: dict[str, Any]):
|
|
40
38
|
if "job" in context:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: aioscrapper
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Async framework for building modular and scalable web scrapers.
|
|
5
5
|
Author: darkstussy
|
|
6
6
|
Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
|
|
@@ -12,18 +12,28 @@ Classifier: Intended Audience :: Developers
|
|
|
12
12
|
Classifier: Operating System :: OS Independent
|
|
13
13
|
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
14
14
|
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
15
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
License-File: LICENSE
|
|
18
18
|
Requires-Dist: aiohttp[speedups]~=3.11.16
|
|
19
19
|
Requires-Dist: aiojobs~=1.4.0
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: flake8~=7.1.2; extra == "dev"
|
|
22
|
+
Requires-Dist: black~=25.1.0; extra == "dev"
|
|
23
|
+
Requires-Dist: pyright~=1.1.399; extra == "dev"
|
|
24
|
+
Requires-Dist: aiohttp[speedups]~=3.11.16; extra == "dev"
|
|
25
|
+
Requires-Dist: aiojobs~=1.4.0; extra == "dev"
|
|
26
|
+
Provides-Extra: test
|
|
27
|
+
Requires-Dist: pytest~=8.3.5; extra == "test"
|
|
28
|
+
Requires-Dist: pytest-asyncio~=0.26.0; extra == "test"
|
|
29
|
+
Requires-Dist: aresponses~=3.0.0; extra == "test"
|
|
20
30
|
Dynamic: license-file
|
|
21
31
|
|
|
22
32
|
# aioscrapper
|
|
23
33
|
|
|
24
34
|
**Asynchronous framework for building modular and scalable web scrapers.**
|
|
25
35
|
|
|
26
|
-
|
|
27
37
|
|
|
28
38
|
|
|
29
39
|
|
|
@@ -44,7 +54,7 @@ pip install aioscrapper
|
|
|
44
54
|
|
|
45
55
|
## Requirements
|
|
46
56
|
|
|
47
|
-
- Python 3.
|
|
57
|
+
- Python 3.10 or higher
|
|
48
58
|
- aiohttp
|
|
49
59
|
- aiojobs
|
|
50
60
|
|
|
@@ -53,8 +63,8 @@ pip install aioscrapper
|
|
|
53
63
|
```python
|
|
54
64
|
import asyncio
|
|
55
65
|
|
|
56
|
-
from aioscrapper import BaseScrapper, AIOScrapper
|
|
57
|
-
from aioscrapper.types import Response
|
|
66
|
+
from aioscrapper import BaseScrapper, AIOScrapper
|
|
67
|
+
from aioscrapper.types import Response, RequestSender
|
|
58
68
|
|
|
59
69
|
|
|
60
70
|
class Scrapper(BaseScrapper):
|
|
@@ -7,7 +7,7 @@ aioscrapper/pipeline/base.py,sha256=Ro7YGUOB-V2NJCtfgwhtQDedY4OYMu-jwEV8iR-L89k,
|
|
|
7
7
|
aioscrapper/pipeline/dispatcher.py,sha256=H4cHNxTyHEF4BnEwaW6nwmcRmK839GqbDTzZh1Zftv4,1156
|
|
8
8
|
aioscrapper/scrapper/__init__.py,sha256=UR7bTck-_YVoP2BqYdPldN9PgaCuJf9wvDdQLTVJ578,65
|
|
9
9
|
aioscrapper/scrapper/base.py,sha256=2_WeLMyJICLmIG7N9r6BGmBg0f-wjEQPsVY076WHKOI,241
|
|
10
|
-
aioscrapper/scrapper/executor.py,sha256=
|
|
10
|
+
aioscrapper/scrapper/executor.py,sha256=5j66FqVaZ5AWXMF-SHfTiqkORlTkY53rZDPO2pWLfww,5466
|
|
11
11
|
aioscrapper/scrapper/request_manager.py,sha256=xhF_feppHQognTbbHjVUC13V4NwJJt7bCWwcyznFK84,5831
|
|
12
12
|
aioscrapper/session/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
13
|
aioscrapper/session/aiohttp.py,sha256=8h4Ca1o2JJ7L24iqEnZ5I8bUudTn2cvTngBhM8eJPD4,1753
|
|
@@ -15,8 +15,8 @@ aioscrapper/session/base.py,sha256=Zxw1VHIe_LgveUufJXh_cl0YeHykiutQveMUwZ6VL54,3
|
|
|
15
15
|
aioscrapper/types/__init__.py,sha256=OK7vKZznJAWXQFBhXbQJha-XqGoRAE3lCaJUF7WXn64,210
|
|
16
16
|
aioscrapper/types/middleware.py,sha256=WtT73QTAlwhdP6UNFgyFHGpFOx1vlxehCAwiO6xjR10,326
|
|
17
17
|
aioscrapper/types/session.py,sha256=WppvDBZ0sBWVddzz7RXLkg8iZCfZipTdHpKuGW-U090,2970
|
|
18
|
-
aioscrapper-0.1.
|
|
19
|
-
aioscrapper-0.1.
|
|
20
|
-
aioscrapper-0.1.
|
|
21
|
-
aioscrapper-0.1.
|
|
22
|
-
aioscrapper-0.1.
|
|
18
|
+
aioscrapper-0.1.2.dist-info/licenses/LICENSE,sha256=EEeV20hghyroJWe2vcHjJma9PcjSkjD6vIwlUtaAjLE,1067
|
|
19
|
+
aioscrapper-0.1.2.dist-info/METADATA,sha256=CBYHtlAtL4dgEp7Y0RMdx4JKvAaLV3roEfHL8YQGRk8,2784
|
|
20
|
+
aioscrapper-0.1.2.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
|
|
21
|
+
aioscrapper-0.1.2.dist-info/top_level.txt,sha256=d7lbzXOwzzk2HLh-A0X7dkqn8q3zGAJcKqx6TkaEEWI,12
|
|
22
|
+
aioscrapper-0.1.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|