aioscrapper 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

aioscrapper/pipeline/__init__.py

@@ -1,2 +1 @@
-from .base import BasePipeline, BaseItem
-from .dispatcher import PipelineDispatcher
+from .base import BasePipeline

aioscrapper/pipeline/base.py

@@ -1,11 +1,7 @@
 import abc
-from typing import TypeVar, Generic, Protocol
-
-
-class BaseItem(Protocol):
-    @property
-    def pipeline_name(self) -> str: ...
+from typing import TypeVar, Generic
 
+from ..types import BaseItem
 
 ItemType = TypeVar("ItemType", bound=BaseItem)
 

aioscrapper/pipeline/dispatcher.py

@@ -10,7 +10,7 @@ class PipelineDispatcher:
         self._logger = logger
         self._pipelines = pipelines
 
-    async def put_item(self, item: BaseItem) -> BaseItem:
+    async def __call__(self, item: BaseItem) -> BaseItem:
         self._logger.debug(f"pipeline item received: {item}")
         try:
             pipelines = self._pipelines[item.pipeline_name]

aioscrapper/scrapper/base.py

@@ -3,8 +3,28 @@ import abc
 
 class BaseScrapper(abc.ABC):
     @abc.abstractmethod
-    async def start(self, *args, **kwargs) -> None: ...
+    async def start(self, *args, **kwargs) -> None:
+        """
+        Starts the scrapper.
 
-    async def initialize(self, *args, **kwargs) -> None: ...
+        This method is called to start the scraper by sending the initial requests required for its operation.
+        """
+        ...
 
-    async def close(self, *args, **kwargs) -> None: ...
+    async def initialize(self, *args, **kwargs) -> None:
+        """
+        Initializes the scrapper.
+
+        This method is called before starting the scrapper. It should be used to initialize any
+        necessary state or resources required by the scrapper.
+        """
+        ...
+
+    async def close(self, *args, **kwargs) -> None:
+        """
+        Closes the scrapper.
+
+        This method is called to clean up any resources created by the scrapper after it has finished
+        running.
+        """
+        ...

aioscrapper/scrapper/executor.py

@@ -9,7 +9,8 @@ from aiojobs import Scheduler
 from .request_manager import RequestManager
 from ..config import Config
 from ..helpers import get_func_kwargs
-from ..pipeline import PipelineDispatcher, BasePipeline
+from ..pipeline import BasePipeline
+from ..pipeline.dispatcher import PipelineDispatcher
 from ..scrapper import BaseScrapper
 from ..session.aiohttp import AiohttpSession
 from ..types import RequestMiddleware, ResponseMiddleware
@@ -32,9 +33,7 @@ class AIOScrapper:
         self._response_middlewares = []
 
         self._pipelines: dict[str, list[BasePipeline]] = {}
-        self._pipeline_dispatcher = PipelineDispatcher(
-            logger=self._logger.getChild("pipeline"), pipelines=self._pipelines
-        )
+        self._pipeline_dispatcher = PipelineDispatcher(self._logger.getChild("pipeline"), pipelines=self._pipelines)
 
         def _exception_handler(_, context: dict[str, Any]):
             if "job" in context:
@@ -96,7 +95,7 @@ class AIOScrapper:
         await self._pipeline_dispatcher.initialize()
         self._request_manager.listen_queue()
 
-        scrapper_kwargs = {"request_sender": self._request_manager.sender, "pipeline": self._pipeline_dispatcher}
+        scrapper_kwargs = {"send_request": self._request_manager.sender, "pipeline": self._pipeline_dispatcher}
         for scrapper in self._scrappers:
             await scrapper.initialize(**get_func_kwargs(scrapper.initialize, scrapper_kwargs))
 

aioscrapper/types/__init__.py

@@ -1,4 +1,5 @@
 from .middleware import RequestMiddleware, ResponseMiddleware
+from .pipeline import BaseItem, Pipeline
 from .session import (
     QueryParams,
     Cookies,

aioscrapper/types/pipeline.py

@@ -0,0 +1,10 @@
+from typing import Protocol
+
+
+class BaseItem(Protocol):
+    @property
+    def pipeline_name(self) -> str: ...
+
+
+class Pipeline(Protocol):
+    async def __call__(self, item: BaseItem) -> BaseItem: ...

{aioscrapper-0.1.1.dist-info → aioscrapper-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aioscrapper
-Version: 0.1.1
+Version: 0.2.0
 Summary: Async framework for building modular and scalable web scrapers.
 Author: darkstussy
 Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
@@ -12,18 +12,28 @@ Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp[speedups]~=3.11.16
 Requires-Dist: aiojobs~=1.4.0
+Provides-Extra: dev
+Requires-Dist: flake8~=7.1.2; extra == "dev"
+Requires-Dist: black~=25.1.0; extra == "dev"
+Requires-Dist: pyright~=1.1.399; extra == "dev"
+Requires-Dist: aiohttp[speedups]~=3.11.16; extra == "dev"
+Requires-Dist: aiojobs~=1.4.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest~=8.3.5; extra == "test"
+Requires-Dist: pytest-asyncio~=0.26.0; extra == "test"
+Requires-Dist: aresponses~=3.0.0; extra == "test"
 Dynamic: license-file
 
 # aioscrapper
 
 **Asynchronous framework for building modular and scalable web scrapers.**
 
-![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![Python](https://img.shields.io/badge/python-3.10%2B-blue)
 ![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
 ![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
 
@@ -44,7 +54,7 @@ pip install aioscrapper
 
 ## Requirements
 
-- Python 3.12 or higher
+- Python 3.10 or higher
 - aiohttp
 - aiojobs
 
@@ -53,13 +63,13 @@ pip install aioscrapper
 ```python
 import asyncio
 
-from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
-from aioscrapper.types import Response
+from aioscrapper import BaseScrapper, AIOScrapper
+from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):
-    async def start(self, request_sender: RequestSender) -> None:
-        await request_sender(url="https://example.com", callback=self.parse)
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://example.com", callback=self.parse)
 
     async def parse(self, response: Response) -> None:
         # handle response

{aioscrapper-0.1.1.dist-info → aioscrapper-0.2.0.dist-info}/RECORD

@@ -2,21 +2,22 @@ aioscrapper/__init__.py,sha256=Yl57BbmijQN_UgP5nRUiWYqgU8kXQ9kplzzzyMcsUMY,197
 aioscrapper/config.py,sha256=yO5ipQUHxA_-CiSqJ0u7WioN6lu8VgT1ss5PRvS1foc,844
 aioscrapper/exceptions.py,sha256=Akk3zDTgws9E7J-Sh8bgdlgS8L3auDKuv3_U3aefxMc,765
 aioscrapper/helpers.py,sha256=slq9r5oCHrR7M9hKZFBLFRsWoqJcw_QFptQI1NjIdQw,610
-aioscrapper/pipeline/__init__.py,sha256=hv7Kcssd2BP0LM9fNZtaMs1tmRuAUu4mwAescoeV3Uk,84
-aioscrapper/pipeline/base.py,sha256=Ro7YGUOB-V2NJCtfgwhtQDedY4OYMu-jwEV8iR-L89k,405
-aioscrapper/pipeline/dispatcher.py,sha256=H4cHNxTyHEF4BnEwaW6nwmcRmK839GqbDTzZh1Zftv4,1156
+aioscrapper/pipeline/__init__.py,sha256=SX4r3KYdApDdMc5uc7hP_KWEBZJfiawgfdKxgbORdv0,31
+aioscrapper/pipeline/base.py,sha256=HN1gpvS0J9zU3DMfW480QrkivFXzRR36SMvgTjZNzCQ,342
+aioscrapper/pipeline/dispatcher.py,sha256=_erY39J0qf4ZP0R1xcN3OgvsbL6EDAh6jbju34iSo-0,1156
 aioscrapper/scrapper/__init__.py,sha256=UR7bTck-_YVoP2BqYdPldN9PgaCuJf9wvDdQLTVJ578,65
-aioscrapper/scrapper/base.py,sha256=2_WeLMyJICLmIG7N9r6BGmBg0f-wjEQPsVY076WHKOI,241
-aioscrapper/scrapper/executor.py,sha256=Rz2dNmdFOvjUXM6-8GLGNTpZmlEkxD24ZK3qlnioSuQ,5495
+aioscrapper/scrapper/base.py,sha256=It3l_X4AzbgxeKfiAqlo-6RmAyYwBFUrYZDJYGYAK3A,821
+aioscrapper/scrapper/executor.py,sha256=-zUibR8gsQ11zkpbMwL04s6tuTD9qadIcUragPQv4G8,5497
 aioscrapper/scrapper/request_manager.py,sha256=xhF_feppHQognTbbHjVUC13V4NwJJt7bCWwcyznFK84,5831
 aioscrapper/session/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aioscrapper/session/aiohttp.py,sha256=8h4Ca1o2JJ7L24iqEnZ5I8bUudTn2cvTngBhM8eJPD4,1753
 aioscrapper/session/base.py,sha256=Zxw1VHIe_LgveUufJXh_cl0YeHykiutQveMUwZ6VL54,356
-aioscrapper/types/__init__.py,sha256=OK7vKZznJAWXQFBhXbQJha-XqGoRAE3lCaJUF7WXn64,210
+aioscrapper/types/__init__.py,sha256=6oaFkDXG57Ban9-46u7HiTFAg91KDb2_CFzfSikF7as,251
 aioscrapper/types/middleware.py,sha256=WtT73QTAlwhdP6UNFgyFHGpFOx1vlxehCAwiO6xjR10,326
+aioscrapper/types/pipeline.py,sha256=yoLTSZMaC6jFGKCrnXnlHEKlrMjHGpUPG90flNRfqbQ,200
 aioscrapper/types/session.py,sha256=WppvDBZ0sBWVddzz7RXLkg8iZCfZipTdHpKuGW-U090,2970
-aioscrapper-0.1.1.dist-info/licenses/LICENSE,sha256=EEeV20hghyroJWe2vcHjJma9PcjSkjD6vIwlUtaAjLE,1067
-aioscrapper-0.1.1.dist-info/METADATA,sha256=Q10bIQpw0JBM8oG3x5mP1pOFtUURYWcMOHvWLs-rVQQ,2350
-aioscrapper-0.1.1.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
-aioscrapper-0.1.1.dist-info/top_level.txt,sha256=d7lbzXOwzzk2HLh-A0X7dkqn8q3zGAJcKqx6TkaEEWI,12
-aioscrapper-0.1.1.dist-info/RECORD,,
+aioscrapper-0.2.0.dist-info/licenses/LICENSE,sha256=EEeV20hghyroJWe2vcHjJma9PcjSkjD6vIwlUtaAjLE,1067
+aioscrapper-0.2.0.dist-info/METADATA,sha256=ud9KV0MB_23bGV1v3aFF-XHGIjeNzgqsZ_TZElvCxtI,2780
+aioscrapper-0.2.0.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+aioscrapper-0.2.0.dist-info/top_level.txt,sha256=d7lbzXOwzzk2HLh-A0X7dkqn8q3zGAJcKqx6TkaEEWI,12
+aioscrapper-0.2.0.dist-info/RECORD,,

{aioscrapper-0.1.1.dist-info → aioscrapper-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.1)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any