aioscrapper 0.1.2__tar.gz → 0.2.0__tar.gz

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aioscrapper
-Version: 0.1.2
+Version: 0.2.0
 Summary: Async framework for building modular and scalable web scrapers.
 Author: darkstussy
 Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
@@ -68,8 +68,8 @@ from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):
-    async def start(self, request_sender: RequestSender) -> None:
-        await request_sender(url="https://example.com", callback=self.parse)
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://example.com", callback=self.parse)
 
     async def parse(self, response: Response) -> None:
         # handle response

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/README.md

@@ -37,8 +37,8 @@ from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):
-    async def start(self, request_sender: RequestSender) -> None:
-        await request_sender(url="https://example.com", callback=self.parse)
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://example.com", callback=self.parse)
 
     async def parse(self, response: Response) -> None:
         # handle response

aioscrapper-0.2.0/aioscrapper/pipeline/__init__.py

@@ -0,0 +1 @@
+from .base import BasePipeline

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/aioscrapper/pipeline/base.py

@@ -1,11 +1,7 @@
 import abc
-from typing import TypeVar, Generic, Protocol
-
-
-class BaseItem(Protocol):
-    @property
-    def pipeline_name(self) -> str: ...
+from typing import TypeVar, Generic
 
+from ..types import BaseItem
 
 ItemType = TypeVar("ItemType", bound=BaseItem)
 

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/aioscrapper/pipeline/dispatcher.py

@@ -10,7 +10,7 @@ class PipelineDispatcher:
         self._logger = logger
         self._pipelines = pipelines
 
-    async def put_item(self, item: BaseItem) -> BaseItem:
+    async def __call__(self, item: BaseItem) -> BaseItem:
         self._logger.debug(f"pipeline item received: {item}")
         try:
             pipelines = self._pipelines[item.pipeline_name]

aioscrapper-0.2.0/aioscrapper/scrapper/base.py

@@ -0,0 +1,30 @@
+import abc
+
+
+class BaseScrapper(abc.ABC):
+    @abc.abstractmethod
+    async def start(self, *args, **kwargs) -> None:
+        """
+        Starts the scrapper.
+
+        This method is called to start the scraper by sending the initial requests required for its operation.
+        """
+        ...
+
+    async def initialize(self, *args, **kwargs) -> None:
+        """
+        Initializes the scrapper.
+
+        This method is called before starting the scrapper. It should be used to initialize any
+        necessary state or resources required by the scrapper.
+        """
+        ...
+
+    async def close(self, *args, **kwargs) -> None:
+        """
+        Closes the scrapper.
+
+        This method is called to clean up any resources created by the scrapper after it has finished
+        running.
+        """
+        ...

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/aioscrapper/scrapper/executor.py

@@ -9,7 +9,8 @@ from aiojobs import Scheduler
 from .request_manager import RequestManager
 from ..config import Config
 from ..helpers import get_func_kwargs
-from ..pipeline import PipelineDispatcher, BasePipeline
+from ..pipeline import BasePipeline
+from ..pipeline.dispatcher import PipelineDispatcher
 from ..scrapper import BaseScrapper
 from ..session.aiohttp import AiohttpSession
 from ..types import RequestMiddleware, ResponseMiddleware
@@ -94,7 +95,7 @@ class AIOScrapper:
         await self._pipeline_dispatcher.initialize()
         self._request_manager.listen_queue()
 
-        scrapper_kwargs = {"request_sender": self._request_manager.sender, "pipeline": self._pipeline_dispatcher}
+        scrapper_kwargs = {"send_request": self._request_manager.sender, "pipeline": self._pipeline_dispatcher}
         for scrapper in self._scrappers:
             await scrapper.initialize(**get_func_kwargs(scrapper.initialize, scrapper_kwargs))
 

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/aioscrapper/types/__init__.py

@@ -1,4 +1,5 @@
 from .middleware import RequestMiddleware, ResponseMiddleware
+from .pipeline import BaseItem, Pipeline
 from .session import (
     QueryParams,
     Cookies,

aioscrapper-0.2.0/aioscrapper/types/pipeline.py

@@ -0,0 +1,10 @@
+from typing import Protocol
+
+
+class BaseItem(Protocol):
+    @property
+    def pipeline_name(self) -> str: ...
+
+
+class Pipeline(Protocol):
+    async def __call__(self, item: BaseItem) -> BaseItem: ...

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/aioscrapper.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aioscrapper
-Version: 0.1.2
+Version: 0.2.0
 Summary: Async framework for building modular and scalable web scrapers.
 Author: darkstussy
 Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
@@ -68,8 +68,8 @@ from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):
-    async def start(self, request_sender: RequestSender) -> None:
-        await request_sender(url="https://example.com", callback=self.parse)
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://example.com", callback=self.parse)
 
     async def parse(self, response: Response) -> None:
         # handle response

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/aioscrapper.egg-info/SOURCES.txt

@@ -22,6 +22,7 @@ aioscrapper/session/aiohttp.py
 aioscrapper/session/base.py
 aioscrapper/types/__init__.py
 aioscrapper/types/middleware.py
+aioscrapper/types/pipeline.py
 aioscrapper/types/session.py
 tests/test_error.py
 tests/test_success.py

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "aioscrapper"
-version = "0.1.2"
+version = "0.2.0"
 authors = [{ name = "darkstussy" }, ]
 description = "Async framework for building modular and scalable web scrapers."
 readme = "README.md"

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/tests/test_error.py

@@ -12,8 +12,8 @@ class Scrapper(BaseScrapper):
         self.status = None
         self.response_data = None
 
-    async def start(self, request_sender: RequestSender) -> None:
-        await request_sender(url="https://api.test.com/v1", errback=self.errback)
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://api.test.com/v1", errback=self.errback)
 
     async def errback(self, exc: ClientException) -> None:
         if isinstance(exc, HTTPException):

{aioscrapper-0.1.2 → aioscrapper-0.2.0}/tests/test_success.py

@@ -10,8 +10,8 @@ class Scrapper(BaseScrapper):
     def __init__(self):
         self.response_data = None
 
-    async def start(self, request_sender: RequestSender) -> None:
-        await request_sender(url="https://api.test.com/v1", callback=self.parse)
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://api.test.com/v1", callback=self.parse)
 
     async def parse(self, response: Response) -> None:
         self.response_data = response.json()

aioscrapper-0.1.2/aioscrapper/pipeline/__init__.py

@@ -1,2 +0,0 @@
-from .base import BasePipeline, BaseItem
-from .dispatcher import PipelineDispatcher

aioscrapper-0.1.2/aioscrapper/scrapper/base.py

@@ -1,10 +0,0 @@
-import abc
-
-
-class BaseScrapper(abc.ABC):
-    @abc.abstractmethod
-    async def start(self, *args, **kwargs) -> None: ...
-
-    async def initialize(self, *args, **kwargs) -> None: ...
-
-    async def close(self, *args, **kwargs) -> None: ...