aioscrapper 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

aioscrapper/__init__.py

@@ -4,7 +4,6 @@ __author__ = "darkstussy"
 
 __copyright__ = f"Copyright (c) 2025 {__author__}"
 
-from .request_sender import RequestSender
 from .scrapper import AIOScrapper, BaseScrapper
 
-__all__ = ["AIOScrapper", "BaseScrapper", "RequestSender"]
+__all__ = ["AIOScrapper", "BaseScrapper"]

aioscrapper/scrapper/base.py

@@ -7,4 +7,4 @@ class BaseScrapper(abc.ABC):
 
     async def initialize(self, *args, **kwargs) -> None: ...
 
-    async def close(self) -> None: ...
+    async def close(self, *args, **kwargs) -> None: ...

aioscrapper/scrapper/executor.py

@@ -6,11 +6,10 @@ from typing import Type, Any
 
 from aiojobs import Scheduler
 
+from .request_manager import RequestManager
 from ..config import Config
 from ..helpers import get_func_kwargs
 from ..pipeline import PipelineDispatcher, BasePipeline
-from ..request_manager import RequestManager
-from ..request_sender import RequestSender
 from ..scrapper import BaseScrapper
 from ..session.aiohttp import AiohttpSession
 from ..types import RequestMiddleware, ResponseMiddleware
@@ -33,13 +32,11 @@ class AIOScrapper:
         self._response_middlewares = []
 
         self._pipelines: dict[str, list[BasePipeline]] = {}
-        self._pipeline_dispatcher = PipelineDispatcher(
-            logger=self._logger.getChild("pipeline"), pipelines=self._pipelines
-        )
+        self._pipeline_dispatcher = PipelineDispatcher(self._logger.getChild("pipeline"), pipelines=self._pipelines)
 
         def _exception_handler(_, context: dict[str, Any]):
             if "job" in context:
-                self._logger.error(f'{context['message']}: {context["exception"]}', extra={"context": context})
+                self._logger.error(f'{context["message"]}: {context["exception"]}', extra={"context": context})
             else:
                 self._logger.error("Unhandled error", extra={"context": context})
 
@@ -51,7 +48,6 @@ class AIOScrapper:
         )
 
         self._request_queue = asyncio.PriorityQueue()
-        self._request_sender = RequestSender(self._request_queue)
         self._request_manager = RequestManager(
             logger=self._logger.getChild("request_worker"),
             session=AiohttpSession(
@@ -59,7 +55,6 @@ class AIOScrapper:
                 ssl=self._config.session.request.ssl,
             ),
             schedule_request=self._scheduler.spawn,
-            sender=self._request_sender,
             queue=self._request_queue,
             delay=self._config.session.request.delay,
             shutdown_timeout=self._config.execution.shutdown_timeout,
@@ -69,8 +64,6 @@ class AIOScrapper:
             response_middlewares=self._response_middlewares,
         )
 
-        self._scrapper_kwargs = {"request_sender": self._request_sender, "pipeline": self._pipeline_dispatcher}
-
     def add_pipeline(self, name: str, pipeline: BasePipeline) -> None:
         if name not in self._pipelines:
             self._pipelines[name] = [pipeline]
@@ -86,7 +79,7 @@ class AIOScrapper:
     def add_response_middlewares(self, *middlewares: ResponseMiddleware) -> None:
         self._response_middlewares.extend(middlewares)
 
-    async def __aenter__(self):
+    async def __aenter__(self) -> "AIOScrapper":
         return self
 
     async def __aexit__(
@@ -101,11 +94,12 @@ class AIOScrapper:
         await self._pipeline_dispatcher.initialize()
         self._request_manager.listen_queue()
 
+        scrapper_kwargs = {"request_sender": self._request_manager.sender, "pipeline": self._pipeline_dispatcher}
         for scrapper in self._scrappers:
-            await scrapper.initialize(**get_func_kwargs(scrapper.initialize, self._scrapper_kwargs))
+            await scrapper.initialize(**get_func_kwargs(scrapper.initialize, scrapper_kwargs))
 
         await asyncio.gather(
-            *[scrapper.start(**get_func_kwargs(scrapper.start, self._scrapper_kwargs)) for scrapper in self._scrappers]
+            *[scrapper.start(**get_func_kwargs(scrapper.start, scrapper_kwargs)) for scrapper in self._scrappers]
         )
 
     async def _shutdown(self) -> bool:
@@ -138,9 +132,11 @@ class AIOScrapper:
         if shutdown:
             await self.shutdown()
 
-        for scrapper in self._scrappers:
-            await scrapper.close()
-
-        await self._scheduler.close()
-        await self._request_manager.close()
-        await self._pipeline_dispatcher.close()
+        scrapper_kwargs = {"pipeline": self._pipeline_dispatcher}
+        try:
+            for scrapper in self._scrappers:
+                await scrapper.close(**get_func_kwargs(scrapper.close, scrapper_kwargs))
+        finally:
+            await self._scheduler.close()
+            await self._request_manager.close()
+            await self._pipeline_dispatcher.close()

aioscrapper/{request_manager.py → scrapper/request_manager.py}

@@ -1,13 +1,76 @@
 import asyncio
+from dataclasses import dataclass, field
 from logging import Logger
-from typing import Callable, Awaitable, Any, Coroutine
+from typing import Callable, Awaitable, Any
+from typing import Coroutine
 
-from .exceptions import HTTPException, RequestException, ClientException
-from .helpers import get_cb_kwargs
-from .request_sender import RequestSender
-from .session.base import BaseSession
-from .types import Request, RequestParams, RequestQueue
-from .types import RequestMiddleware, ResponseMiddleware
+from ..exceptions import HTTPException, RequestException, ClientException
+from ..helpers import get_cb_kwargs
+from ..session.base import BaseSession
+from ..types import (
+    QueryParams,
+    Cookies,
+    Headers,
+    BasicAuth,
+    Request,
+    RequestParams,
+    RequestMiddleware,
+    ResponseMiddleware,
+    RequestSender,
+)
+
+
+@dataclass(slots=True, order=True)
+class _PRPRequest:
+    priority: int
+    request: Request = field(compare=False)
+    request_params: RequestParams = field(compare=False)
+
+
+_RequestQueue = asyncio.PriorityQueue[_PRPRequest | None]
+
+
+def _get_request_sender(queue: _RequestQueue) -> RequestSender:
+    async def sender(
+        url: str,
+        method: str = "GET",
+        callback: Callable[..., Awaitable] | None = None,
+        cb_kwargs: dict[str, Any] | None = None,
+        errback: Callable[..., Awaitable] | None = None,
+        params: QueryParams | None = None,
+        data: Any = None,
+        json_data: Any = None,
+        cookies: Cookies | None = None,
+        headers: Headers | None = None,
+        proxy: str | None = None,
+        auth: BasicAuth | None = None,
+        timeout: float | None = None,
+        priority: int = 0,
+    ) -> None:
+        await queue.put(
+            _PRPRequest(
+                priority=priority,
+                request=Request(
+                    method=method,
+                    url=url,
+                    params=params,
+                    data=data,
+                    json_data=json_data,
+                    cookies=cookies,
+                    headers=headers,
+                    auth=auth,
+                    proxy=proxy,
+                    timeout=timeout,
+                ),
+                request_params=RequestParams(
+                    callback=callback,
+                    cb_kwargs=cb_kwargs,
+                    errback=errback,
+                ),
+            )
+        )
+
+    return sender
 
 
 class RequestManager:
@@ -16,8 +79,7 @@ class RequestManager:
         logger: Logger,
         session: BaseSession,
         schedule_request: Callable[[Coroutine], Awaitable],
-        sender: RequestSender,
-        queue: RequestQueue,
+        queue: _RequestQueue,
         delay: float,
         shutdown_timeout: float,
         srv_kwargs: dict[str, Any],
@@ -31,12 +93,17 @@ class RequestManager:
         self._queue = queue
         self._delay = delay
         self._shutdown_timeout = shutdown_timeout
-        self._srv_kwargs = {"send_request": sender, **srv_kwargs}
+        self._request_sender = _get_request_sender(queue)
+        self._srv_kwargs = {"send_request": self._request_sender, **srv_kwargs}
         self._request_outer_middlewares = request_outer_middlewares
         self._request_inner_middlewares = request_inner_middlewares
         self._response_middlewares = response_middlewares
         self._task: asyncio.Task | None = None
 
+    @property
+    def sender(self) -> RequestSender:
+        return self._request_sender
+
     async def _send_request(self, request: Request, params: RequestParams) -> None:
         full_url = request.full_url
         self._logger.debug(f"request: {request.method} {full_url}")

aioscrapper/types/__init__.py

@@ -6,7 +6,6 @@ from .session import (
     BasicAuth,
     Request,
     RequestParams,
-    PRPRequest,
-    RequestQueue,
+    RequestSender,
     Response,
 )

aioscrapper/types/session.py

@@ -1,7 +1,6 @@
-import asyncio
 import json
-from dataclasses import field, dataclass
-from typing import Union, Mapping, Any, Callable, Awaitable, TypedDict
+from dataclasses import dataclass
+from typing import Union, Mapping, Any, Callable, Awaitable, TypedDict, Protocol
 from urllib.parse import urlencode
 
 QueryParams = Mapping[str, Union[str, int, float]]
@@ -40,14 +39,24 @@ class RequestParams:
     errback: Callable[..., Awaitable] | None = None
 
 
-@dataclass(slots=True, order=True)
-class PRPRequest:
-    priority: int
-    request: Request = field(compare=False)
-    request_params: RequestParams = field(compare=False)
-
-
-RequestQueue = asyncio.PriorityQueue[PRPRequest | None]
+class RequestSender(Protocol):
+    async def __call__(
+        self,
+        url: str,
+        method: str = "GET",
+        callback: Callable[..., Awaitable] | None = None,
+        cb_kwargs: dict[str, Any] | None = None,
+        errback: Callable[..., Awaitable] | None = None,
+        params: QueryParams | None = None,
+        data: Any = None,
+        json_data: Any = None,
+        cookies: Cookies | None = None,
+        headers: Headers | None = None,
+        proxy: str | None = None,
+        auth: BasicAuth | None = None,
+        timeout: float | None = None,
+        priority: int = 0,
+    ) -> None: ...
 
 
 class Response:

{aioscrapper-0.1.0.dist-info → aioscrapper-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aioscrapper
-Version: 0.1.0
+Version: 0.1.2
 Summary: Async framework for building modular and scalable web scrapers.
 Author: darkstussy
 Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
@@ -12,18 +12,28 @@ Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp[speedups]~=3.11.16
 Requires-Dist: aiojobs~=1.4.0
+Provides-Extra: dev
+Requires-Dist: flake8~=7.1.2; extra == "dev"
+Requires-Dist: black~=25.1.0; extra == "dev"
+Requires-Dist: pyright~=1.1.399; extra == "dev"
+Requires-Dist: aiohttp[speedups]~=3.11.16; extra == "dev"
+Requires-Dist: aiojobs~=1.4.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest~=8.3.5; extra == "test"
+Requires-Dist: pytest-asyncio~=0.26.0; extra == "test"
+Requires-Dist: aresponses~=3.0.0; extra == "test"
 Dynamic: license-file
 
 # aioscrapper
 
 **Asynchronous framework for building modular and scalable web scrapers.**
 
-![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![Python](https://img.shields.io/badge/python-3.10%2B-blue)
 ![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
 ![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
 
@@ -44,7 +54,7 @@ pip install aioscrapper
 
 ## Requirements
 
-- Python 3.12 or higher
+- Python 3.10 or higher
 - aiohttp
 - aiojobs
 
@@ -53,8 +63,8 @@ pip install aioscrapper
 ```python
 import asyncio
 
-from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
-from aioscrapper.types import Response
+from aioscrapper import BaseScrapper, AIOScrapper
+from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):
@@ -83,5 +93,6 @@ Copyright (c) 2025 darkstussy
 
 ## Links
 
+- [PyPI](https://pypi.org/project/aioscrapper)
 - [GitHub](https://github.com/darkstussy/aioscrapper)
 - [Issues](https://github.com/darkstussy/aioscrapper/issues)

aioscrapper-0.1.2.dist-info/RECORD

@@ -0,0 +1,22 @@
+aioscrapper/__init__.py,sha256=Yl57BbmijQN_UgP5nRUiWYqgU8kXQ9kplzzzyMcsUMY,197
+aioscrapper/config.py,sha256=yO5ipQUHxA_-CiSqJ0u7WioN6lu8VgT1ss5PRvS1foc,844
+aioscrapper/exceptions.py,sha256=Akk3zDTgws9E7J-Sh8bgdlgS8L3auDKuv3_U3aefxMc,765
+aioscrapper/helpers.py,sha256=slq9r5oCHrR7M9hKZFBLFRsWoqJcw_QFptQI1NjIdQw,610
+aioscrapper/pipeline/__init__.py,sha256=hv7Kcssd2BP0LM9fNZtaMs1tmRuAUu4mwAescoeV3Uk,84
+aioscrapper/pipeline/base.py,sha256=Ro7YGUOB-V2NJCtfgwhtQDedY4OYMu-jwEV8iR-L89k,405
+aioscrapper/pipeline/dispatcher.py,sha256=H4cHNxTyHEF4BnEwaW6nwmcRmK839GqbDTzZh1Zftv4,1156
+aioscrapper/scrapper/__init__.py,sha256=UR7bTck-_YVoP2BqYdPldN9PgaCuJf9wvDdQLTVJ578,65
+aioscrapper/scrapper/base.py,sha256=2_WeLMyJICLmIG7N9r6BGmBg0f-wjEQPsVY076WHKOI,241
+aioscrapper/scrapper/executor.py,sha256=5j66FqVaZ5AWXMF-SHfTiqkORlTkY53rZDPO2pWLfww,5466
+aioscrapper/scrapper/request_manager.py,sha256=xhF_feppHQognTbbHjVUC13V4NwJJt7bCWwcyznFK84,5831
+aioscrapper/session/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+aioscrapper/session/aiohttp.py,sha256=8h4Ca1o2JJ7L24iqEnZ5I8bUudTn2cvTngBhM8eJPD4,1753
+aioscrapper/session/base.py,sha256=Zxw1VHIe_LgveUufJXh_cl0YeHykiutQveMUwZ6VL54,356
+aioscrapper/types/__init__.py,sha256=OK7vKZznJAWXQFBhXbQJha-XqGoRAE3lCaJUF7WXn64,210
+aioscrapper/types/middleware.py,sha256=WtT73QTAlwhdP6UNFgyFHGpFOx1vlxehCAwiO6xjR10,326
+aioscrapper/types/session.py,sha256=WppvDBZ0sBWVddzz7RXLkg8iZCfZipTdHpKuGW-U090,2970
+aioscrapper-0.1.2.dist-info/licenses/LICENSE,sha256=EEeV20hghyroJWe2vcHjJma9PcjSkjD6vIwlUtaAjLE,1067
+aioscrapper-0.1.2.dist-info/METADATA,sha256=CBYHtlAtL4dgEp7Y0RMdx4JKvAaLV3roEfHL8YQGRk8,2784
+aioscrapper-0.1.2.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+aioscrapper-0.1.2.dist-info/top_level.txt,sha256=d7lbzXOwzzk2HLh-A0X7dkqn8q3zGAJcKqx6TkaEEWI,12
+aioscrapper-0.1.2.dist-info/RECORD,,

{aioscrapper-0.1.0.dist-info → aioscrapper-0.1.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{aioscrapper-0.1.0.dist-info → aioscrapper-0.1.2.dist-info}/licenses/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Stanislav
+Copyright (c) 2025 darkstussy
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

aioscrapper/request_sender.py

@@ -1,52 +0,0 @@
-import asyncio
-from typing import Callable, Awaitable, Any
-
-from .types import QueryParams, Cookies, Headers, BasicAuth, Request, RequestParams, RequestQueue, PRPRequest
-
-
-class RequestSender:
-    def __init__(self, queue: RequestQueue) -> None:
-        self._queue = queue
-
-    async def __call__(
-        self,
-        url: str,
-        method: str = "GET",
-        callback: Callable[..., Awaitable] | None = None,
-        cb_kwargs: dict[str, Any] | None = None,
-        errback: Callable[..., Awaitable] | None = None,
-        params: QueryParams | None = None,
-        data: Any = None,
-        json_data: Any = None,
-        cookies: Cookies | None = None,
-        headers: Headers | None = None,
-        proxy: str | None = None,
-        auth: BasicAuth | None = None,
-        timeout: float | None = None,
-        priority: int = 0,
-        delay: float | None = None,
-    ) -> None:
-        await self._queue.put(
-            PRPRequest(
-                priority=priority,
-                request=Request(
-                    method=method,
-                    url=url,
-                    params=params,
-                    data=data,
-                    json_data=json_data,
-                    cookies=cookies,
-                    headers=headers,
-                    auth=auth,
-                    proxy=proxy,
-                    timeout=timeout,
-                ),
-                request_params=RequestParams(
-                    callback=callback,
-                    cb_kwargs=cb_kwargs,
-                    errback=errback,
-                ),
-            )
-        )
-        if delay:
-            await asyncio.sleep(delay)

aioscrapper-0.1.0.dist-info/RECORD

@@ -1,23 +0,0 @@
-aioscrapper/__init__.py,sha256=_01EI59FLQmHspoN9HqJMoJ9OHpEaYGDyFXAKtfnnYY,256
-aioscrapper/config.py,sha256=yO5ipQUHxA_-CiSqJ0u7WioN6lu8VgT1ss5PRvS1foc,844
-aioscrapper/exceptions.py,sha256=Akk3zDTgws9E7J-Sh8bgdlgS8L3auDKuv3_U3aefxMc,765
-aioscrapper/helpers.py,sha256=slq9r5oCHrR7M9hKZFBLFRsWoqJcw_QFptQI1NjIdQw,610
-aioscrapper/request_manager.py,sha256=YLZvuPthhFMnJVQ7pV9-YCsni0Kdu8baP5tmOccEDOM,4037
-aioscrapper/request_sender.py,sha256=_Vx_LJyV_5qb23-C3VdnOUUUcQPW42OJNbtERVu1DIA,1644
-aioscrapper/pipeline/__init__.py,sha256=hv7Kcssd2BP0LM9fNZtaMs1tmRuAUu4mwAescoeV3Uk,84
-aioscrapper/pipeline/base.py,sha256=Ro7YGUOB-V2NJCtfgwhtQDedY4OYMu-jwEV8iR-L89k,405
-aioscrapper/pipeline/dispatcher.py,sha256=H4cHNxTyHEF4BnEwaW6nwmcRmK839GqbDTzZh1Zftv4,1156
-aioscrapper/scrapper/__init__.py,sha256=UR7bTck-_YVoP2BqYdPldN9PgaCuJf9wvDdQLTVJ578,65
-aioscrapper/scrapper/base.py,sha256=_wFrI0UVsTBIAV7EOZCk_QMy2-chPjr1pKzu6w8Huso,224
-aioscrapper/scrapper/executor.py,sha256=TrZBh0JyFeQIJd_O4S86cSZNgywjKxJWdb2QSzZyObU,5475
-aioscrapper/session/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aioscrapper/session/aiohttp.py,sha256=8h4Ca1o2JJ7L24iqEnZ5I8bUudTn2cvTngBhM8eJPD4,1753
-aioscrapper/session/base.py,sha256=Zxw1VHIe_LgveUufJXh_cl0YeHykiutQveMUwZ6VL54,356
-aioscrapper/types/__init__.py,sha256=SC6De0ThepMK-wPQGFYGq6x8aGlzsWscphFxGQGBWek,225
-aioscrapper/types/middleware.py,sha256=WtT73QTAlwhdP6UNFgyFHGpFOx1vlxehCAwiO6xjR10,326
-aioscrapper/types/session.py,sha256=ffJelDaZmeIoNOk_ivGb_nSC5bBpgKwCyiSsUl4e-B0,2595
-aioscrapper-0.1.0.dist-info/licenses/LICENSE,sha256=LefKIkLsd_UuLWYOatzEjY5yscQS8nZAFi8rzCs54OM,1066
-aioscrapper-0.1.0.dist-info/METADATA,sha256=pbjxh2xCDlGRYDEFXT40JBL6_7Xkw31-BXivX_Fmog0,2303
-aioscrapper-0.1.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-aioscrapper-0.1.0.dist-info/top_level.txt,sha256=d7lbzXOwzzk2HLh-A0X7dkqn8q3zGAJcKqx6TkaEEWI,12
-aioscrapper-0.1.0.dist-info/RECORD,,