aioscrapper 0.1.0__tar.gz

aioscrapper-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Stanislav
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

aioscrapper-0.1.0/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: aioscrapper
+Version: 0.1.0
+Summary: Async framework for building modular and scalable web scrapers.
+Author: darkstussy
+Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
+Project-URL: Issues, https://github.com/darkstussy/aioscrapper/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Framework :: AsyncIO
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiohttp[speedups]~=3.11.16
+Requires-Dist: aiojobs~=1.4.0
+Dynamic: license-file
+
+# aioscrapper
+
+**Asynchronous framework for building modular and scalable web scrapers.**
+
+![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
+![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
+
+## Features
+
+- 🚀 Fully asynchronous architecture powered by `aiohttp` and `aiojobs`
+- 🔧 Modular system with middleware support
+- 📦 Pipeline data processing
+- ⚙️ Flexible configuration
+- 🔄 Priority-based request queue management
+- 🛡️ Built-in error handling
+
+## Installation
+
+```bash
+pip install aioscrapper
+```
+
+## Requirements
+
+- Python 3.12 or higher
+- aiohttp
+- aiojobs
+
+## Quick Start
+
+```python
+import asyncio
+
+from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
+from aioscrapper.types import Response
+
+
+class Scrapper(BaseScrapper):
+    async def start(self, request_sender: RequestSender) -> None:
+        await request_sender(url="https://example.com", callback=self.parse)
+
+    async def parse(self, response: Response) -> None:
+        # handle response
+        pass
+
+
+async def main():
+    async with AIOScrapper(scrappers=[Scrapper()]) as scrapper:
+        await scrapper.start()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## License
+
+MIT License
+
+Copyright (c) 2025 darkstussy
+
+## Links
+
+- [GitHub](https://github.com/darkstussy/aioscrapper)
+- [Issues](https://github.com/darkstussy/aioscrapper/issues)

aioscrapper-0.1.0/README.md

@@ -0,0 +1,66 @@
+# aioscrapper
+
+**Asynchronous framework for building modular and scalable web scrapers.**
+
+![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
+![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
+
+## Features
+
+- 🚀 Fully asynchronous architecture powered by `aiohttp` and `aiojobs`
+- 🔧 Modular system with middleware support
+- 📦 Pipeline data processing
+- ⚙️ Flexible configuration
+- 🔄 Priority-based request queue management
+- 🛡️ Built-in error handling
+
+## Installation
+
+```bash
+pip install aioscrapper
+```
+
+## Requirements
+
+- Python 3.12 or higher
+- aiohttp
+- aiojobs
+
+## Quick Start
+
+```python
+import asyncio
+
+from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
+from aioscrapper.types import Response
+
+
+class Scrapper(BaseScrapper):
+    async def start(self, request_sender: RequestSender) -> None:
+        await request_sender(url="https://example.com", callback=self.parse)
+
+    async def parse(self, response: Response) -> None:
+        # handle response
+        pass
+
+
+async def main():
+    async with AIOScrapper(scrappers=[Scrapper()]) as scrapper:
+        await scrapper.start()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## License
+
+MIT License
+
+Copyright (c) 2025 darkstussy
+
+## Links
+
+- [GitHub](https://github.com/darkstussy/aioscrapper)
+- [Issues](https://github.com/darkstussy/aioscrapper/issues)

aioscrapper-0.1.0/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "aioscrapper"
+version = "0.1.0"
+authors = [{ name = "darkstussy" }, ]
+description = "Async framework for building modular and scalable web scrapers."
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "aiohttp[speedups] ~= 3.11.16",
+    "aiojobs ~= 1.4.0",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Framework :: AsyncIO",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+]
+
+[project.urls]
+Homepage = "https://github.com/darkstussy/aioscrapper"
+Issues = "https://github.com/darkstussy/aioscrapper/issues"

aioscrapper-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

aioscrapper-0.1.0/src/aioscrapper/__init__.py

@@ -0,0 +1,10 @@
+__title__ = "aioscrapper"
+
+__author__ = "darkstussy"
+
+__copyright__ = f"Copyright (c) 2025 {__author__}"
+
+from .request_sender import RequestSender
+from .scrapper import AIOScrapper, BaseScrapper
+
+__all__ = ["AIOScrapper", "BaseScrapper", "RequestSender"]

aioscrapper-0.1.0/src/aioscrapper/config.py

@@ -0,0 +1,36 @@
+import logging
+from dataclasses import dataclass
+
+
+@dataclass(slots=True, frozen=True)
+class RequestConfig:
+    timeout: int = 60
+    delay: float = 0.0
+    ssl: bool = True
+
+
+@dataclass(slots=True, frozen=True)
+class SessionConfig:
+    request: RequestConfig = RequestConfig()
+
+
+@dataclass(slots=True, frozen=True)
+class SchedulerConfig:
+    concurrent_requests: int = 64
+    pending_requests: int = 1
+    close_timeout: float | None = 0.1
+
+
+@dataclass(slots=True, frozen=True)
+class ExecutionConfig:
+    timeout: float | None = None
+    shutdown_timeout: float = 0.1
+    shutdown_check_interval: float = 0.1
+    log_level: int = logging.ERROR
+
+
+@dataclass(slots=True, frozen=True)
+class Config:
+    session: SessionConfig = SessionConfig()
+    scheduler: SchedulerConfig = SchedulerConfig()
+    execution: ExecutionConfig = ExecutionConfig()

aioscrapper-0.1.0/src/aioscrapper/exceptions.py

@@ -0,0 +1,26 @@
+class ClientException(Exception):
+    pass
+
+
+class HTTPException(ClientException):
+    def __init__(self, status_code: int, message: str | None, url: str, method: str) -> None:
+        self.status_code = status_code
+        self.message = message
+        self.url = url
+        self.method = method
+
+    def __str__(self) -> str:
+        return f"{self.method} {self.url}: {self.status_code}: {self.message}"
+
+
+class RequestException(ClientException):
+    def __init__(self, src: Exception | str, url: str, method: str) -> None:
+        self.src = src
+        self.url = url
+        self.method = method
+
+    def __str__(self) -> str:
+        return f"[{self.src.__class__.__name__}]: {self.method} {self.url}: {self.src}"
+
+
+class PipelineException(Exception): ...

aioscrapper-0.1.0/src/aioscrapper/helpers.py

@@ -0,0 +1,22 @@
+import inspect
+from typing import Callable, Any
+
+
+def get_func_kwargs(func: Callable, kwargs: dict[str, Any]) -> dict[str, Any]:
+    return {param: kwargs[param] for param in inspect.signature(func).parameters.keys() if param in kwargs}
+
+
+def get_cb_kwargs(
+    callback: Callable,
+    srv_kwargs: dict[str, Any] | None,
+    cb_kwargs: dict[str, Any] | None,
+) -> dict[str, Any]:
+    if cb_kwargs is None and srv_kwargs is None:
+        return {}
+
+    if cb_kwargs is None:
+        cb_kwargs = {}
+    if srv_kwargs is None:
+        srv_kwargs = {}
+
+    return get_func_kwargs(callback, cb_kwargs | srv_kwargs)

aioscrapper-0.1.0/src/aioscrapper/pipeline/__init__.py

@@ -0,0 +1,2 @@
+from .base import BasePipeline, BaseItem
+from .dispatcher import PipelineDispatcher

aioscrapper-0.1.0/src/aioscrapper/pipeline/base.py

@@ -0,0 +1,19 @@
+import abc
+from typing import TypeVar, Generic, Protocol
+
+
+class BaseItem(Protocol):
+    @property
+    def pipeline_name(self) -> str: ...
+
+
+ItemType = TypeVar("ItemType", bound=BaseItem)
+
+
+class BasePipeline(abc.ABC, Generic[ItemType]):
+    @abc.abstractmethod
+    async def put_item(self, item: ItemType) -> None: ...
+
+    async def initialize(self) -> None: ...
+
+    async def close(self) -> None: ...

aioscrapper-0.1.0/src/aioscrapper/pipeline/dispatcher.py

@@ -0,0 +1,36 @@
+from logging import Logger
+from typing import Generator
+
+from .base import BasePipeline, BaseItem
+from ..exceptions import PipelineException
+
+
+class PipelineDispatcher:
+    def __init__(self, logger: Logger, pipelines: dict[str, list[BasePipeline]]) -> None:
+        self._logger = logger
+        self._pipelines = pipelines
+
+    async def put_item(self, item: BaseItem) -> BaseItem:
+        self._logger.debug(f"pipeline item received: {item}")
+        try:
+            pipelines = self._pipelines[item.pipeline_name]
+        except KeyError:
+            raise PipelineException(f"Pipelines for item {item} not found")
+
+        for pipeline in pipelines:
+            await pipeline.put_item(item)
+
+        return item
+
+    def _get_pipelines(self) -> Generator[BasePipeline, None, None]:
+        for pipelines in self._pipelines.values():
+            for pipeline in pipelines:
+                yield pipeline
+
+    async def initialize(self) -> None:
+        for pipeline in self._get_pipelines():
+            await pipeline.initialize()
+
+    async def close(self) -> None:
+        for pipeline in self._get_pipelines():
+            await pipeline.close()

aioscrapper-0.1.0/src/aioscrapper/request_manager.py

@@ -0,0 +1,101 @@
+import asyncio
+from logging import Logger
+from typing import Callable, Awaitable, Any, Coroutine
+
+from .exceptions import HTTPException, RequestException, ClientException
+from .helpers import get_cb_kwargs
+from .request_sender import RequestSender
+from .session.base import BaseSession
+from .types import Request, RequestParams, RequestQueue
+from .types import RequestMiddleware, ResponseMiddleware
+
+
+class RequestManager:
+    def __init__(
+        self,
+        logger: Logger,
+        session: BaseSession,
+        schedule_request: Callable[[Coroutine], Awaitable],
+        sender: RequestSender,
+        queue: RequestQueue,
+        delay: float,
+        shutdown_timeout: float,
+        srv_kwargs: dict[str, Any],
+        request_outer_middlewares: list[RequestMiddleware],
+        request_inner_middlewares: list[RequestMiddleware],
+        response_middlewares: list[ResponseMiddleware],
+    ) -> None:
+        self._logger = logger
+        self._session = session
+        self._schedule_request = schedule_request
+        self._queue = queue
+        self._delay = delay
+        self._shutdown_timeout = shutdown_timeout
+        self._srv_kwargs = {"send_request": sender, **srv_kwargs}
+        self._request_outer_middlewares = request_outer_middlewares
+        self._request_inner_middlewares = request_inner_middlewares
+        self._response_middlewares = response_middlewares
+        self._task: asyncio.Task | None = None
+
+    async def _send_request(self, request: Request, params: RequestParams) -> None:
+        full_url = request.full_url
+        self._logger.debug(f"request: {request.method} {full_url}")
+        try:
+            for inner_middleware in self._request_inner_middlewares:
+                await inner_middleware(request, params)
+
+            response = await self._session.make_request(request)
+            for response_middleware in self._response_middlewares:
+                await response_middleware(params, response)
+
+            if response.status >= 400:
+                await self._handle_client_exception(
+                    params,
+                    client_exc=HTTPException(
+                        status_code=response.status,
+                        message=response.text(),
+                        url=full_url,
+                        method=response.method,
+                    ),
+                )
+            elif params.callback is not None:
+                await params.callback(
+                    response,
+                    **get_cb_kwargs(params.callback, srv_kwargs=self._srv_kwargs, cb_kwargs=params.cb_kwargs),
+                )
+        except Exception as exc:
+            await self._handle_client_exception(
+                params,
+                client_exc=RequestException(src=exc, url=full_url, method=request.method),
+            )
+
+    async def _handle_client_exception(self, params: RequestParams, client_exc: ClientException) -> None:
+        if params.errback is None:
+            raise client_exc
+
+        try:
+            await params.errback(
+                client_exc,
+                **get_cb_kwargs(params.errback, srv_kwargs=self._srv_kwargs, cb_kwargs=params.cb_kwargs),
+            )
+        except Exception as exc:
+            self._logger.exception(exc)
+
+    def listen_queue(self) -> None:
+        self._task = asyncio.create_task(self._listen_queue())
+
+    async def _listen_queue(self) -> None:
+        while (r := (await self._queue.get())) is not None:
+            for outer_middleware in self._request_outer_middlewares:
+                await outer_middleware(r.request, r.request_params)
+
+            await self._schedule_request(self._send_request(r.request, r.request_params))
+            await asyncio.sleep(self._delay)
+
+    async def shutdown(self, force: bool = False) -> None:
+        await self._queue.put(None)
+        if self._task is not None:
+            await asyncio.wait_for(self._task, timeout=self._shutdown_timeout) if force else await self._task
+
+    async def close(self) -> None:
+        await self._session.close()

aioscrapper-0.1.0/src/aioscrapper/request_sender.py

@@ -0,0 +1,52 @@
+import asyncio
+from typing import Callable, Awaitable, Any
+
+from .types import QueryParams, Cookies, Headers, BasicAuth, Request, RequestParams, RequestQueue, PRPRequest
+
+
+class RequestSender:
+    def __init__(self, queue: RequestQueue) -> None:
+        self._queue = queue
+
+    async def __call__(
+        self,
+        url: str,
+        method: str = "GET",
+        callback: Callable[..., Awaitable] | None = None,
+        cb_kwargs: dict[str, Any] | None = None,
+        errback: Callable[..., Awaitable] | None = None,
+        params: QueryParams | None = None,
+        data: Any = None,
+        json_data: Any = None,
+        cookies: Cookies | None = None,
+        headers: Headers | None = None,
+        proxy: str | None = None,
+        auth: BasicAuth | None = None,
+        timeout: float | None = None,
+        priority: int = 0,
+        delay: float | None = None,
+    ) -> None:
+        await self._queue.put(
+            PRPRequest(
+                priority=priority,
+                request=Request(
+                    method=method,
+                    url=url,
+                    params=params,
+                    data=data,
+                    json_data=json_data,
+                    cookies=cookies,
+                    headers=headers,
+                    auth=auth,
+                    proxy=proxy,
+                    timeout=timeout,
+                ),
+                request_params=RequestParams(
+                    callback=callback,
+                    cb_kwargs=cb_kwargs,
+                    errback=errback,
+                ),
+            )
+        )
+        if delay:
+            await asyncio.sleep(delay)

aioscrapper-0.1.0/src/aioscrapper/scrapper/__init__.py

@@ -0,0 +1,2 @@
+from .base import BaseScrapper
+from .executor import AIOScrapper

aioscrapper-0.1.0/src/aioscrapper/scrapper/base.py

@@ -0,0 +1,10 @@
+import abc
+
+
+class BaseScrapper(abc.ABC):
+    @abc.abstractmethod
+    async def start(self, *args, **kwargs) -> None: ...
+
+    async def initialize(self, *args, **kwargs) -> None: ...
+
+    async def close(self) -> None: ...

aioscrapper-0.1.0/src/aioscrapper/scrapper/executor.py

@@ -0,0 +1,146 @@
+import asyncio
+import time
+from logging import Logger, getLogger
+from types import TracebackType
+from typing import Type, Any
+
+from aiojobs import Scheduler
+
+from ..config import Config
+from ..helpers import get_func_kwargs
+from ..pipeline import PipelineDispatcher, BasePipeline
+from ..request_manager import RequestManager
+from ..request_sender import RequestSender
+from ..scrapper import BaseScrapper
+from ..session.aiohttp import AiohttpSession
+from ..types import RequestMiddleware, ResponseMiddleware
+
+
+class AIOScrapper:
+    def __init__(
+        self,
+        scrappers: list[BaseScrapper],
+        config: Config | None = None,
+        logger: Logger | None = None,
+    ) -> None:
+        self._start_time = time.time()
+        self._config = config or Config()
+        self._logger = logger or getLogger("aioscrapper")
+
+        self._scrappers = scrappers
+        self._request_outer_middlewares = []
+        self._request_inner_middlewares = []
+        self._response_middlewares = []
+
+        self._pipelines: dict[str, list[BasePipeline]] = {}
+        self._pipeline_dispatcher = PipelineDispatcher(
+            logger=self._logger.getChild("pipeline"), pipelines=self._pipelines
+        )
+
+        def _exception_handler(_, context: dict[str, Any]):
+            if "job" in context:
+                self._logger.error(f'{context['message']}: {context["exception"]}', extra={"context": context})
+            else:
+                self._logger.error("Unhandled error", extra={"context": context})
+
+        self._scheduler = Scheduler(
+            limit=self._config.scheduler.concurrent_requests,
+            pending_limit=self._config.scheduler.pending_requests,
+            close_timeout=self._config.scheduler.close_timeout,
+            exception_handler=_exception_handler,
+        )
+
+        self._request_queue = asyncio.PriorityQueue()
+        self._request_sender = RequestSender(self._request_queue)
+        self._request_manager = RequestManager(
+            logger=self._logger.getChild("request_worker"),
+            session=AiohttpSession(
+                timeout=self._config.session.request.timeout,
+                ssl=self._config.session.request.ssl,
+            ),
+            schedule_request=self._scheduler.spawn,
+            sender=self._request_sender,
+            queue=self._request_queue,
+            delay=self._config.session.request.delay,
+            shutdown_timeout=self._config.execution.shutdown_timeout,
+            srv_kwargs={"pipeline": self._pipeline_dispatcher},
+            request_outer_middlewares=self._request_outer_middlewares,
+            request_inner_middlewares=self._request_inner_middlewares,
+            response_middlewares=self._response_middlewares,
+        )
+
+        self._scrapper_kwargs = {"request_sender": self._request_sender, "pipeline": self._pipeline_dispatcher}
+
+    def add_pipeline(self, name: str, pipeline: BasePipeline) -> None:
+        if name not in self._pipelines:
+            self._pipelines[name] = [pipeline]
+        else:
+            self._pipelines[name].append(pipeline)
+
+    def add_outer_request_middlewares(self, *middlewares: RequestMiddleware) -> None:
+        self._request_outer_middlewares.extend(middlewares)
+
+    def add_inner_request_middlewares(self, *middlewares: RequestMiddleware) -> None:
+        self._request_inner_middlewares.extend(middlewares)
+
+    def add_response_middlewares(self, *middlewares: ResponseMiddleware) -> None:
+        self._response_middlewares.extend(middlewares)
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        await self.close()
+
+    async def start(self) -> None:
+        await self._pipeline_dispatcher.initialize()
+        self._request_manager.listen_queue()
+
+        for scrapper in self._scrappers:
+            await scrapper.initialize(**get_func_kwargs(scrapper.initialize, self._scrapper_kwargs))
+
+        await asyncio.gather(
+            *[scrapper.start(**get_func_kwargs(scrapper.start, self._scrapper_kwargs)) for scrapper in self._scrappers]
+        )
+
+    async def _shutdown(self) -> bool:
+        status = False
+        execution_timeout = (
+            max(self._config.execution.timeout - (time.time() - self._start_time), 0.1)
+            if self._config.execution.timeout
+            else None
+        )
+        while True:
+            if execution_timeout is not None and time.time() - self._start_time > execution_timeout:
+                self._logger.log(
+                    level=self._config.execution.log_level,
+                    msg=f"execution timeout: {self._config.execution.timeout}",
+                )
+                status = True
+                break
+            if len(self._scheduler) == 0 and self._request_queue.qsize() == 0:
+                break
+
+            await asyncio.sleep(self._config.execution.shutdown_check_interval)
+
+        return status
+
+    async def shutdown(self) -> None:
+        force = await self._shutdown()
+        await self._request_manager.shutdown(force)
+
+    async def close(self, shutdown: bool = True) -> None:
+        if shutdown:
+            await self.shutdown()
+
+        for scrapper in self._scrappers:
+            await scrapper.close()
+
+        await self._scheduler.close()
+        await self._request_manager.close()
+        await self._pipeline_dispatcher.close()

aioscrapper-0.1.0/src/aioscrapper/session/aiohttp.py

@@ -0,0 +1,46 @@
+from aiohttp import ClientSession, ClientTimeout, TCPConnector
+from aiohttp.helpers import BasicAuth
+
+from .base import BaseSession
+from ..types import Response, Request
+
+
+class AiohttpSession(BaseSession):
+    def __init__(self, timeout: float | None = None, ssl: bool | None = None, **kwargs) -> None:
+        super().__init__(timeout, ssl)
+        self._session = ClientSession(
+            timeout=ClientTimeout(total=timeout),
+            connector=TCPConnector(ssl=ssl) if ssl is not None else None,
+            **kwargs,
+        )
+
+    async def make_request(self, request: Request) -> Response:
+        async with self._session.request(
+            url=request.url,
+            method=request.method,
+            params=request.params,
+            data=request.data,
+            json=request.json_data,
+            cookies=request.cookies,
+            headers=request.headers,
+            proxy=request.proxy,
+            auth=(
+                BasicAuth(login=request.auth["username"], password=request.auth["password"])
+                if request.auth is not None
+                else None
+            ),
+            timeout=ClientTimeout(total=request.timeout) if request.timeout is not None else None,
+        ) as response:
+            return Response(
+                url=request.url,
+                method=request.method,
+                params=request.params,
+                status=response.status,
+                headers=dict(response.headers),
+                cookies={k: f"{v.key}={v.value}" for k, v in response.cookies.items()},
+                content=await response.read(),
+                content_type=response.headers.get("Content-Type"),
+            )
+
+    async def close(self) -> None:
+        await self._session.close()

aioscrapper-0.1.0/src/aioscrapper/session/base.py

@@ -0,0 +1,14 @@
+import abc
+
+from ..types import Request, Response
+
+
+class BaseSession(abc.ABC):
+    def __init__(self, timeout: float | None = None, ssl: bool | None = None) -> None:
+        self._timeout = timeout
+        self._ssl = ssl
+
+    @abc.abstractmethod
+    async def make_request(self, request: Request) -> Response: ...
+
+    async def close(self) -> None: ...

aioscrapper-0.1.0/src/aioscrapper/types/__init__.py

@@ -0,0 +1,12 @@
+from .middleware import RequestMiddleware, ResponseMiddleware
+from .session import (
+    QueryParams,
+    Cookies,
+    Headers,
+    BasicAuth,
+    Request,
+    RequestParams,
+    PRPRequest,
+    RequestQueue,
+    Response,
+)

aioscrapper-0.1.0/src/aioscrapper/types/middleware.py

@@ -0,0 +1,11 @@
+from typing import Protocol
+
+from .session import Request, RequestParams, Response
+
+
+class RequestMiddleware(Protocol):
+    async def __call__(self, request: Request, params: RequestParams) -> None: ...
+
+
+class ResponseMiddleware(Protocol):
+    async def __call__(self, params: RequestParams, response: Response) -> None: ...

aioscrapper-0.1.0/src/aioscrapper/types/session.py

@@ -0,0 +1,109 @@
+import asyncio
+import json
+from dataclasses import field, dataclass
+from typing import Union, Mapping, Any, Callable, Awaitable, TypedDict
+from urllib.parse import urlencode
+
+QueryParams = Mapping[str, Union[str, int, float]]
+
+Cookies = Mapping[str, str]
+Headers = Mapping[str, str]
+
+
+class BasicAuth(TypedDict):
+    username: str
+    password: str
+
+
+@dataclass(slots=True)
+class Request:
+    url: str
+    method: str
+    params: QueryParams | None = None
+    data: Any = None
+    json_data: Any = None
+    cookies: Cookies | None = None
+    headers: Headers | None = None
+    auth: BasicAuth | None = None
+    proxy: str | None = None
+    timeout: float | None = None
+
+    @property
+    def full_url(self) -> str:
+        return f"{self.url}{urlencode(self.params or {})}"
+
+
+@dataclass(slots=True)
+class RequestParams:
+    callback: Callable[..., Awaitable] | None = None
+    cb_kwargs: dict[str, Any] | None = None
+    errback: Callable[..., Awaitable] | None = None
+
+
+@dataclass(slots=True, order=True)
+class PRPRequest:
+    priority: int
+    request: Request = field(compare=False)
+    request_params: RequestParams = field(compare=False)
+
+
+RequestQueue = asyncio.PriorityQueue[PRPRequest | None]
+
+
+class Response:
+    def __init__(
+        self,
+        url: str,
+        method: str,
+        params: QueryParams | None,
+        status: int,
+        headers: Headers,
+        cookies: Cookies,
+        content: bytes,
+        content_type: str | None,
+    ) -> None:
+        self._url = url
+        self._method = method
+        self._params = params
+        self._status = status
+        self._headers = headers
+        self._cookies = cookies
+        self._content = content
+        self._content_type = content_type
+
+    @property
+    def url(self) -> str:
+        return self._url
+
+    @property
+    def method(self) -> str:
+        return self._method
+
+    @property
+    def params(self) -> QueryParams | None:
+        return self._params
+
+    @property
+    def status(self) -> int:
+        return self._status
+
+    @property
+    def headers(self) -> Headers | None:
+        return self._headers
+
+    @property
+    def cookies(self) -> Cookies | None:
+        return self._cookies
+
+    @property
+    def content_type(self) -> str | None:
+        return self._content_type
+
+    def bytes(self) -> bytes:
+        return self._content
+
+    def json(self) -> Any:
+        return json.loads(self._content) if self._content is not None else None
+
+    def text(self, encoding: str = "utf-8") -> str | None:
+        return self._content.decode(encoding) if self._content is not None else None

aioscrapper-0.1.0/src/aioscrapper.egg-info/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: aioscrapper
+Version: 0.1.0
+Summary: Async framework for building modular and scalable web scrapers.
+Author: darkstussy
+Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
+Project-URL: Issues, https://github.com/darkstussy/aioscrapper/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Framework :: AsyncIO
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiohttp[speedups]~=3.11.16
+Requires-Dist: aiojobs~=1.4.0
+Dynamic: license-file
+
+# aioscrapper
+
+**Asynchronous framework for building modular and scalable web scrapers.**
+
+![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
+![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
+
+## Features
+
+- 🚀 Fully asynchronous architecture powered by `aiohttp` and `aiojobs`
+- 🔧 Modular system with middleware support
+- 📦 Pipeline data processing
+- ⚙️ Flexible configuration
+- 🔄 Priority-based request queue management
+- 🛡️ Built-in error handling
+
+## Installation
+
+```bash
+pip install aioscrapper
+```
+
+## Requirements
+
+- Python 3.12 or higher
+- aiohttp
+- aiojobs
+
+## Quick Start
+
+```python
+import asyncio
+
+from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
+from aioscrapper.types import Response
+
+
+class Scrapper(BaseScrapper):
+    async def start(self, request_sender: RequestSender) -> None:
+        await request_sender(url="https://example.com", callback=self.parse)
+
+    async def parse(self, response: Response) -> None:
+        # handle response
+        pass
+
+
+async def main():
+    async with AIOScrapper(scrappers=[Scrapper()]) as scrapper:
+        await scrapper.start()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## License
+
+MIT License
+
+Copyright (c) 2025 darkstussy
+
+## Links
+
+- [GitHub](https://github.com/darkstussy/aioscrapper)
+- [Issues](https://github.com/darkstussy/aioscrapper/issues)

aioscrapper-0.1.0/src/aioscrapper.egg-info/SOURCES.txt

@@ -0,0 +1,26 @@
+LICENSE
+README.md
+pyproject.toml
+src/aioscrapper/__init__.py
+src/aioscrapper/config.py
+src/aioscrapper/exceptions.py
+src/aioscrapper/helpers.py
+src/aioscrapper/request_manager.py
+src/aioscrapper/request_sender.py
+src/aioscrapper.egg-info/PKG-INFO
+src/aioscrapper.egg-info/SOURCES.txt
+src/aioscrapper.egg-info/dependency_links.txt
+src/aioscrapper.egg-info/requires.txt
+src/aioscrapper.egg-info/top_level.txt
+src/aioscrapper/pipeline/__init__.py
+src/aioscrapper/pipeline/base.py
+src/aioscrapper/pipeline/dispatcher.py
+src/aioscrapper/scrapper/__init__.py
+src/aioscrapper/scrapper/base.py
+src/aioscrapper/scrapper/executor.py
+src/aioscrapper/session/__init__.py
+src/aioscrapper/session/aiohttp.py
+src/aioscrapper/session/base.py
+src/aioscrapper/types/__init__.py
+src/aioscrapper/types/middleware.py
+src/aioscrapper/types/session.py

aioscrapper-0.1.0/src/aioscrapper.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

aioscrapper-0.1.0/src/aioscrapper.egg-info/requires.txt

@@ -0,0 +1,2 @@
+aiohttp[speedups]~=3.11.16
+aiojobs~=1.4.0

aioscrapper-0.1.0/src/aioscrapper.egg-info/top_level.txt

@@ -0,0 +1 @@
+aioscrapper