aioscrapper 0.1.1__tar.gz → 0.2.0__tar.gz

{aioscrapper-0.1.1/src/aioscrapper.egg-info → aioscrapper-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aioscrapper
-Version: 0.1.1
+Version: 0.2.0
 Summary: Async framework for building modular and scalable web scrapers.
 Author: darkstussy
 Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
@@ -12,18 +12,28 @@ Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp[speedups]~=3.11.16
 Requires-Dist: aiojobs~=1.4.0
+Provides-Extra: dev
+Requires-Dist: flake8~=7.1.2; extra == "dev"
+Requires-Dist: black~=25.1.0; extra == "dev"
+Requires-Dist: pyright~=1.1.399; extra == "dev"
+Requires-Dist: aiohttp[speedups]~=3.11.16; extra == "dev"
+Requires-Dist: aiojobs~=1.4.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest~=8.3.5; extra == "test"
+Requires-Dist: pytest-asyncio~=0.26.0; extra == "test"
+Requires-Dist: aresponses~=3.0.0; extra == "test"
 Dynamic: license-file
 
 # aioscrapper
 
 **Asynchronous framework for building modular and scalable web scrapers.**
 
-![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![Python](https://img.shields.io/badge/python-3.10%2B-blue)
 ![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
 ![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
 
@@ -44,7 +54,7 @@ pip install aioscrapper
 
 ## Requirements
 
-- Python 3.12 or higher
+- Python 3.10 or higher
 - aiohttp
 - aiojobs
 
@@ -53,13 +63,13 @@ pip install aioscrapper
 ```python
 import asyncio
 
-from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
-from aioscrapper.types import Response
+from aioscrapper import BaseScrapper, AIOScrapper
+from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):
-    async def start(self, request_sender: RequestSender) -> None:
-        await request_sender(url="https://example.com", callback=self.parse)
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://example.com", callback=self.parse)
 
     async def parse(self, response: Response) -> None:
         # handle response

{aioscrapper-0.1.1 → aioscrapper-0.2.0}/README.md

@@ -2,7 +2,7 @@
 
 **Asynchronous framework for building modular and scalable web scrapers.**
 
-![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![Python](https://img.shields.io/badge/python-3.10%2B-blue)
 ![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
 ![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
 
@@ -23,7 +23,7 @@ pip install aioscrapper
 
 ## Requirements
 
-- Python 3.12 or higher
+- Python 3.10 or higher
 - aiohttp
 - aiojobs
 
@@ -32,13 +32,13 @@ pip install aioscrapper
 ```python
 import asyncio
 
-from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
-from aioscrapper.types import Response
+from aioscrapper import BaseScrapper, AIOScrapper
+from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):
-    async def start(self, request_sender: RequestSender) -> None:
-        await request_sender(url="https://example.com", callback=self.parse)
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://example.com", callback=self.parse)
 
     async def parse(self, response: Response) -> None:
         # handle response

aioscrapper-0.2.0/aioscrapper/pipeline/__init__.py

@@ -0,0 +1 @@
+from .base import BasePipeline

{aioscrapper-0.1.1/src → aioscrapper-0.2.0}/aioscrapper/pipeline/base.py

@@ -1,11 +1,7 @@
 import abc
-from typing import TypeVar, Generic, Protocol
-
-
-class BaseItem(Protocol):
-    @property
-    def pipeline_name(self) -> str: ...
+from typing import TypeVar, Generic
 
+from ..types import BaseItem
 
 ItemType = TypeVar("ItemType", bound=BaseItem)
 

{aioscrapper-0.1.1/src → aioscrapper-0.2.0}/aioscrapper/pipeline/dispatcher.py

@@ -10,7 +10,7 @@ class PipelineDispatcher:
         self._logger = logger
         self._pipelines = pipelines
 
-    async def put_item(self, item: BaseItem) -> BaseItem:
+    async def __call__(self, item: BaseItem) -> BaseItem:
         self._logger.debug(f"pipeline item received: {item}")
         try:
             pipelines = self._pipelines[item.pipeline_name]

aioscrapper-0.2.0/aioscrapper/scrapper/base.py

@@ -0,0 +1,30 @@
+import abc
+
+
+class BaseScrapper(abc.ABC):
+    @abc.abstractmethod
+    async def start(self, *args, **kwargs) -> None:
+        """
+        Starts the scrapper.
+
+        This method is called to start the scraper by sending the initial requests required for its operation.
+        """
+        ...
+
+    async def initialize(self, *args, **kwargs) -> None:
+        """
+        Initializes the scrapper.
+
+        This method is called before starting the scrapper. It should be used to initialize any
+        necessary state or resources required by the scrapper.
+        """
+        ...
+
+    async def close(self, *args, **kwargs) -> None:
+        """
+        Closes the scrapper.
+
+        This method is called to clean up any resources created by the scrapper after it has finished
+        running.
+        """
+        ...

{aioscrapper-0.1.1/src → aioscrapper-0.2.0}/aioscrapper/scrapper/executor.py

@@ -9,7 +9,8 @@ from aiojobs import Scheduler
 from .request_manager import RequestManager
 from ..config import Config
 from ..helpers import get_func_kwargs
-from ..pipeline import PipelineDispatcher, BasePipeline
+from ..pipeline import BasePipeline
+from ..pipeline.dispatcher import PipelineDispatcher
 from ..scrapper import BaseScrapper
 from ..session.aiohttp import AiohttpSession
 from ..types import RequestMiddleware, ResponseMiddleware
@@ -32,9 +33,7 @@ class AIOScrapper:
         self._response_middlewares = []
 
         self._pipelines: dict[str, list[BasePipeline]] = {}
-        self._pipeline_dispatcher = PipelineDispatcher(
-            logger=self._logger.getChild("pipeline"), pipelines=self._pipelines
-        )
+        self._pipeline_dispatcher = PipelineDispatcher(self._logger.getChild("pipeline"), pipelines=self._pipelines)
 
         def _exception_handler(_, context: dict[str, Any]):
             if "job" in context:
@@ -96,7 +95,7 @@ class AIOScrapper:
         await self._pipeline_dispatcher.initialize()
         self._request_manager.listen_queue()
 
-        scrapper_kwargs = {"request_sender": self._request_manager.sender, "pipeline": self._pipeline_dispatcher}
+        scrapper_kwargs = {"send_request": self._request_manager.sender, "pipeline": self._pipeline_dispatcher}
         for scrapper in self._scrappers:
             await scrapper.initialize(**get_func_kwargs(scrapper.initialize, scrapper_kwargs))
 

{aioscrapper-0.1.1/src → aioscrapper-0.2.0}/aioscrapper/types/__init__.py

@@ -1,4 +1,5 @@
 from .middleware import RequestMiddleware, ResponseMiddleware
+from .pipeline import BaseItem, Pipeline
 from .session import (
     QueryParams,
     Cookies,

aioscrapper-0.2.0/aioscrapper/types/pipeline.py

@@ -0,0 +1,10 @@
+from typing import Protocol
+
+
+class BaseItem(Protocol):
+    @property
+    def pipeline_name(self) -> str: ...
+
+
+class Pipeline(Protocol):
+    async def __call__(self, item: BaseItem) -> BaseItem: ...

{aioscrapper-0.1.1 → aioscrapper-0.2.0/aioscrapper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aioscrapper
-Version: 0.1.1
+Version: 0.2.0
 Summary: Async framework for building modular and scalable web scrapers.
 Author: darkstussy
 Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
@@ -12,18 +12,28 @@ Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp[speedups]~=3.11.16
 Requires-Dist: aiojobs~=1.4.0
+Provides-Extra: dev
+Requires-Dist: flake8~=7.1.2; extra == "dev"
+Requires-Dist: black~=25.1.0; extra == "dev"
+Requires-Dist: pyright~=1.1.399; extra == "dev"
+Requires-Dist: aiohttp[speedups]~=3.11.16; extra == "dev"
+Requires-Dist: aiojobs~=1.4.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest~=8.3.5; extra == "test"
+Requires-Dist: pytest-asyncio~=0.26.0; extra == "test"
+Requires-Dist: aresponses~=3.0.0; extra == "test"
 Dynamic: license-file
 
 # aioscrapper
 
 **Asynchronous framework for building modular and scalable web scrapers.**
 
-![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![Python](https://img.shields.io/badge/python-3.10%2B-blue)
 ![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
 ![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
 
@@ -44,7 +54,7 @@ pip install aioscrapper
 
 ## Requirements
 
-- Python 3.12 or higher
+- Python 3.10 or higher
 - aiohttp
 - aiojobs
 
@@ -53,13 +63,13 @@ pip install aioscrapper
 ```python
 import asyncio
 
-from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
-from aioscrapper.types import Response
+from aioscrapper import BaseScrapper, AIOScrapper
+from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):
-    async def start(self, request_sender: RequestSender) -> None:
-        await request_sender(url="https://example.com", callback=self.parse)
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://example.com", callback=self.parse)
 
     async def parse(self, response: Response) -> None:
         # handle response

aioscrapper-0.2.0/aioscrapper.egg-info/SOURCES.txt

@@ -0,0 +1,28 @@
+LICENSE
+README.md
+pyproject.toml
+aioscrapper/__init__.py
+aioscrapper/config.py
+aioscrapper/exceptions.py
+aioscrapper/helpers.py
+aioscrapper.egg-info/PKG-INFO
+aioscrapper.egg-info/SOURCES.txt
+aioscrapper.egg-info/dependency_links.txt
+aioscrapper.egg-info/requires.txt
+aioscrapper.egg-info/top_level.txt
+aioscrapper/pipeline/__init__.py
+aioscrapper/pipeline/base.py
+aioscrapper/pipeline/dispatcher.py
+aioscrapper/scrapper/__init__.py
+aioscrapper/scrapper/base.py
+aioscrapper/scrapper/executor.py
+aioscrapper/scrapper/request_manager.py
+aioscrapper/session/__init__.py
+aioscrapper/session/aiohttp.py
+aioscrapper/session/base.py
+aioscrapper/types/__init__.py
+aioscrapper/types/middleware.py
+aioscrapper/types/pipeline.py
+aioscrapper/types/session.py
+tests/test_error.py
+tests/test_success.py

aioscrapper-0.2.0/aioscrapper.egg-info/requires.txt

@@ -0,0 +1,14 @@
+aiohttp[speedups]~=3.11.16
+aiojobs~=1.4.0
+
+[dev]
+flake8~=7.1.2
+black~=25.1.0
+pyright~=1.1.399
+aiohttp[speedups]~=3.11.16
+aiojobs~=1.4.0
+
+[test]
+pytest~=8.3.5
+pytest-asyncio~=0.26.0
+aresponses~=3.0.0

aioscrapper-0.2.0/pyproject.toml

@@ -0,0 +1,63 @@
+[project]
+name = "aioscrapper"
+version = "0.2.0"
+authors = [{ name = "darkstussy" }, ]
+description = "Async framework for building modular and scalable web scrapers."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "aiohttp[speedups]~=3.11.16",
+    "aiojobs~=1.4.0",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Framework :: AsyncIO",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+]
+
+[project.urls]
+Homepage = "https://github.com/darkstussy/aioscrapper"
+Issues = "https://github.com/darkstussy/aioscrapper/issues"
+
+[project.optional-dependencies]
+dev = [
+    "flake8~=7.1.2",
+    "black~=25.1.0",
+    "pyright~=1.1.399",
+    "aiohttp[speedups]~=3.11.16",
+    "aiojobs~=1.4.0",
+]
+test = [
+    "pytest~=8.3.5",
+    "pytest-asyncio~=0.26.0",
+    "aresponses~=3.0.0"
+]
+
+[tool.black]
+line-length = 120
+include = '\.pyi?$'
+exclude = '''
+    /(
+        \.eggs/|
+        \.git/|
+        \.hg/|
+        \.mypy_cache/|
+        \.tox/|
+        \.venv/|
+        \venv/|
+        _build/|
+        buck-out/|
+        build/|
+        dist/
+    )'''
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = [
+    "tests",
+]
+asyncio_default_fixture_loop_scope = "function"

aioscrapper-0.2.0/tests/test_error.py

@@ -0,0 +1,37 @@
+import pytest
+from aresponses import ResponsesMockServer
+
+from aioscrapper import AIOScrapper
+from aioscrapper.exceptions import ClientException, HTTPException
+from aioscrapper.scrapper.base import BaseScrapper
+from aioscrapper.types import RequestSender
+
+
+class Scrapper(BaseScrapper):
+    def __init__(self):
+        self.status = None
+        self.response_data = None
+
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://api.test.com/v1", errback=self.errback)
+
+    async def errback(self, exc: ClientException) -> None:
+        if isinstance(exc, HTTPException):
+            self.status = exc.status_code
+            self.response_data = exc.message
+
+
+@pytest.mark.asyncio
+async def test_error(aresponses: ResponsesMockServer):
+    def handle_request(request):
+        return aresponses.Response(status=500, text="Internal Server Error")
+
+    aresponses.add("api.test.com", "/v1", "GET", response=handle_request)  # pyright: ignore
+
+    scrapper = Scrapper()
+    async with AIOScrapper(scrappers=[scrapper]) as executor:
+        await executor.start()
+
+    assert scrapper.status == 500
+    assert scrapper.response_data == "Internal Server Error"
+    aresponses.assert_plan_strictly_followed()

aioscrapper-0.2.0/tests/test_success.py

@@ -0,0 +1,29 @@
+import pytest
+from aresponses import ResponsesMockServer
+
+from aioscrapper import AIOScrapper
+from aioscrapper.scrapper.base import BaseScrapper
+from aioscrapper.types import Response, RequestSender
+
+
+class Scrapper(BaseScrapper):
+    def __init__(self):
+        self.response_data = None
+
+    async def start(self, send_request: RequestSender) -> None:
+        await send_request(url="https://api.test.com/v1", callback=self.parse)
+
+    async def parse(self, response: Response) -> None:
+        self.response_data = response.json()
+
+
+@pytest.mark.asyncio
+async def test_success(aresponses: ResponsesMockServer):
+    aresponses.add("api.test.com", "/v1", "GET", response={"status": "OK"})  # pyright: ignore
+
+    scrapper = Scrapper()
+    async with AIOScrapper(scrappers=[scrapper]) as executor:
+        await executor.start()
+
+    assert scrapper.response_data == {"status": "OK"}
+    aresponses.assert_plan_strictly_followed()

aioscrapper-0.1.1/pyproject.toml

@@ -1,24 +0,0 @@
-[project]
-name = "aioscrapper"
-version = "0.1.1"
-authors = [{ name = "darkstussy" }, ]
-description = "Async framework for building modular and scalable web scrapers."
-readme = "README.md"
-requires-python = ">=3.12"
-dependencies = [
-    "aiohttp[speedups] ~= 3.11.16",
-    "aiojobs ~= 1.4.0",
-]
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.12",
-    "Framework :: AsyncIO",
-    "Intended Audience :: Developers",
-    "Operating System :: OS Independent",
-    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
-    "Topic :: Software Development :: Libraries :: Application Frameworks",
-]
-
-[project.urls]
-Homepage = "https://github.com/darkstussy/aioscrapper"
-Issues = "https://github.com/darkstussy/aioscrapper/issues"

aioscrapper-0.1.1/src/aioscrapper/pipeline/__init__.py

@@ -1,2 +0,0 @@
-from .base import BasePipeline, BaseItem
-from .dispatcher import PipelineDispatcher

aioscrapper-0.1.1/src/aioscrapper/scrapper/base.py

@@ -1,10 +0,0 @@
-import abc
-
-
-class BaseScrapper(abc.ABC):
-    @abc.abstractmethod
-    async def start(self, *args, **kwargs) -> None: ...
-
-    async def initialize(self, *args, **kwargs) -> None: ...
-
-    async def close(self, *args, **kwargs) -> None: ...

aioscrapper-0.1.1/src/aioscrapper.egg-info/SOURCES.txt

@@ -1,25 +0,0 @@
-LICENSE
-README.md
-pyproject.toml
-src/aioscrapper/__init__.py
-src/aioscrapper/config.py
-src/aioscrapper/exceptions.py
-src/aioscrapper/helpers.py
-src/aioscrapper.egg-info/PKG-INFO
-src/aioscrapper.egg-info/SOURCES.txt
-src/aioscrapper.egg-info/dependency_links.txt
-src/aioscrapper.egg-info/requires.txt
-src/aioscrapper.egg-info/top_level.txt
-src/aioscrapper/pipeline/__init__.py
-src/aioscrapper/pipeline/base.py
-src/aioscrapper/pipeline/dispatcher.py
-src/aioscrapper/scrapper/__init__.py
-src/aioscrapper/scrapper/base.py
-src/aioscrapper/scrapper/executor.py
-src/aioscrapper/scrapper/request_manager.py
-src/aioscrapper/session/__init__.py
-src/aioscrapper/session/aiohttp.py
-src/aioscrapper/session/base.py
-src/aioscrapper/types/__init__.py
-src/aioscrapper/types/middleware.py
-src/aioscrapper/types/session.py

aioscrapper-0.1.1/src/aioscrapper.egg-info/requires.txt

@@ -1,2 +0,0 @@
-aiohttp[speedups]~=3.11.16
-aiojobs~=1.4.0