aioscrapper 0.1.1__tar.gz → 0.1.2__tar.gz

{aioscrapper-0.1.1/src/aioscrapper.egg-info → aioscrapper-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aioscrapper
-Version: 0.1.1
+Version: 0.1.2
 Summary: Async framework for building modular and scalable web scrapers.
 Author: darkstussy
 Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
@@ -12,18 +12,28 @@ Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp[speedups]~=3.11.16
 Requires-Dist: aiojobs~=1.4.0
+Provides-Extra: dev
+Requires-Dist: flake8~=7.1.2; extra == "dev"
+Requires-Dist: black~=25.1.0; extra == "dev"
+Requires-Dist: pyright~=1.1.399; extra == "dev"
+Requires-Dist: aiohttp[speedups]~=3.11.16; extra == "dev"
+Requires-Dist: aiojobs~=1.4.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest~=8.3.5; extra == "test"
+Requires-Dist: pytest-asyncio~=0.26.0; extra == "test"
+Requires-Dist: aresponses~=3.0.0; extra == "test"
 Dynamic: license-file
 
 # aioscrapper
 
 **Asynchronous framework for building modular and scalable web scrapers.**
 
-![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![Python](https://img.shields.io/badge/python-3.10%2B-blue)
 ![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
 ![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
 
@@ -44,7 +54,7 @@ pip install aioscrapper
 
 ## Requirements
 
-- Python 3.12 or higher
+- Python 3.10 or higher
 - aiohttp
 - aiojobs
 
@@ -53,8 +63,8 @@ pip install aioscrapper
 ```python
 import asyncio
 
-from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
-from aioscrapper.types import Response
+from aioscrapper import BaseScrapper, AIOScrapper
+from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):

{aioscrapper-0.1.1 → aioscrapper-0.1.2}/README.md

@@ -2,7 +2,7 @@
 
 **Asynchronous framework for building modular and scalable web scrapers.**
 
-![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![Python](https://img.shields.io/badge/python-3.10%2B-blue)
 ![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
 ![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
 
@@ -23,7 +23,7 @@ pip install aioscrapper
 
 ## Requirements
 
-- Python 3.12 or higher
+- Python 3.10 or higher
 - aiohttp
 - aiojobs
 
@@ -32,8 +32,8 @@ pip install aioscrapper
 ```python
 import asyncio
 
-from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
-from aioscrapper.types import Response
+from aioscrapper import BaseScrapper, AIOScrapper
+from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):

{aioscrapper-0.1.1/src → aioscrapper-0.1.2}/aioscrapper/scrapper/executor.py

@@ -32,9 +32,7 @@ class AIOScrapper:
         self._response_middlewares = []
 
         self._pipelines: dict[str, list[BasePipeline]] = {}
-        self._pipeline_dispatcher = PipelineDispatcher(
-            logger=self._logger.getChild("pipeline"), pipelines=self._pipelines
-        )
+        self._pipeline_dispatcher = PipelineDispatcher(self._logger.getChild("pipeline"), pipelines=self._pipelines)
 
         def _exception_handler(_, context: dict[str, Any]):
             if "job" in context:

{aioscrapper-0.1.1 → aioscrapper-0.1.2/aioscrapper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aioscrapper
-Version: 0.1.1
+Version: 0.1.2
 Summary: Async framework for building modular and scalable web scrapers.
 Author: darkstussy
 Project-URL: Homepage, https://github.com/darkstussy/aioscrapper
@@ -12,18 +12,28 @@ Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp[speedups]~=3.11.16
 Requires-Dist: aiojobs~=1.4.0
+Provides-Extra: dev
+Requires-Dist: flake8~=7.1.2; extra == "dev"
+Requires-Dist: black~=25.1.0; extra == "dev"
+Requires-Dist: pyright~=1.1.399; extra == "dev"
+Requires-Dist: aiohttp[speedups]~=3.11.16; extra == "dev"
+Requires-Dist: aiojobs~=1.4.0; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest~=8.3.5; extra == "test"
+Requires-Dist: pytest-asyncio~=0.26.0; extra == "test"
+Requires-Dist: aresponses~=3.0.0; extra == "test"
 Dynamic: license-file
 
 # aioscrapper
 
 **Asynchronous framework for building modular and scalable web scrapers.**
 
-![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![Python](https://img.shields.io/badge/python-3.10%2B-blue)
 ![License](https://img.shields.io/github/license/darkstussy/aioscrapper)
 ![Version](https://img.shields.io/github/v/tag/darkstussy/aioscrapper?label=version)
 
@@ -44,7 +54,7 @@ pip install aioscrapper
 
 ## Requirements
 
-- Python 3.12 or higher
+- Python 3.10 or higher
 - aiohttp
 - aiojobs
 
@@ -53,8 +63,8 @@ pip install aioscrapper
 ```python
 import asyncio
 
-from aioscrapper import BaseScrapper, AIOScrapper, RequestSender
-from aioscrapper.types import Response
+from aioscrapper import BaseScrapper, AIOScrapper
+from aioscrapper.types import Response, RequestSender
 
 
 class Scrapper(BaseScrapper):

aioscrapper-0.1.2/aioscrapper.egg-info/SOURCES.txt

@@ -0,0 +1,27 @@
+LICENSE
+README.md
+pyproject.toml
+aioscrapper/__init__.py
+aioscrapper/config.py
+aioscrapper/exceptions.py
+aioscrapper/helpers.py
+aioscrapper.egg-info/PKG-INFO
+aioscrapper.egg-info/SOURCES.txt
+aioscrapper.egg-info/dependency_links.txt
+aioscrapper.egg-info/requires.txt
+aioscrapper.egg-info/top_level.txt
+aioscrapper/pipeline/__init__.py
+aioscrapper/pipeline/base.py
+aioscrapper/pipeline/dispatcher.py
+aioscrapper/scrapper/__init__.py
+aioscrapper/scrapper/base.py
+aioscrapper/scrapper/executor.py
+aioscrapper/scrapper/request_manager.py
+aioscrapper/session/__init__.py
+aioscrapper/session/aiohttp.py
+aioscrapper/session/base.py
+aioscrapper/types/__init__.py
+aioscrapper/types/middleware.py
+aioscrapper/types/session.py
+tests/test_error.py
+tests/test_success.py

aioscrapper-0.1.2/aioscrapper.egg-info/requires.txt

@@ -0,0 +1,14 @@
+aiohttp[speedups]~=3.11.16
+aiojobs~=1.4.0
+
+[dev]
+flake8~=7.1.2
+black~=25.1.0
+pyright~=1.1.399
+aiohttp[speedups]~=3.11.16
+aiojobs~=1.4.0
+
+[test]
+pytest~=8.3.5
+pytest-asyncio~=0.26.0
+aresponses~=3.0.0

aioscrapper-0.1.2/pyproject.toml

@@ -0,0 +1,63 @@
+[project]
+name = "aioscrapper"
+version = "0.1.2"
+authors = [{ name = "darkstussy" }, ]
+description = "Async framework for building modular and scalable web scrapers."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "aiohttp[speedups]~=3.11.16",
+    "aiojobs~=1.4.0",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Framework :: AsyncIO",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+]
+
+[project.urls]
+Homepage = "https://github.com/darkstussy/aioscrapper"
+Issues = "https://github.com/darkstussy/aioscrapper/issues"
+
+[project.optional-dependencies]
+dev = [
+    "flake8~=7.1.2",
+    "black~=25.1.0",
+    "pyright~=1.1.399",
+    "aiohttp[speedups]~=3.11.16",
+    "aiojobs~=1.4.0",
+]
+test = [
+    "pytest~=8.3.5",
+    "pytest-asyncio~=0.26.0",
+    "aresponses~=3.0.0"
+]
+
+[tool.black]
+line-length = 120
+include = '\.pyi?$'
+exclude = '''
+    /(
+        \.eggs/|
+        \.git/|
+        \.hg/|
+        \.mypy_cache/|
+        \.tox/|
+        \.venv/|
+        \venv/|
+        _build/|
+        buck-out/|
+        build/|
+        dist/
+    )'''
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = [
+    "tests",
+]
+asyncio_default_fixture_loop_scope = "function"

aioscrapper-0.1.2/tests/test_error.py

@@ -0,0 +1,37 @@
+import pytest
+from aresponses import ResponsesMockServer
+
+from aioscrapper import AIOScrapper
+from aioscrapper.exceptions import ClientException, HTTPException
+from aioscrapper.scrapper.base import BaseScrapper
+from aioscrapper.types import RequestSender
+
+
+class Scrapper(BaseScrapper):
+    def __init__(self):
+        self.status = None
+        self.response_data = None
+
+    async def start(self, request_sender: RequestSender) -> None:
+        await request_sender(url="https://api.test.com/v1", errback=self.errback)
+
+    async def errback(self, exc: ClientException) -> None:
+        if isinstance(exc, HTTPException):
+            self.status = exc.status_code
+            self.response_data = exc.message
+
+
+@pytest.mark.asyncio
+async def test_error(aresponses: ResponsesMockServer):
+    def handle_request(request):
+        return aresponses.Response(status=500, text="Internal Server Error")
+
+    aresponses.add("api.test.com", "/v1", "GET", response=handle_request)  # pyright: ignore
+
+    scrapper = Scrapper()
+    async with AIOScrapper(scrappers=[scrapper]) as executor:
+        await executor.start()
+
+    assert scrapper.status == 500
+    assert scrapper.response_data == "Internal Server Error"
+    aresponses.assert_plan_strictly_followed()

aioscrapper-0.1.2/tests/test_success.py

@@ -0,0 +1,29 @@
+import pytest
+from aresponses import ResponsesMockServer
+
+from aioscrapper import AIOScrapper
+from aioscrapper.scrapper.base import BaseScrapper
+from aioscrapper.types import Response, RequestSender
+
+
+class Scrapper(BaseScrapper):
+    def __init__(self):
+        self.response_data = None
+
+    async def start(self, request_sender: RequestSender) -> None:
+        await request_sender(url="https://api.test.com/v1", callback=self.parse)
+
+    async def parse(self, response: Response) -> None:
+        self.response_data = response.json()
+
+
+@pytest.mark.asyncio
+async def test_success(aresponses: ResponsesMockServer):
+    aresponses.add("api.test.com", "/v1", "GET", response={"status": "OK"})  # pyright: ignore
+
+    scrapper = Scrapper()
+    async with AIOScrapper(scrappers=[scrapper]) as executor:
+        await executor.start()
+
+    assert scrapper.response_data == {"status": "OK"}
+    aresponses.assert_plan_strictly_followed()

aioscrapper-0.1.1/pyproject.toml

@@ -1,24 +0,0 @@
-[project]
-name = "aioscrapper"
-version = "0.1.1"
-authors = [{ name = "darkstussy" }, ]
-description = "Async framework for building modular and scalable web scrapers."
-readme = "README.md"
-requires-python = ">=3.12"
-dependencies = [
-    "aiohttp[speedups] ~= 3.11.16",
-    "aiojobs ~= 1.4.0",
-]
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.12",
-    "Framework :: AsyncIO",
-    "Intended Audience :: Developers",
-    "Operating System :: OS Independent",
-    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
-    "Topic :: Software Development :: Libraries :: Application Frameworks",
-]
-
-[project.urls]
-Homepage = "https://github.com/darkstussy/aioscrapper"
-Issues = "https://github.com/darkstussy/aioscrapper/issues"

aioscrapper-0.1.1/src/aioscrapper.egg-info/SOURCES.txt

@@ -1,25 +0,0 @@
-LICENSE
-README.md
-pyproject.toml
-src/aioscrapper/__init__.py
-src/aioscrapper/config.py
-src/aioscrapper/exceptions.py
-src/aioscrapper/helpers.py
-src/aioscrapper.egg-info/PKG-INFO
-src/aioscrapper.egg-info/SOURCES.txt
-src/aioscrapper.egg-info/dependency_links.txt
-src/aioscrapper.egg-info/requires.txt
-src/aioscrapper.egg-info/top_level.txt
-src/aioscrapper/pipeline/__init__.py
-src/aioscrapper/pipeline/base.py
-src/aioscrapper/pipeline/dispatcher.py
-src/aioscrapper/scrapper/__init__.py
-src/aioscrapper/scrapper/base.py
-src/aioscrapper/scrapper/executor.py
-src/aioscrapper/scrapper/request_manager.py
-src/aioscrapper/session/__init__.py
-src/aioscrapper/session/aiohttp.py
-src/aioscrapper/session/base.py
-src/aioscrapper/types/__init__.py
-src/aioscrapper/types/middleware.py
-src/aioscrapper/types/session.py

aioscrapper-0.1.1/src/aioscrapper.egg-info/requires.txt

@@ -1,2 +0,0 @@
-aiohttp[speedups]~=3.11.16
-aiojobs~=1.4.0