fastcrawl 0.1.0__tar.gz

fastcrawl-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Ilarion Kuleshov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fastcrawl-0.1.0/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.1
+Name: fastcrawl
+Version: 0.1.0
+Summary: Fast and asynchronous web crawling and scraping library for Python.
+License: MIT
+Author: ilarionkuleshov
+Author-email: ilarion.kuleshov@gmail.com
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: parsel (>=1.9.1,<2.0.0)
+Requires-Dist: pydantic (>=2.10.4,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.7.0,<3.0.0)
+Description-Content-Type: text/markdown
+
+# FastCrawl
+
+<p align="left">
+<a href="https://github.com/ilarionkuleshov/fastcrawl/actions/workflows/code-quality.yml/?query=event%3Apush+branch%3Amain">
+    <img src="https://github.com/ilarionkuleshov/fastcrawl/actions/workflows/code-quality.yml/badge.svg?event=push&branch=main">
+</a>
+<a href="https://coverage-badge.samuelcolvin.workers.dev/redirect/ilarionkuleshov/fastcrawl">
+    <img src="https://coverage-badge.samuelcolvin.workers.dev/ilarionkuleshov/fastcrawl.svg">
+</a>
+<a href="https://pypi.org/project/fastcrawl">
+    <img src="https://img.shields.io/pypi/v/fastcrawl?color=%2334D058">
+</a>
+<a href="https://pypi.org/project/fastcrawl">
+    <img src="https://img.shields.io/pypi/pyversions/fastcrawl.svg?color=%2334D058">
+</a>
+</p>
+
+FastCrawl is a Python library for web crawling and scraping, inspired by [Scrapy](https://github.com/scrapy/scrapy) but designed to run seamlessly in asynchronous applications. Built on top of [Httpx](https://github.com/encode/httpx), it provides a lightweight foundation for creating custom crawlers based on the `BaseCrawler` class. The library supports defining custom Pipelines for processing scraped items, which can be easily implemented by extending the `BasePipeline` class. While its functionality is still growing, FastCrawl offers flexible settings options for the crawler, HTTP client, requests, and more.
+
+
+## Installation
+FastCrawl is available on PyPI and can be installed using pip:
+```bash
+pip install fastcrawl
+```
+
+
+## Usage
+Here is a simple example of how to create a custom crawler using FastCrawl:
+```python
+import asyncio
+from typing import AsyncIterator
+
+from pydantic import BaseModel
+from fastcrawl import BaseCrawler, BasePipeline, Request, Response, CrawlerSettings
+
+
+class ExampleItem(BaseModel):
+    title: str
+
+
+class ExamplePipeline(BasePipeline[ExampleItem]):
+    async def process_item(self, item: ExampleItem) -> ExampleItem | None:
+        self.logger.info(f"Processing item: {item}")
+        return item
+
+
+class ExampleCrawler(BaseCrawler):
+    settings = CrawlerSettings(
+        pipelines=[ExamplePipeline()],
+    )
+
+    async def generate_requests(self) -> AsyncIterator[Request]:
+        yield Request(url="http://example.com/", callback=self.parse)
+
+    async def parse(self, response: Response) -> AsyncIterator[ExampleItem]:
+        title = response.selector.xpath(".//h1/text()").get() or "unknown"
+        yield ExampleItem(title=title)
+
+
+asyncio.run(ExampleCrawler().run())
+```
+
+In this example, we define a custom `ExampleItem` model, a `ExamplePipeline` for processing scraped items, and an `ExampleCrawler` that generates requests and parses responses.
+
+Crawler can be set up using the `settings` class attribute or by passing a `CrawlerSettings` instance to the constructor. See the model definition for all available settings.
+
+Method `generate_requests` is executed once at the beginning of the crawl and should yield `Request` objects to start the crawl. Each request should have a callback function that will be called with the `Response` object when the request is completed.
+
+In request callbacks, you can use the `Response` object to extract data using XPath selectors or other methods. Also you can yield another requests to follow links or scrape paginated content.
+
+In pipelines, you can implement custom logic for processing items, such as saving them to a database, sending them to a message queue, or logging them. When defining a pipeline, you specify the type of item it will work with. The example specifies the `ExampleItem` pydantic model, but you can use any type you need. If the crawler returned an item of a different type, the pipeline would be skipped for that item.
+
+
+## License
+This project is licensed under the MIT License.
+

fastcrawl-0.1.0/README.md

@@ -0,0 +1,76 @@
+# FastCrawl
+
+<p align="left">
+<a href="https://github.com/ilarionkuleshov/fastcrawl/actions/workflows/code-quality.yml/?query=event%3Apush+branch%3Amain">
+    <img src="https://github.com/ilarionkuleshov/fastcrawl/actions/workflows/code-quality.yml/badge.svg?event=push&branch=main">
+</a>
+<a href="https://coverage-badge.samuelcolvin.workers.dev/redirect/ilarionkuleshov/fastcrawl">
+    <img src="https://coverage-badge.samuelcolvin.workers.dev/ilarionkuleshov/fastcrawl.svg">
+</a>
+<a href="https://pypi.org/project/fastcrawl">
+    <img src="https://img.shields.io/pypi/v/fastcrawl?color=%2334D058">
+</a>
+<a href="https://pypi.org/project/fastcrawl">
+    <img src="https://img.shields.io/pypi/pyversions/fastcrawl.svg?color=%2334D058">
+</a>
+</p>
+
+FastCrawl is a Python library for web crawling and scraping, inspired by [Scrapy](https://github.com/scrapy/scrapy) but designed to run seamlessly in asynchronous applications. Built on top of [Httpx](https://github.com/encode/httpx), it provides a lightweight foundation for creating custom crawlers based on the `BaseCrawler` class. The library supports defining custom Pipelines for processing scraped items, which can be easily implemented by extending the `BasePipeline` class. While its functionality is still growing, FastCrawl offers flexible settings options for the crawler, HTTP client, requests, and more.
+
+
+## Installation
+FastCrawl is available on PyPI and can be installed using pip:
+```bash
+pip install fastcrawl
+```
+
+
+## Usage
+Here is a simple example of how to create a custom crawler using FastCrawl:
+```python
+import asyncio
+from typing import AsyncIterator
+
+from pydantic import BaseModel
+from fastcrawl import BaseCrawler, BasePipeline, Request, Response, CrawlerSettings
+
+
+class ExampleItem(BaseModel):
+    title: str
+
+
+class ExamplePipeline(BasePipeline[ExampleItem]):
+    async def process_item(self, item: ExampleItem) -> ExampleItem | None:
+        self.logger.info(f"Processing item: {item}")
+        return item
+
+
+class ExampleCrawler(BaseCrawler):
+    settings = CrawlerSettings(
+        pipelines=[ExamplePipeline()],
+    )
+
+    async def generate_requests(self) -> AsyncIterator[Request]:
+        yield Request(url="http://example.com/", callback=self.parse)
+
+    async def parse(self, response: Response) -> AsyncIterator[ExampleItem]:
+        title = response.selector.xpath(".//h1/text()").get() or "unknown"
+        yield ExampleItem(title=title)
+
+
+asyncio.run(ExampleCrawler().run())
+```
+
+In this example, we define a custom `ExampleItem` model, a `ExamplePipeline` for processing scraped items, and an `ExampleCrawler` that generates requests and parses responses.
+
+Crawler can be set up using the `settings` class attribute or by passing a `CrawlerSettings` instance to the constructor. See the model definition for all available settings.
+
+Method `generate_requests` is executed once at the beginning of the crawl and should yield `Request` objects to start the crawl. Each request should have a callback function that will be called with the `Response` object when the request is completed.
+
+In request callbacks, you can use the `Response` object to extract data using XPath selectors or other methods. Also you can yield another requests to follow links or scrape paginated content.
+
+In pipelines, you can implement custom logic for processing items, such as saving them to a database, sending them to a message queue, or logging them. When defining a pipeline, you specify the type of item it will work with. The example specifies the `ExampleItem` pydantic model, but you can use any type you need. If the crawler returned an item of a different type, the pipeline would be skipped for that item.
+
+
+## License
+This project is licensed under the MIT License.

fastcrawl-0.1.0/fastcrawl/__init__.py

@@ -0,0 +1,10 @@
+from .base_crawler import BaseCrawler
+from .base_pipeline import BasePipeline
+from .models import (
+    CrawlerHttpClientSettings,
+    CrawlerLoggingSettings,
+    CrawlerSettings,
+    CrawlerStats,
+    Request,
+    Response,
+)

fastcrawl-0.1.0/fastcrawl/base_crawler.py

@@ -0,0 +1,139 @@
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from typing import AsyncIterator, Optional
+
+from httpx import AsyncClient, Limits
+
+from fastcrawl.models import CrawlerSettings, CrawlerStats, Request, Response
+
+
+class BaseCrawler(ABC):
+    """Base for all crawlers.
+
+    Args:
+        settings (Optional[CrawlerSettings]): Settings for the crawler.
+            If not provided, the default settings will be used. Default is None.
+
+    Attributes:
+        logger (logging.Logger): Logger for the crawler.
+        settings (CrawlerSettings): Settings for the crawler. Override it to set custom settings.
+        stats (CrawlerStats): Statistics for the crawler.
+
+    """
+
+    logger: logging.Logger
+    settings: CrawlerSettings = CrawlerSettings()
+    stats: CrawlerStats
+
+    _queue: asyncio.Queue
+    _http_client: AsyncClient
+
+    def __init__(self, settings: Optional[CrawlerSettings] = None) -> None:
+        if settings:
+            self.settings = settings
+
+        if self.settings.setup_logging:
+            self._setup_logging()
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.stats = CrawlerStats()
+        self._queue = asyncio.Queue()
+        self._http_client = AsyncClient(**self._get_http_client_kwargs())
+
+    def _setup_logging(self) -> None:
+        """Sets up logging for the crawler."""
+        logging.basicConfig(
+            level=self.settings.logging.level,
+            format=self.settings.logging.format,
+        )
+        logging.getLogger("asyncio").setLevel(self.settings.logging.level_asyncio)
+        logging.getLogger("httpx").setLevel(self.settings.logging.level_httpx)
+        logging.getLogger("httpcore").setLevel(self.settings.logging.level_httpcore)
+
+    def _get_http_client_kwargs(self):
+        kwargs = self.settings.http_client.model_dump()
+        kwargs["params"] = kwargs.pop("query_params")
+        kwargs["trust_env"] = False
+        kwargs["limits"] = Limits(
+            max_connections=kwargs.pop("max_connections"),
+            max_keepalive_connections=kwargs.pop("max_keepalive_connections"),
+            keepalive_expiry=kwargs.pop("keepalive_expiry"),
+        )
+        return kwargs
+
+    @abstractmethod
+    async def generate_requests(self) -> AsyncIterator[Request]:
+        """Yields requests to be processed."""
+        if False:  # pylint: disable=W0125  # pragma: no cover
+            yield Request(url="https://example.com/", callback=lambda _: None)  # just a stub for mypy
+
+    async def run(self) -> None:
+        """Runs the crawler."""
+        self.logger.info("Running crawler with settings: %s", self.settings.model_dump_json(indent=2))
+        self.stats.start_crawling()
+        for pipeline in self.settings.pipelines:
+            await pipeline.on_crawler_start()
+
+        async for request in self.generate_requests():
+            await self._queue.put(request)
+
+        workers = [asyncio.create_task(self._worker()) for _ in range(self.settings.workers)]
+        await self._queue.join()
+        for worker in workers:
+            worker.cancel()
+
+        await self._http_client.aclose()
+
+        for pipeline in self.settings.pipelines:
+            await pipeline.on_crawler_finish()
+        self.stats.finish_crawling()
+        self.logger.info("Crawling finished with stats: %s", self.stats.model_dump_json(indent=2))
+
+    async def _worker(self) -> None:
+        """Worker to process requests from the queue."""
+        while True:
+            request = await self._queue.get()
+            try:
+                await self._process_request(request)
+            except Exception as exc:  # pylint: disable=W0718
+                self.logger.error("Error processing request %s: %s", request, exc)
+            finally:
+                self._queue.task_done()
+
+    async def _process_request(self, request: Request) -> None:
+        """Executes the request, callback and processes the results.
+
+        Args:
+            request (Request): The request to process.
+
+        """
+        self.logger.debug("Processing request: %s", request)
+        self.stats.add_request()
+
+        request_kwargs = request.model_dump(exclude_none=True, exclude={"callback"})
+        if "query_params" in request_kwargs:
+            request_kwargs["params"] = request_kwargs.pop("query_params")
+        if "form_data" in request_kwargs:
+            request_kwargs["data"] = request_kwargs.pop("form_data")
+        if "json_data" in request_kwargs:
+            request_kwargs["json"] = request_kwargs.pop("json_data")
+        httpx_response = await self._http_client.request(**request_kwargs)
+
+        response = await Response.from_httpx_response(httpx_response, request)
+        self.logger.debug("Got response: %s", response)
+        self.stats.add_response(response.status_code)
+
+        callback_result = request.callback(response)
+        if hasattr(callback_result, "__aiter__"):
+            async for item in callback_result:
+                if isinstance(item, Request):
+                    await self._queue.put(item)
+                elif item is not None:
+                    for pipeline in self.settings.pipelines:
+                        item = await pipeline.process_item_with_check(item)
+                        if item is None:
+                            break
+                    else:
+                        self.stats.add_item()
+        else:
+            await callback_result

fastcrawl-0.1.0/fastcrawl/base_pipeline.py

@@ -0,0 +1,60 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Generic, Optional, TypeVar, get_args
+
+T = TypeVar("T")
+
+
+class BasePipeline(ABC, Generic[T]):
+    """Base for all pipelines.
+
+    Attributes:
+        logger (logging.Logger): Logger for the crawler.
+
+    """
+
+    logger: logging.Logger
+    _expected_type: type[T]
+
+    def __init__(self) -> None:
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self._expected_type = get_args(self.__orig_bases__[0])[0]  # type: ignore[attr-defined]  # pylint: disable=E1101
+
+    async def process_item_with_check(self, item: Any) -> Any:
+        """Processes an item with type checking.
+
+        Note:
+            If the item is not an instance of the expected type, it will be returned as is.
+
+        Args:
+            item (Any): Item to process.
+
+        Returns:
+            Any: Processed item or the item itself.
+
+        """
+        if not isinstance(item, self._expected_type):
+            return item
+        return await self.process_item(item)
+
+    @abstractmethod
+    async def process_item(self, item: T) -> Optional[T]:
+        """Processes an item returned by the crawler.
+
+        Args:
+            item (T): Item to process.
+
+        Returns:
+            T: Processed item.
+            None: If the item should be dropped and not passed to the next pipelines.
+
+        """
+
+    async def on_crawler_start(self) -> None:
+        """Called when the crawler starts."""
+
+    async def on_crawler_finish(self) -> None:
+        """Called when the crawler finishes."""
+
+    def __str__(self) -> str:
+        return f"<{self.__class__.__name__}[{self._expected_type.__name__}]>"

fastcrawl-0.1.0/fastcrawl/models/__init__.py

@@ -0,0 +1,8 @@
+from .crawler_settings import (
+    CrawlerHttpClientSettings,
+    CrawlerLoggingSettings,
+    CrawlerSettings,
+)
+from .crawler_stats import CrawlerStats
+from .request import Request
+from .response import Response

fastcrawl-0.1.0/fastcrawl/models/crawler_settings.py

@@ -0,0 +1,103 @@
+from typing import Annotated, Optional, Union
+
+from dotenv import find_dotenv
+from httpx import URL
+from pydantic import BaseModel, ConfigDict
+from pydantic.functional_serializers import PlainSerializer
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from fastcrawl.base_pipeline import BasePipeline
+from fastcrawl.types import Auth, Cookies, Headers, QueryParams
+
+
+class CrawlerLoggingSettings(BaseModel):
+    """Crawler logging settings model.
+
+    Attributes:
+        level (str): Logging level for the crawler. Default is "INFO".
+        format (str): Logging format for the crawler.
+            Default is "%(asctime)s [%(name)s] %(levelname)s: %(message)s".
+        level_asyncio (str): Logging level for asyncio library. Default is "WARNING".
+        level_httpx (str): Logging level for httpx library. Default is "WARNING".
+        level_httpcore (str): Logging level for httpcore library. Default is "WARNING".
+
+    """
+
+    level: str = "INFO"
+    format: str = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
+    level_asyncio: str = "WARNING"
+    level_httpx: str = "WARNING"
+    level_httpcore: str = "WARNING"
+
+
+class CrawlerHttpClientSettings(BaseModel):
+    """Crawler HTTP client settings model.
+
+    Attributes:
+        base_url (Union[URL, str]): Base URL for the HTTP client. Default is "".
+        auth (Optional[Auth]): Authentication for the HTTP client. Default is None.
+        query_params (Optional[QueryParams]): Query parameters for the HTTP client. Default is None.
+        headers (Optional[Headers]): Headers for the HTTP client. Default is None.
+        cookies (Optional[Cookies]): Cookies for the HTTP client. Default is None.
+        verify (bool): Whether to verify SSL certificates. Default is True.
+        http1 (bool): Whether to use HTTP/1.1. Default is True.
+        http2 (bool): Whether to use HTTP/2. Default is False.
+        proxy (Optional[Union[URL, str]]): Proxy for the HTTP client. Default is None.
+        timeout (float): Timeout for the HTTP client. Default is 5.0.
+        max_connections (Optional[int]): Specifies the maximum number of concurrent connections allowed. Default is 100.
+        max_keepalive_connections (Optional[int]): The maximum number of keep-alive connections the pool can maintain.
+            Must not exceed `max_connections`. Default is 20.
+        keepalive_expiry (Optional[float]): The maximum duration in seconds that a keep-alive
+            connection can remain idle. Default is 5.0.
+        follow_redirects (bool): Whether to follow redirects. Default is False.
+        max_redirects (int): Maximum number of redirects to follow. Default is 20.
+        default_encoding (str): Default encoding for the HTTP client. Default is "utf-8".
+
+    """
+
+    base_url: Union[URL, str] = ""
+    auth: Optional[Auth] = None
+    query_params: Optional[QueryParams] = None
+    headers: Optional[Headers] = None
+    cookies: Optional[Cookies] = None
+    verify: bool = True
+    http1: bool = True
+    http2: bool = False
+    proxy: Optional[Union[URL, str]] = None
+    timeout: float = 5.0
+    max_connections: Optional[int] = 100
+    max_keepalive_connections: Optional[int] = 20
+    keepalive_expiry: Optional[float] = 5.0
+    follow_redirects: bool = False
+    max_redirects: int = 20
+    default_encoding: str = "utf-8"
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class CrawlerSettings(BaseSettings):
+    """Crawler settings model.
+
+    Attributes:
+        workers (int): Number of workers to process requests. Default is 15.
+        pipelines (list[BasePipeline]): List of pipelines to process responses.
+            Pipelines will be executed in the order they are defined. Default is [].
+        setup_logging (bool): Whether to setup logging for the crawler. Default is True.
+        logging (CrawlerLoggingSettings): Logging settings for the crawler. Default is CrawlerLoggingSettings().
+        http_client (CrawlerHttpClientSettings): HTTP client settings for the crawler.
+            Default is CrawlerHttpClientSettings().
+
+    """
+
+    workers: int = 15
+    pipelines: list[Annotated[BasePipeline, PlainSerializer(str)]] = []
+    setup_logging: bool = True
+    logging: CrawlerLoggingSettings = CrawlerLoggingSettings()
+    http_client: CrawlerHttpClientSettings = CrawlerHttpClientSettings()
+
+    model_config = SettingsConfigDict(
+        env_file=find_dotenv(),
+        env_prefix="fastcrawl_",
+        env_nested_delimiter="__",
+        extra="ignore",
+    )

fastcrawl-0.1.0/fastcrawl/models/crawler_stats.py

@@ -0,0 +1,58 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class CrawlerStats(BaseModel):
+    """Crawler statistics model.
+
+    Note:
+        To update statistics while crawling, use corresponding methods.
+
+    Attributes:
+        started_at (Optional[datetime]): The time when the crawling started. Default is None.
+        finished_at (Optional[datetime]): The time when the crawling finished. Default is None.
+        requests (Optional[int]): The number of requests made during the crawling. Default is None.
+        responses_by_codes (Optional[dict[int, int]]): The number of responses by status code. Default is None.
+        items (Optional[int]): The number of items crawled. Default is None.
+
+    """
+
+    started_at: Optional[datetime] = None
+    finished_at: Optional[datetime] = None
+    requests: Optional[int] = None
+    responses_by_codes: Optional[dict[int, int]] = None
+    items: Optional[int] = None
+
+    def start_crawling(self) -> None:
+        """Sets the time when the crawling started."""
+        self.started_at = datetime.now()
+
+    def finish_crawling(self) -> None:
+        """Sets the time when the crawling finished."""
+        self.finished_at = datetime.now()
+
+    def add_request(self) -> None:
+        """Increases the number of requests made during the crawling."""
+        if self.requests:
+            self.requests += 1
+        else:
+            self.requests = 1
+
+    def add_response(self, status_code: int) -> None:
+        """Increases the number of responses by status code."""
+        if self.responses_by_codes is None:
+            self.responses_by_codes = {}
+
+        if status_code in self.responses_by_codes:
+            self.responses_by_codes[status_code] += 1
+        else:
+            self.responses_by_codes[status_code] = 1
+
+    def add_item(self) -> None:
+        """Increases the number of items crawled."""
+        if self.items:
+            self.items += 1
+        else:
+            self.items = 1

fastcrawl-0.1.0/fastcrawl/models/request.py

@@ -0,0 +1,53 @@
+from typing import Optional, Union
+
+from httpx import URL
+from pydantic import BaseModel, ConfigDict
+
+from fastcrawl.types import (
+    Auth,
+    Cookies,
+    Files,
+    FormData,
+    Headers,
+    JsonData,
+    QueryParams,
+    RequestCallback,
+)
+
+
+class Request(BaseModel):
+    """Request model.
+
+    Attributes:
+        method (str): HTTP method. Default is "GET".
+        url (Union[URL, str]): URL to request.
+        callback (RequestCallback): Callback to process the response.
+        query_params (Optional[QueryParams]): Query parameters for the URL. Default is None.
+        headers (Optional[Headers]): Headers for the request. Default is None.
+        cookies (Optional[Cookies]): Cookies for the request. Default is None.
+        form_data (Optional[FormData]): Form data for the request. Default is None.
+        json_data (Optional[JsonData]): JSON data for the request. Default is None.
+        files (Optional[Files]): Files for the request. Default is None.
+        auth (Optional[Auth]): Authentication credentials. Default is None.
+        timeout (Optional[float]): Timeout for the request in seconds. Default is None.
+        follow_redirects (Optional[bool]): Whether to follow redirects. Default is None.
+
+    """
+
+    method: str = "GET"
+    url: Union[URL, str]
+    callback: RequestCallback
+    query_params: Optional[QueryParams] = None
+    headers: Optional[Headers] = None
+    cookies: Optional[Cookies] = None
+    form_data: Optional[FormData] = None
+    json_data: Optional[JsonData] = None
+    files: Optional[Files] = None
+    auth: Optional[Auth] = None
+    timeout: Optional[float] = None
+    follow_redirects: Optional[bool] = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    def __str__(self) -> str:
+        return f"<{self.__class__.__name__}({self.method}, {self.url})>"

fastcrawl-0.1.0/fastcrawl/models/response.py

@@ -0,0 +1,74 @@
+import json
+from typing import Any, Optional
+
+from httpx import URL
+from httpx import Response as HttpxResponse
+from httpx import ResponseNotRead
+from parsel import Selector
+from pydantic import BaseModel, ConfigDict, PrivateAttr
+
+from fastcrawl.models.request import Request
+
+
+class Response(BaseModel):
+    """Response model.
+
+    Attributes:
+        url (URL): URL of the response.
+        status_code (int): Status code of the response.
+        content (bytes): Content of the response.
+        text (str): Text of the response.
+        headers (Optional[dict[str, str]]): Headers of the response. Default is None.
+        cookies (Optional[dict[str, str]]): Cookies of the response. Default is None.
+        request (Request): Request used to fetch the response.
+
+    """
+
+    url: URL
+    status_code: int
+    content: bytes
+    text: str
+    headers: Optional[dict[str, str]] = None
+    cookies: Optional[dict[str, str]] = None
+    request: Request
+    _cached_selector: Optional[Selector] = PrivateAttr(default=None)
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @classmethod
+    async def from_httpx_response(cls, httpx_response: HttpxResponse, request: Request) -> "Response":
+        """Returns new instance from an httpx response.
+
+        Args:
+            httpx_response (HttpxResponse): Response from httpx.
+            request (Request): Request used to fetch the response.
+
+        """
+        try:
+            content = httpx_response.content
+        except ResponseNotRead:
+            content = await httpx_response.aread()
+
+        return cls(
+            url=httpx_response.url,
+            status_code=httpx_response.status_code,
+            content=content,
+            text=httpx_response.text,
+            headers=dict(httpx_response.headers),
+            cookies=dict(httpx_response.cookies),
+            request=request,
+        )
+
+    def get_json_data(self) -> Any:
+        """Returns JSON data from the response."""
+        return json.loads(self.text)
+
+    @property
+    def selector(self) -> Selector:
+        """Selector for xpath and css queries."""
+        if self._cached_selector is None:
+            self._cached_selector = Selector(text=self.text)
+        return self._cached_selector
+
+    def __str__(self) -> str:
+        return f"<{self.__class__.__name__}({self.status_code}, {self.url})>"

fastcrawl-0.1.0/fastcrawl/types.py

@@ -0,0 +1,25 @@
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncIterator,
+    Callable,
+    Coroutine,
+    Mapping,
+    Optional,
+    Sequence,
+    Union,
+)
+
+if TYPE_CHECKING:
+    from fastcrawl.models import Response  # pragma: no cover
+
+PrimitiveData = Optional[Union[str, int, float, bool]]
+
+RequestCallback = Callable[["Response"], Union[Coroutine[Any, Any, Optional[AsyncIterator[Any]]], AsyncIterator[Any]]]
+QueryParams = Mapping[str, Union[PrimitiveData, Sequence[PrimitiveData]]]
+Headers = Mapping[str, str]
+Cookies = Mapping[str, str]
+FormData = Mapping[str, Any]
+JsonData = Any
+Files = Mapping[str, bytes]
+Auth = tuple[str, str]

fastcrawl-0.1.0/pyproject.toml

@@ -0,0 +1,57 @@
+[tool.poetry]
+name = "fastcrawl"
+version = "0.1.0"
+description = "Fast and asynchronous web crawling and scraping library for Python."
+authors = ["ilarionkuleshov <ilarion.kuleshov@gmail.com>"]
+readme = "README.md"
+license = "MIT"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+httpx = "^0.28.1"
+pydantic = "^2.10.4"
+parsel = "^1.9.1"
+pydantic-settings = "^2.7.0"
+
+[tool.poetry.group.dev.dependencies]
+isort = "^5.13.2"
+autoflake = "^2.3.1"
+black = "^24.10.0"
+flake8 = "^7.1.1"
+flake8-pyproject = "^1.2.3"
+pylint = "^3.3.3"
+mypy = "^1.14.0"
+pytest = "^8.3.4"
+pytest-asyncio = "^0.25.0"
+coverage = "^7.6.10"
+freezegun = "^1.5.1"
+pytest-httpx = "^0.35.0"
+smokeshow = "^0.4.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.isort]
+profile = "black"
+
+[tool.autoflake]
+remove-all-unused-imports = true
+recursive = true
+remove-unused-variables = true
+ignore-init-module-imports = true
+in-place = true
+
+[tool.black]
+line-length = 120
+
+[tool.flake8]
+max-line-length = 120
+per-file-ignores = "__init__.py:F401"
+
+[tool.pylint]
+max-line-length = 120
+disable="C0114"
+
+[tool.coverage.html]
+directory = ".coverage_html"