careful 0.2.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

careful/httpx/__init__.py

@@ -1,11 +1,20 @@
+import os
+from urllib.parse import urlparse
 from .retries import make_retry_client, retry_default_rule
 from .throttle import make_throttled_client
+from .robots import (
+    make_robots_txt_client,
+    RobotExclusionError,
+    RobotsRejectFunc,
+    raise_robots_txt,
+)
 from .dev_cache import (
     make_dev_caching_client,
     MemoryCache,
     FileCache,
     SqliteCache,
-    CacheStorageBase,
+    CacheStorage,
+    CacheResponse,
     _cache_200s,
     _default_keyfunc,
 )
@@ -20,11 +29,14 @@ def make_careful_client(
     retry_wait_seconds: float = 10,
     should_retry: ResponsePredicate = retry_default_rule,
     requests_per_minute: int = 0,
-    cache_storage: CacheStorageBase = None,
+    cache_storage: CacheStorage | None = None,
     cache_write_only: bool = False,
     should_cache: ResponsePredicate = _cache_200s,
     cache_keyfunc: CacheKeyfunc = _default_keyfunc,
-):
+    check_robots_txt: bool = False,
+    robots_txt_user_agent: str | None = None,
+    robots_txt_on_reject: RobotsRejectFunc = raise_robots_txt,
+) -> Client:
     """
     This function patches an `httpx.Client` so that all requests made with the client support
      [retries](#retries), [throttling](#throttling), and [development caching](#development-caching).
@@ -51,43 +63,6 @@ def make_careful_client(
 
         cache_keyfunc: Function that takes request details and returns a unique cache key.
 
-    ## Retries
-
-    If `retry_attempts` is set, responses will be passed to `should_retry`.
-    Responses that are rejected (return `True`) will be retried after a wait based on
-    `retry_wait_seconds`.
-    Each retry will wait twice as long as the one before.
-
-    ## Throttling
-
-    If `requests_per_minute` is set, standard (non-retry) requests will automatically
-    sleep for a short period to target the given rate.
-
-    For example, at 30rpm, the sleep time on a fast request will be close to 2 seconds.
-
-    ## Development Caching
-
-    Why **development caching?**
-
-    This feature is named as a reminder that **this is not true HTTP caching**, which
-    should take various headers into account. Look at libraries like [hishel](https://hishel.com) if that's what you are after.
-
-    The purpose of this feature is to allow you to cache all of your HTTP requests during development.
-    Often when writing a scraper or crawler, you wind up hitting the site you are working on more often than you'd like-- each time you iterate on your code you're likely making redundant requests to pages that haven't changed.
-
-    By caching all successful requests (configurable with the `should_cache` parameter),
-    you can easily re-run scrapers without making redundant HTTP requests.
-    This means faster development time & happier upstream servers.
-
-    To enable development caching, assign a [`MemoryCache`][careful.httpx.MemoryCache],
-    [`FileCache`][careful.httpx.FileCache], or [`SqliteCache`][careful.httpx.SqliteCache] to
-    the `cache_storage` property of a `scrapelib.Scraper`.
-
-    ---
-
-    When multiple features are applied, the order of wrapping ensures that:
-       - the cache is checked first, and bypasses throttling if hit
-       - retries use their own delays, but not throttled separately
     """
     if client is None:
         client = Client()
@@ -104,7 +79,7 @@ def make_careful_client(
         client = make_throttled_client(
             client=client, requests_per_minute=requests_per_minute
         )
-    # caching on top layer, so cache will be checked first
+    # caching on top layer, so cache will be checked before throttling/etc.
     if cache_storage:
         client = make_dev_caching_client(
             client=client,
@@ -113,15 +88,111 @@ def make_careful_client(
             should_cache=should_cache,
             write_only=cache_write_only,
         )
+    # robots.txt before cache
+    if check_robots_txt:
+        client = make_robots_txt_client(
+            client=client,
+            as_user_agent=robots_txt_user_agent,
+            on_rejection=robots_txt_on_reject,
+        )
 
     return client
 
 
+def _int_env(var_name: str, default: int) -> int:
+    return int(os.environ.get(var_name, default))
+
+
+def _float_env(var_name: str, default: float) -> float:
+    return float(os.environ.get(var_name, default))
+
+
+def _bool_env(var_name: str, default: bool) -> bool:
+    """helper function for bool env vars"""
+    return bool(os.environ.get(var_name, "T" if default else ""))
+
+
+def _cache_env(var_name: str, default: CacheStorage | None) -> CacheStorage | None:
+    """
+    helper function that reads cache as a protocol string
+    """
+    cache_str = os.environ.get(var_name)
+    if not cache_str:
+        return default
+    parsed = urlparse(cache_str)
+    if parsed.scheme == "memory":
+        return MemoryCache()
+    elif parsed.scheme == "file":
+        return FileCache(parsed.path)
+    elif parsed.scheme == "sqlite":
+        return SqliteCache(parsed.path)
+
+
+def make_careful_client_from_env(
+    *,
+    client: Client | None = None,
+    retry_attempts: int = 0,
+    retry_wait_seconds: float = 10,
+    should_retry: ResponsePredicate = retry_default_rule,
+    requests_per_minute: int = 0,
+    cache_storage: CacheStorage | None = None,
+    cache_write_only: bool = False,
+    should_cache: ResponsePredicate = _cache_200s,
+    cache_keyfunc: CacheKeyfunc = _default_keyfunc,
+    check_robots_txt: bool = False,
+    robots_txt_user_agent: str | None = None,
+    robots_txt_on_reject: RobotsRejectFunc = raise_robots_txt,
+) -> Client:
+    """
+    Make a careful client from environment variables.
+
+    Any set environment variables will override parameters if set.
+
+    Numeric:
+        - CAREFUL_RETRY_ATTEMPTS
+        - CAREFUL_RETRY_WAIT_SECONDS
+        - CAREFUL_REQUESTS_PER_MINUTE
+        - CAREFUL_CHECK_ROBOTS_TXT
+    Booleans (any non-empty value is true):
+        - CAREFUL_CACHE_WRITE_ONLY
+        - CAREFUL_ROBOTS_TXT_USER_AGENT
+    Cache:
+        - CAREFUL_CACHE, which can be:
+            memory://
+            cache://path/to/db.sqlite3
+            file://path/to/directory
+
+    Function parameters do not have environment variables.
+    """
+    return make_careful_client(
+        client=client,
+        retry_attempts=_int_env("CAREFUL_RETRY_ATTEMPTS", retry_attempts),
+        retry_wait_seconds=_float_env("CAREFUL_RETRY_WAIT_SECONDS", retry_wait_seconds),
+        should_retry=should_retry,
+        requests_per_minute=_int_env(
+            "CAREFUL_REQUESTS_PER_MINUTE", requests_per_minute
+        ),
+        cache_storage=_cache_env("CAREFUL_CACHE", cache_storage),
+        cache_write_only=_bool_env("CAREFUL_CACHE_WRITE_ONLY", cache_write_only),
+        should_cache=should_cache,
+        cache_keyfunc=cache_keyfunc,
+        check_robots_txt=_bool_env("CAREFUL_CHECK_ROBOTS_TXT", check_robots_txt),
+        robots_txt_user_agent=os.environ.get(
+            "CAREFUL_ROBOTS_TXT_USER_AGENT", robots_txt_user_agent
+        ),
+        robots_txt_on_reject=robots_txt_on_reject,
+    )
+
+
 __all__ = [
+    "make_careful_client",
     "make_retry_client",
     "make_throttled_client",
     "make_dev_caching_client",
+    "make_robots_txt_client",
     "MemoryCache",
     "FileCache",
     "SqliteCache",
+    "CacheResponse",
+    "RobotExclusionError",
 ]

careful/httpx/_types.py

@@ -3,4 +3,4 @@ from typing import Callable
 
 ResponsePredicate = Callable[[Response], bool]
 
-CacheKeyfunc = Callable[[str,str,dict], str]
+CacheKeyfunc = Callable[[str, str, dict], str]

careful/httpx/dev_cache.py

@@ -1,4 +1,3 @@
-import abc
 import types
 import functools
 import logging
@@ -8,10 +7,27 @@ import glob
 import hashlib
 import sqlite3
 import json
+from typing import cast, Protocol, Callable
+from ._types import ResponsePredicate, CacheKeyfunc
 
 from httpx import Client, Response, Request
 
-log = logging.getLogger("httpx")
+log = logging.getLogger("careful")
+
+
+class CacheStorage(Protocol):
+    def get(self, key: str) -> None | Response: ...
+    def set(self, key: str, response: Response) -> None: ...
+
+
+class DevCacheClient(Protocol):
+    _retry_attempts: int
+    _cache_storage: CacheStorage
+    _write_only: bool
+    _should_cache: ResponsePredicate
+    _cache_keyfunc: CacheKeyfunc
+    _no_cache_request: Callable
+    request: Callable
 
 
 def _default_keyfunc(
@@ -28,7 +44,7 @@ def _default_keyfunc(
     if method.lower() != "get":
         return None
 
-    return Request(url=url, method=method, params=params).url
+    return str(Request(url=url, method=method, params=params).url)
 
 
 def _cache_200s(response: Response) -> bool:
@@ -40,7 +56,11 @@ def _cache_200s(response: Response) -> bool:
     return response.status_code == 200
 
 
-def _cached_request(client: Client, *args, **kwargs):
+class CacheResponse(Response):
+    fromcache: bool
+
+
+def _cached_request(client: DevCacheClient, *args, **kwargs) -> CacheResponse:
     method, url = args
     request_key = client._cache_keyfunc(method, url, kwargs["params"])
 
@@ -50,30 +70,28 @@ def _cached_request(client: Client, *args, **kwargs):
         cached_resp = client._cache_storage.get(request_key)
 
     if cached_resp:
-        # resp = cast(CacheResponse, resp_maybe)
         log.info("using cached response request_key=%s", request_key)
-        cached_resp.fromcache = True
-        resp = cached_resp
+        new_resp = cast(CacheResponse, cached_resp)
+        new_resp.fromcache = True
     else:
-        resp = client._no_cache_request(*args, **kwargs)
+        new_resp = cast(CacheResponse, client._no_cache_request(*args, **kwargs))
+        new_resp.fromcache = False
         # save to cache if request and response meet criteria
-        log.debug("XX %s %s", request_key, client._should_cache(resp))
-        if request_key and client._should_cache(resp):
-            client._cache_storage.set(request_key, resp)
+        if request_key and client._should_cache(new_resp):
+            client._cache_storage.set(request_key, new_resp)
             log.info("caching response request_key=%s", request_key)
-        resp.fromcache = False
 
-    return resp
+    return new_resp
 
 
 def make_dev_caching_client(
     *,
+    cache_storage: CacheStorage,
     client: Client | None = None,
-    cache_storage=None,
     cache_keyfunc=_default_keyfunc,
     should_cache=_cache_200s,
     write_only=False,
-):
+) -> Client:
     """
     Returns an enhanced `httpx.Client` where requests are saved to a
     specified cache.
@@ -98,29 +116,21 @@ def make_dev_caching_client(
     if client is None:
         client = Client()
 
-    client._cache_storage = cache_storage
-    client._cache_keyfunc = cache_keyfunc
-    client._should_cache = should_cache
-    client._write_only = write_only
+    tclient = cast(DevCacheClient, client)
+
+    tclient._cache_storage = cache_storage
+    tclient._cache_keyfunc = cache_keyfunc
+    tclient._should_cache = should_cache
+    tclient._write_only = write_only
 
-    client._no_cache_request = client.request
-    client.request = types.MethodType(
+    tclient._no_cache_request = client.request
+    tclient.request = types.MethodType(
         functools.wraps(client.request)(_cached_request), client
     )
     return client
 
 
-class CacheStorageBase(abc.ABC):
-    @abc.abstractmethod
-    def get(self, key: str) -> None | Response:
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def set(self, key: str, response: Response) -> None:
-        raise NotImplementedError()
-
-
-class MemoryCache(CacheStorageBase):
+class MemoryCache(CacheStorage):
     """
     In memory cache for request responses.
 
@@ -144,13 +154,13 @@ class MemoryCache(CacheStorageBase):
         self.cache[key] = response
 
 
-class FileCache(CacheStorageBase):
+class FileCache(CacheStorage):
     """
     File-based cache for request responses.
 
     Parameters:
         cache_dir: directory for storing responses
-        
+
     Example:
 
         make_careful_client(
@@ -260,7 +270,7 @@ class FileCache(CacheStorageBase):
             os.remove(fname)
 
 
-class SqliteCache(CacheStorageBase):
+class SqliteCache(CacheStorage):
     """
     sqlite cache for request responses.
 

careful/httpx/retries.py

@@ -2,25 +2,37 @@ import time
 import types
 import functools
 import logging
+from typing import Protocol, Callable, cast
 from httpx import Client, Response, HTTPError
+from ._types import ResponsePredicate
 
-log = logging.getLogger("httpx")
+log = logging.getLogger("careful")
+
+
+class RetryClient(Protocol):
+    _retry_attempts: int
+    _retry_wait_seconds: float
+    _should_retry: ResponsePredicate
+    _no_retry_request: Callable
+    request: Callable
 
 
 def retry_default_rule(response: Response) -> bool:
-    # default behavior is to retry 400s and 500s but not 404s
+    """default behavior is to retry 400s and 500s but not 404s"""
     return response.status_code >= 400 and response.status_code != 404
 
 
 def retry_only_500s(response: Response) -> bool:
+    """retry all status codes that are 500 or above"""
     return response.status_code >= 500
 
 
 def retry_all_400s_500s(response: Response) -> bool:
+    """retry all status codes that are 400 or above"""
     return response.status_code >= 400
 
 
-def _retry_request(client: Client, *args, **kwargs):
+def _retry_request(client: RetryClient, *args, **kwargs) -> Response:
     # the retry loop
     tries = 0
     exception_raised = None
@@ -75,15 +87,17 @@ def make_retry_client(
     attempts: int = 1,
     wait_seconds: float = 10,
     should_retry=retry_default_rule,
-):
+) -> Client:
     if client is None:
         client = Client()
-    client._retry_attempts = max(0, attempts)
-    client._retry_wait_seconds = wait_seconds
-    client._should_retry = should_retry
 
-    client._no_retry_request = client.request
-    client.request = types.MethodType(
+    tclient = cast(RetryClient, client)
+    tclient._retry_attempts = max(0, attempts)
+    tclient._retry_wait_seconds = wait_seconds
+    tclient._should_retry = should_retry
+
+    tclient._no_retry_request = client.request
+    tclient.request = types.MethodType(
         functools.wraps(client.request)(_retry_request), client
     )
 

careful/httpx/robots.py

@@ -0,0 +1,72 @@
+import types
+import functools
+import warnings
+from urllib.robotparser import RobotFileParser
+from typing import Protocol, cast, Callable
+from httpx import Client, Response, URL
+
+
+class RobotExclusionError(Exception):
+    pass
+
+
+def raise_robots_txt(url, robots):
+    raise RobotExclusionError(f"{url} excluded by {robots.url}")
+
+
+def warn_robots_txt(url, robots):
+    warnings.warn(f"{url} excluded by {robots.url}")
+
+
+class RobotsClient(Protocol):
+    _robots_for_domain: dict[str, RobotFileParser]
+    _robots_ua: str
+    _rejected_action: Callable[[str, RobotFileParser], None]
+    _no_check_request: Callable
+    request: Callable
+    headers: dict
+
+
+def _robot_check_request(client: RobotsClient, *args, **kwargs) -> Response:
+    method, url = args
+    uurl = URL(url)
+    domain = uurl.host
+    if domain not in client._robots_for_domain:
+        robots_url = f"{uurl.scheme}://{domain}/robots.txt"
+        robots_resp = client._no_check_request("GET", robots_url)
+        # pass url for output, but don't do read
+        parser = RobotFileParser(robots_url)
+        parser.parse(robots_resp.text.splitlines())
+        client._robots_for_domain[domain] = parser
+    if not client._robots_for_domain[domain].can_fetch(client._robots_ua, url):
+        client._rejected_action(url, client._robots_for_domain[domain])
+    # if action doesn't raise an exception, the request goes through
+    return client._no_check_request(*args, **kwargs)
+
+
+RobotsRejectFunc = Callable[[str, RobotFileParser], None]
+
+
+def make_robots_txt_client(
+    *,
+    client: Client | None = None,
+    as_user_agent: str | None = None,
+    on_rejection: RobotsRejectFunc = raise_robots_txt,
+) -> Client:
+    if client is None:
+        client = Client()
+
+    tclient = cast(RobotsClient, client)
+
+    tclient._robots_for_domain = {}
+    if as_user_agent:
+        tclient._robots_ua = as_user_agent
+    else:
+        tclient._robots_ua = tclient.headers["user-agent"]
+    tclient._rejected_action = on_rejection
+
+    tclient._no_check_request = client.request
+    tclient.request = types.MethodType(
+        functools.wraps(client.request)(_robot_check_request), client
+    )
+    return client

careful/httpx/throttle.py

@@ -2,12 +2,21 @@ import time
 import types
 import functools
 import logging
-from httpx import Client
+from typing import Protocol, cast, Callable
+from httpx import Client, Response
 
-log = logging.getLogger("httpx")
+log = logging.getLogger("careful")
 
 
-def _throttle_request(client: Client, *args, **kwargs):
+class ThrottledClient(Protocol):
+    _last_request: float
+    _requests_per_minute: float
+    _request_frequency: float
+    _no_throttle_request: Callable
+    request: Callable
+
+
+def _throttle_request(client: ThrottledClient, *args, **kwargs) -> Response:
     now = time.time()
     diff = client._request_frequency - (now - client._last_request)
     if diff > 0:
@@ -23,19 +32,21 @@ def make_throttled_client(
     *,
     client: Client | None = None,
     requests_per_minute: float = 0,
-):
+) -> Client:
     if requests_per_minute <= 0:
         raise ValueError("requests per minute must be a positive number")
 
     if client is None:
         client = Client()
 
-    client._last_request = 0.0
-    client._requests_per_minute = requests_per_minute
-    client._request_frequency = 60.0 / requests_per_minute
+    tclient = cast(ThrottledClient, client)
+
+    tclient._last_request = 0.0
+    tclient._requests_per_minute = requests_per_minute
+    tclient._request_frequency = 60.0 / requests_per_minute
 
-    client._no_throttle_request = client.request
-    client.request = types.MethodType(
+    tclient._no_throttle_request = client.request
+    tclient.request = types.MethodType(
         functools.wraps(client.request)(_throttle_request), client
     )
     return client

careful-0.3.1.dist-info/METADATA

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: careful
+Version: 0.3.1
+Summary: a small library for writing resilient, well-behaved HTTP code
+Project-URL: Repository, https://codeberg.org/jpt/careful
+Author-email: jpt <dev@jpt.sh>
+License: BSD-2-Clause
+License-File: LICENSE
+Classifier: Development Status :: 6 - Mature
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.28.1
+Description-Content-Type: text/markdown
+
+# careful
+
+<img src="https://jpt.sh/projects/careful/carefully-3681327.svg" width=100 height=100 alt="logo of a warning sign">
+
+**careful** is a Python library for writing resilient, well-behaved HTTP clients.
+
+**Code**: <https://codeberg.org/jpt/careful>
+
+**Docs**: <https://jpt.sh/projects/careful/>
+
+![PyPI - Version](https://img.shields.io/pypi/v/careful)
+[![status-badge](https://ci.codeberg.org/api/badges/15185/status.svg)](https://ci.codeberg.org/repos/15185)
+
+Call one function to enchant an
+**[httpx.Client](https://www.python-httpx.org)**, making your HTTP connections more resilient and better mannered.
+
+- Configure **throttling** to avoid accidental Denial-of-Service / risking getting banned.
+- **Retries** help overcome intermittent failures on flaky sites or long crawls.
+- **Development caching** Cache persists between runs during development, reduces redundant requests made while iterating on your crawlers & scrapers.
+
+### Example
+
+```python
+from httpx import Client
+from careful.httpx import make_careful_client
+
+# the only function you need to call is make_careful_client
+# this wraps your existing `httpx.Client` with your preferred
+# careful behaviors
+
+client = make_careful_client(
+    client=Client(headers={'user-agent': 'spiderman/1.0'}),
+
+    # retries are configurable w/ exponential back off
+    retry_attempts=2,
+    retry_wait_seconds=5,
+
+    # can cache to process memory, filesystem, or SQLite
+    cache_storage=MemoryCache(),
+
+    # easy-to-configure throttling
+    requests_per_minute=60,
+)
+
+# methods on client are called as they always are
+# configured behaviors occur without further code changes
+client.get("https://example.com")
+```
+
+
+---
+
+Logo licensed from [Adrien Coquet via Noun Project](https://thenounproject.com/icon/carefully-3681327/)

careful-0.3.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+careful/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+careful/httpx/__init__.py,sha256=sIKMgGf-Ea5VEsEFqxXFbAiaF1tYOYm9CIOvSCPooqw,6674
+careful/httpx/_types.py,sha256=jefYDxSbLRUatU8QKeyxStc9UC3AJwAba2SfhNkM0RY,151
+careful/httpx/dev_cache.py,sha256=sfHX2TkKZQRTrOXSGSMgIWE0HbX3Fvuz88o8_jK9P8g,11316
+careful/httpx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+careful/httpx/retries.py,sha256=3kjuHKYnK1N4Rtum5gUyY_XO4o4cL4jc59d17Y6UwrI,2949
+careful/httpx/robots.py,sha256=jfqQdplTap_RCENu6MHEIabFVznFLruMvSIaG_u0v_8,2168
+careful/httpx/throttle.py,sha256=b1fbmUskcm343D1bbPbY-ITLdL1zVm1dXtjt9LT1bEA,1412
+careful-0.3.1.dist-info/METADATA,sha256=fkI2V9YX5JKhPTYZga5q_3cyfatB4B_PMdTci3sRMmc,2692
+careful-0.3.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+careful-0.3.1.dist-info/licenses/LICENSE,sha256=oHe4LmCuo6CZne42DRXfiR3uqqIfGsk4dAvDKucAi4M,1315
+careful-0.3.1.dist-info/RECORD,,

{careful-0.2.0.dist-info → careful-0.3.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

careful-0.2.0.dist-info/METADATA

@@ -1,71 +0,0 @@
-Metadata-Version: 2.4
-Name: careful
-Version: 0.2.0
-Summary: careful extensions to httpx: throttle, retry, cache
-Project-URL: Repository, https://codeberg.org/jpt/careful
-Author-email: jpt <dev@jpt.sh>
-License: BSD-2-Clause
-License-File: LICENSE
-Classifier: Development Status :: 6 - Mature
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Natural Language :: English
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.10
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: mkdocs-material>=9.6.18
-Requires-Dist: mkdocstrings-python>=1.18.2
-Requires-Dist: mkdocstrings>=0.30.0
-Requires-Dist: pytest-httpbin>=2.1.0
-Requires-Dist: pytest>=8.4.2
-Description-Content-Type: text/markdown
-
-# careful
-
-<img src="/carefully-3681327.svg" width=100 height=100 alt="logo of a warning sign">
-
-**careful** is a library for making requests to unreliable websites with httpx.
-
-**Code**: <https://codeberg.org/jpt/careful>
-
-**Docs**: <https://careful.jpt.sh>
-
-It offers enhancements to 
-[`httpx.Client`](https://www.python-httpx.org)
-useful for writing long-running scrapers & crawlers, particularly against sites that are slow or have intermittent errors.
-
-- **configurable retry support.** retry on timeouts or other errors, with exponential back-off.
-- **simple request throttling.** set a maximum number of requests per minute.
-- **development cache.** configurable caching aimed at reducing redundant requests made while authoring/testing web scrapers.
-
-### example
-
-```python
-from httpx import Client
-from careful.httpx import make_careful_client
-
-client = make_careful_client(
-    # can configure httpx.Client however you usually would
-    client=Client(headers={'user-agent': 'careful/1.0'}),
-    # retries are configurable w/ exponential back off
-    retry_attempts=2,
-    retry_wait_seconds=5,
-    # can cache to process memory, filesystem, or SQLite
-    cache_storage=MemoryCache(),
-    # requests will automatically be throttled to aim at this rate
-    requests_per_minute=60,
-)
-
-# all normal methods on httpx.Client make use of configured enhancements
-client.get("https://example.com")
-```
-
-
----
-
-Logo licensed from [Adrien Coquet via Noun Project](https://thenounproject.com/icon/carefully-3681327/)

careful-0.2.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-careful/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-careful/httpx/__init__.py,sha256=u-n0uKIWAd3NXsZUd1UA4wzJJTEhRR74diHzDV2EpEU,4885
-careful/httpx/_types.py,sha256=NwyQ-ItodN9HnO7d7b0M1M4M9y90TjRkhQFqNuypKRI,149
-careful/httpx/dev_cache.py,sha256=KR35u0CvutqTOWQ8pO-hzwbPy0lDBhShJfhCAbOvqv0,11032
-careful/httpx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-careful/httpx/retries.py,sha256=Kszm0wDITyPZ3qx5TsDL__HjCYVJyAZ2WehrlpXV5Cc,2500
-careful/httpx/throttle.py,sha256=ZpuFABYHGQ4D0zks922SCXp7WZG_-Ysafz-Npa2QVwQ,1096
-careful-0.2.0.dist-info/METADATA,sha256=A82D5ltN7bDh1dXkOqdBLcW8fxxxqsonFgf9hZQlors,2541
-careful-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-careful-0.2.0.dist-info/licenses/LICENSE,sha256=oHe4LmCuo6CZne42DRXfiR3uqqIfGsk4dAvDKucAi4M,1315
-careful-0.2.0.dist-info/RECORD,,