careful 0.1.0__tar.gz

careful-0.1.0/.gitignore

@@ -0,0 +1,14 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+uv.lock
+
+# Virtual environments
+.venv
+
+# cache
+cache.db

careful-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+# updated 2025-04-16
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.11.5
+  hooks:
+    - id: ruff
+    - id: ruff-format
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0  # Use the ref you want to point at
+  hooks:
+    - id: trailing-whitespace
+    - id: check-added-large-files
+      args: ['--maxkb=1024']
+    - id: check-case-conflict
+    - id: check-executables-have-shebangs
+    - id: check-json
+    - id: check-merge-conflict
+    - id: check-symlinks
+    - id: check-toml
+    - id: check-yaml
+    - id: debug-statements
+    - id: forbid-submodules
+    - id: mixed-line-ending
+    #- id: no-commit-to-branch

careful-0.1.0/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2025, James Turk
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, 
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, 
+      this list of conditions and the following disclaimer in the documentation 
+      and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

careful-0.1.0/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.4
+Name: careful
+Version: 0.1.0
+Summary: Add your description here
+Author-email: jpt <dev@jpt.sh>
+License-File: LICENSE
+Requires-Python: >=3.13
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pytest-httpbin>=2.1.0
+Requires-Dist: pytest>=8.4.2
+Description-Content-Type: text/markdown
+
+**careful_httpx** is a library for making requests to less-than-reliable websites.
+
+It is based on [scrapelib](https://pypi.org/scrapelib/), which has powered Open States & many other Python scrapers for over 15 years.
+
+Code: <https://codeberg.org/jpt/careful_httpx>
+
+Documentation: TODO
+
+## Features
+
+Enhances [`httpx.Client`](https://www.python-httpx.org) with features useful for writing long-running scrapers & crawlers, particularly against sites that are slow or have intermittent errors.
+
+- retries
+- throttling
+- dev-cache for iterating on scrapers
+
+### example
+
+TODO
+
+### features this has that scrapelib doesn't
+
+- httpx support
+- composable interface, can augment Client with just the enhancements you want
+
+TODO: don't allow instantiating bad patch classes, and check for incompatible configs
+
+### features scrapelib had that this doesn't
+
+Open to considering if there is interest, but didn't seem necessary.
+
+- HTTP(S) and FTP requests via an identical API
+- allow setting custom ciphers
+- have urlretrieve
+- support FTP
+- set custom user-agent/mess w/ headers

careful-0.1.0/README.md

@@ -0,0 +1,36 @@
+**careful_httpx** is a library for making requests to less-than-reliable websites.
+
+It is based on [scrapelib](https://pypi.org/scrapelib/), which has powered Open States & many other Python scrapers for over 15 years.
+
+Code: <https://codeberg.org/jpt/careful_httpx>
+
+Documentation: TODO
+
+## Features
+
+Enhances [`httpx.Client`](https://www.python-httpx.org) with features useful for writing long-running scrapers & crawlers, particularly against sites that are slow or have intermittent errors.
+
+- retries
+- throttling
+- dev-cache for iterating on scrapers
+
+### example
+
+TODO
+
+### features this has that scrapelib doesn't
+
+- httpx support
+- composable interface, can augment Client with just the enhancements you want
+
+TODO: don't allow instantiating bad patch classes, and check for incompatible configs
+
+### features scrapelib had that this doesn't
+
+Open to considering if there is interest, but didn't seem necessary.
+
+- HTTP(S) and FTP requests via an identical API
+- allow setting custom ciphers
+- have urlretrieve
+- support FTP
+- set custom user-agent/mess w/ headers

careful-0.1.0/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "careful"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+authors = [
+    { name = "jpt", email = "dev@jpt.sh" }
+]
+requires-python = ">=3.13"
+dependencies = [
+    "httpx>=0.28.1",
+    "pytest>=8.4.2",
+    "pytest-httpbin>=2.1.0",
+]
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

careful-0.1.0/src/careful/httpx/__init__.py

@@ -0,0 +1,59 @@
+from .retries import make_retry_client, _default_accept_response
+from .throttle import make_throttled_client
+from .dev_cache import (
+    make_dev_caching_client,
+    MemoryCache,
+    FileCache,
+    SQLiteCache,
+    _cache_200s,
+    _default_keyfunc,
+)
+from httpx import Client
+
+
+def make_careful_client(
+    client: Client,
+    *,
+    retry_attempts: int = 0,
+    retry_wait_seconds: float = 10,
+    retry_on_404: bool = False,
+    accept_response=_default_accept_response,
+    requests_per_minute: int = 0,
+    cache_storage=None,
+    cache_write_only=False,
+    should_cache=_cache_200s,
+    cache_keyfunc=_default_keyfunc,
+):
+    # order matters, retry on inside b/c it is last-chance scenario
+    if retry_attempts:
+        client = make_retry_client(
+            client=client,
+            attempts=retry_attempts,
+            wait_seconds=retry_wait_seconds,
+            retry_on_404=retry_on_404,
+            accept_response=accept_response,
+        )
+    # throttling around retries
+    if requests_per_minute:
+        client = make_throttled_client(client, requests_per_minute=requests_per_minute)
+    # caching on top layer, so cache will be checked first
+    if cache_storage:
+        client = make_dev_caching_client(
+            client=client,
+            cache_storage=cache_storage,
+            cache_keyfunc=cache_keyfunc,
+            should_cache=should_cache,
+            write_only=cache_write_only,
+        )
+
+    return client
+
+
+__all__ = [
+    "make_retry_client",
+    "make_throttled_client",
+    "make_dev_caching_client",
+    "MemoryCache",
+    "FileCache",
+    "SQLiteCache",
+]

careful-0.1.0/src/careful/httpx/dev_cache.py

@@ -0,0 +1,296 @@
+import types
+import functools
+import logging
+import re
+import os
+import glob
+import hashlib
+import sqlite3
+import json
+
+from httpx import Client, Response, Request
+
+log = logging.getLogger("httpx")
+
+
+def _default_keyfunc(
+    method: str,
+    url: str,
+    params: dict | None = None,
+) -> str | None:
+    """
+    Return a cache key from a given set of request parameters.
+
+    Default behavior is to return a complete URL for all GET
+    requests, and None otherwise.
+    """
+    if method.lower() != "get":
+        return None
+
+    return Request(url=url, method=method, params=params).url
+
+
+def _cache_200s(response: Response) -> bool:
+    """
+    Check if a given Response object should be cached.
+
+    Default behavior is to only cache responses with a 200 status code.
+    """
+    return response.status_code == 200
+
+
+def _cached_request(client: Client, *args, **kwargs):
+    # short circuit if cache isn't configured
+    if not client._cache_storage:
+        log.debug("bypassing cache, no storage configured")
+        resp = client._wrapped_request(*args, **kwargs)
+        resp.fromcache = False
+        return resp
+
+    method, url = args
+    request_key = client._cache_keyfunc(method, url, kwargs["params"])
+
+    # check cache for response
+    cached_resp = None
+    if request_key and not client._write_only:
+        cached_resp = client._cache_storage.get(request_key)
+
+    if cached_resp:
+        # resp = cast(CacheResponse, resp_maybe)
+        log.info("using cached response request_key=%s", request_key)
+        cached_resp.fromcache = True
+        resp = cached_resp
+    else:
+        resp = client._wrapped_request(*args, **kwargs)
+        # save to cache if request and response meet criteria
+        log.debug("XX %s %s", request_key, client._should_cache(resp))
+        if request_key and client._should_cache(resp):
+            client._cache_storage.set(request_key, resp)
+            log.info("caching response request_key=%s", request_key)
+        resp.fromcache = False
+
+    return resp
+
+
+def make_dev_caching_client(
+    *,
+    client: Client | None = None,
+    cache_storage=None,
+    cache_keyfunc=_default_keyfunc,
+    should_cache=_cache_200s,
+    write_only=False,
+):
+    if client is None:
+        client = Client()
+
+    client._cache_storage = cache_storage
+    client._cache_keyfunc = cache_keyfunc
+    client._should_cache = should_cache
+    client._write_only = write_only
+
+    client._wrapped_request = client.request
+    client.request = types.MethodType(
+        functools.wraps(client.request)(_cached_request), client
+    )
+    return client
+
+
+class CacheStorageBase:
+    def get(self, key: str) -> None | Response:
+        raise NotImplementedError()
+
+    def set(self, key: str, response: Response) -> None:
+        raise NotImplementedError()
+
+
+class MemoryCache(CacheStorageBase):
+    """In memory cache for request responses."""
+
+    def __init__(self) -> None:
+        self.cache: dict[str, Response] = {}
+
+    def get(self, key: str) -> None | Response:
+        """Get cache entry for key, or return None."""
+        return self.cache.get(key, None)
+
+    def set(self, key: str, response: Response) -> None:
+        """Set cache entry for key with contents of response."""
+        self.cache[key] = response
+
+
+class FileCache(CacheStorageBase):
+    """
+    File-based cache for request responses.
+
+    :param cache_dir: directory for storing responses
+    :param check_last_modified:  set to True to compare last-modified
+        timestamp in cached response with value from HEAD request
+    """
+
+    # file name escaping inspired by httplib2
+    _prefix = re.compile(r"^\w+://")
+    _illegal = re.compile(r"[?/:|]+")
+    _header_re = re.compile(r"([-\w]+): (.*)")
+    _maxlen = 200
+
+    def _clean_key(self, key: str) -> str:
+        # strip scheme
+        md5 = hashlib.md5(key.encode("utf8")).hexdigest()
+        key = self._prefix.sub("", key)
+        key = self._illegal.sub(",", key)
+        return ",".join((key[: self._maxlen], md5))
+
+    def __init__(self, cache_dir: str, check_last_modified: bool = False):
+        # normalize path
+        self.cache_dir = os.path.join(os.getcwd(), cache_dir)
+        self.check_last_modified = check_last_modified
+        # create directory
+        if not os.path.isdir(self.cache_dir):
+            os.makedirs(self.cache_dir)
+
+    def get(self, orig_key: str) -> None | Response:
+        """Get cache entry for key, or return None."""
+        key = self._clean_key(orig_key)
+        path = os.path.join(self.cache_dir, key)
+        resp_headers = {}
+
+        try:
+            with open(path, "rb") as f:
+                # read lines one at a time
+                while True:
+                    line = f.readline().decode("utf8").strip("\r\n")
+                    # set headers
+
+                    # if self.check_last_modified and re.search(
+                    #     "last-modified", line, flags=re.I
+                    # ):
+                    #     # line contains last modified header
+                    #     head_resp = requests.head(orig_key)
+
+                    #     try:
+                    #         new_lm = head_resp.headers["last-modified"]
+                    #         old_lm = line[line.find(":") + 1 :].strip()
+                    #         if old_lm != new_lm:
+                    #             # last modified timestamps don't match, need to download again
+                    #             return None
+                    #     except KeyError:
+                    #         # no last modified header present, so redownload
+                    #         return None
+
+                    header = self._header_re.match(line)
+                    if header:
+                        resp_headers[header.group(1)] = header.group(2)
+                    else:
+                        break
+                # everything left is the real content
+                resp_content = f.read()
+
+            # status & encoding will be in headers, but are faked
+            # need to split spaces out of status to get code (e.g. '200 OK')
+            resp = Response(
+                status_code = int(resp_headers.pop("status").split(" ")[0]),
+                content=resp_content,
+                default_encoding=resp_headers.pop("encoding"),
+                headers=resp_headers,
+            )
+            return resp
+        except IOError:
+            return None
+
+    def set(self, key: str, response: Response) -> None:
+        """Set cache entry for key with contents of response."""
+        key = self._clean_key(key)
+        path = os.path.join(self.cache_dir, key)
+
+        with open(path, "wb") as f:
+            status_str = "status: {0}\n".format(response.status_code)
+            f.write(status_str.encode("utf8"))
+            encoding_str = "encoding: {0}\n".format(response.encoding)
+            f.write(encoding_str.encode("utf8"))
+            for h, v in response.headers.items():
+                # header: value\n
+                f.write(h.encode("utf8"))
+                f.write(b": ")
+                f.write(v.encode("utf8"))
+                f.write(b"\n")
+            # one blank line
+            f.write(b"\n")
+            f.write(response.content)
+
+    def clear(self) -> None:
+        # only delete things that end w/ a md5, less dangerous this way
+        cache_glob = "*," + ("[0-9a-f]" * 32)
+        for fname in glob.glob(os.path.join(self.cache_dir, cache_glob)):
+            os.remove(fname)
+
+
+class SQLiteCache(CacheStorageBase):
+    """SQLite cache for request responses.
+
+    :param cache_path: path for SQLite database file
+    :param check_last_modified: set to True to compare last-modified
+        timestamp in cached response with value from HEAD request
+
+    """
+
+    _columns = ["key", "status", "modified", "encoding", "data", "headers"]
+
+    def __init__(self, cache_path: str, check_last_modified: bool = False):
+        self.cache_path = cache_path
+        self.check_last_modified = check_last_modified
+        self._conn = sqlite3.connect(cache_path)
+        self._conn.text_factory = str
+        self._build_table()
+
+    def _build_table(self) -> None:
+        """Create table for storing request information and response."""
+        self._conn.execute(
+            """CREATE TABLE IF NOT EXISTS cache
+                (key text UNIQUE, status integer, modified text,
+                 encoding text, data blob, headers blob)"""
+        )
+
+    def set(self, key: str, response: Response) -> None:
+        """Set cache entry for key with contents of response."""
+        mod = response.headers.pop("last-modified", None)
+        status = int(response.status_code)
+        rec = (
+            key,
+            status,
+            mod,
+            response.encoding,
+            response.content,
+            json.dumps(dict(response.headers)),
+        )
+        with self._conn:
+            self._conn.execute("DELETE FROM cache WHERE key=?", (key,))
+            self._conn.execute("INSERT INTO cache VALUES (?,?,?,?,?,?)", rec)
+
+    def get(self, key: str) -> None | Response:
+        """Get cache entry for key, or return None."""
+        query = self._conn.execute("SELECT * FROM cache WHERE key=?", (key,))
+        rec = query.fetchone()
+        if rec is None:
+            return None
+        rec = dict(zip(self._columns, rec))
+
+        # TODO evaluate/remove?
+        # if self.check_last_modified:
+        #     if rec["modified"] is None:
+        #         return None  # no last modified header present, so redownload
+
+        #     head_resp = requests.head(key)
+        #     new_lm = head_resp.headers.get("last-modified", None)
+        #     if rec["modified"] != new_lm:
+        #         return None
+
+        resp = Response(rec["status"], content=rec["data"], default_encoding=rec["encoding"], headers=json.loads(rec["headers"]))
+        return resp
+
+    def clear(self) -> None:
+        """Remove all records from cache."""
+        with self._conn:
+            self._conn.execute("DELETE FROM cache")
+
+    def __del__(self) -> None:
+        self._conn.close()

careful-0.1.0/src/careful/httpx/retries.py

@@ -0,0 +1,87 @@
+import time
+import types
+import functools
+import logging
+from httpx import Client, Response
+
+log = logging.getLogger("httpx")
+
+
+def _default_accept_response(response: Response) -> bool:
+    return response.status_code < 400
+
+
+def _retry_request(client: Client, *args, **kwargs):
+    # the retry loop
+    tries = 0
+    exception_raised = None
+
+    while tries <= client._retry_attempts:
+        exception_raised = None
+
+        try:
+            resp = client._wrapped_request(*args, **kwargs)
+
+            # break from loop on an accepted response
+            if client._accept_response(resp) or (
+                resp.status_code == 404 and not client._retry_on_404
+            ):
+                break
+
+        except Exception as e:
+            # TODO: exclude certain kinds of exceptions (SSL?) from retry
+            exception_raised = e
+
+            if exception_response := getattr(e, "response", None):
+                if client._accept_response(exception_response):
+                    break
+
+        # if we're going to retry, sleep first
+        tries += 1
+        if tries <= client._retry_attempts:
+            # twice as long each time
+            wait = client._retry_wait_seconds * (2 ** (tries - 1))
+            if exception_raised:
+                log.info(
+                    "exception %s, sleeping for %s seconds before retry #%s",
+                    exception_raised,
+                    wait,
+                    tries,
+                )
+            else:
+                log.info(
+                    "response %s, sleeping for %s seconds before retry #%s",
+                    resp,
+                    wait,
+                    tries,
+                )
+            time.sleep(wait)
+
+    # out of the loop, either an exception was raised or we had a success
+    if exception_raised:
+        raise exception_raised
+    return resp
+
+
+def make_retry_client(
+    *,
+    client: Client | None = None,
+    attempts: int = 1,
+    wait_seconds: float = 10,
+    retry_on_404: bool = False,
+    accept_response=_default_accept_response,
+):
+    if client is None:
+        client = Client()
+    client._retry_attempts = max(0, attempts)
+    client._retry_wait_seconds = wait_seconds
+    client._retry_on_404 = retry_on_404
+    client._accept_response = accept_response
+
+    client._wrapped_request = client.request
+    client.request = types.MethodType(
+        functools.wraps(client.request)(_retry_request), client
+    )
+
+    return client
+

careful-0.1.0/src/careful/httpx/throttle.py

@@ -0,0 +1,41 @@
+import time
+import types
+import functools
+import logging
+from httpx import Client
+
+log = logging.getLogger("httpx")
+
+
+def _throttle_request(client: Client, *args, **kwargs):
+    now = time.time()
+    diff = client._request_frequency - (now - client._last_request)
+    if diff > 0:
+        log.debug("throttled, sleeping for %fs", diff)
+        time.sleep(diff)
+        client._last_request = time.time()
+    else:
+        client._last_request = now
+    return client._wrapped_request(*args, **kwargs)
+
+
+def make_throttled_client(
+    *,
+    client: Client | None = None,
+    requests_per_minute: float = 0,
+):
+    if requests_per_minute <= 0:
+        raise ValueError("requests per minute must be a positive number")
+
+    if client is None:
+        client = Client()
+
+    client._last_request = 0.0
+    client._requests_per_minute = requests_per_minute
+    client._request_frequency = 60.0 / requests_per_minute
+
+    client._wrapped_request = client.request
+    client.request = types.MethodType(
+        functools.wraps(client.request)(_throttle_request), client
+    )
+    return client

careful-0.1.0/tests/fakeresponse.py

@@ -0,0 +1,17 @@
+
+class FakeResponse:
+    def __init__(
+        self,
+        url: str,
+        code: int,
+        content: str | bytes,
+        encoding: str = "utf-8",
+        headers: dict | None = None,
+    ):
+        self.url = url
+        self.status_code = code
+        self.content = content
+        self.text = str(content)
+        self.encoding = encoding
+        self.headers = headers or {}
+

careful-0.1.0/tests/test_cache.py

@@ -0,0 +1,65 @@
+from pytest_httpbin.serve import Server  # type: ignore
+from httpx import Response
+from careful.httpx import make_dev_caching_client, MemoryCache, FileCache, SQLiteCache
+
+
+def test_dev_caching(httpbin: Server) -> None:
+    client = make_dev_caching_client(cache_storage=MemoryCache(), write_only=False)
+
+    resp = client.get(httpbin.url + "/status/200")
+    assert not resp.fromcache
+    resp = client.get(httpbin.url + "/status/200")
+    assert resp.fromcache
+
+
+def test_dev_caching_params(httpbin: Server) -> None:
+    client = make_dev_caching_client(cache_storage=MemoryCache(), write_only=False)
+
+    resp = client.get(httpbin.url + "/status/200?a=1&b=2")
+    assert not resp.fromcache
+    resp = client.get(httpbin.url + "/status/200?a=1&b=2")
+    assert resp.fromcache
+    resp = client.get(httpbin.url + "/status/200?a=1&b=3")
+    assert not resp.fromcache
+
+
+# test storages #####
+
+
+def _test_cache_storage(storage_obj) -> None:
+    # unknown key returns None
+    assert storage_obj.get("one") is None
+
+    _content_as_bytes = b"here's unicode: \xe2\x98\x83"
+    _content_as_unicode = "here's unicode: \u2603"
+
+    # set 'one'
+    resp = Response(200)
+    resp.headers["x-num"] = "one"
+    resp._content = _content_as_bytes
+    storage_obj.set("one", resp)
+    cached_resp = storage_obj.get("one")
+    assert cached_resp is not None
+    if cached_resp is not None:
+        assert cached_resp.headers["x-num"] == "one"
+        assert cached_resp.status_code == 200
+        cached_resp.encoding = "utf8"
+        assert cached_resp.text == _content_as_unicode
+
+
+def test_memory_cache() -> None:
+    _test_cache_storage(MemoryCache())
+
+
+def test_file_cache() -> None:
+    fc = FileCache("cache")
+    fc.clear()
+    _test_cache_storage(fc)
+    fc.clear()
+
+
+def test_sqlite_cache() -> None:
+    sc = SQLiteCache("cache.db")
+    sc.clear()
+    _test_cache_storage(sc)
+    sc.clear()

careful-0.1.0/tests/test_careful.py

@@ -0,0 +1,4 @@
+from careful.httpx import make_careful_client
+
+def test_full_careful_client():
+    client = make_careful_client()

careful-0.1.0/tests/test_retries.py

@@ -0,0 +1,85 @@
+from careful.httpx import make_retry_client
+from unittest import mock
+from fakeresponse import FakeResponse
+
+
+def test_retry() -> None:
+    client = make_retry_client(attempts=3, wait_seconds=0.001)
+
+    # On the first call return a 500, then a 200
+    mock_request = mock.Mock(
+        side_effect=[
+            FakeResponse("http://dummy/", 500, "failure!"),
+            FakeResponse("http://dummy/", 200, "success!"),
+        ]
+    )
+
+    with mock.patch.object(client, "_wrapped_request", mock_request):
+        resp = client.get("http://dummy/")
+    assert mock_request.call_count == 2
+
+    # 500 always
+    mock_request = mock.Mock(
+        return_value=FakeResponse("http://dummy/", 500, "failure!")
+    )
+
+    with mock.patch.object(client, "_wrapped_request", mock_request):
+        resp = client.get("http://dummy/")
+    assert resp.status_code == 500
+    assert mock_request.call_count == 4  # try four times
+
+
+def test_retry_404() -> None:
+    client = make_retry_client(attempts=3, wait_seconds=0.001, retry_on_404=True)
+
+    # On the first call return a 404, then a 200
+    mock_request = mock.Mock(
+        side_effect=[
+            FakeResponse("http://dummy/", 404, "failure!"),
+            FakeResponse("http://dummy/", 200, "success!"),
+        ]
+    )
+
+    with mock.patch.object(client, "_wrapped_request", mock_request):
+        resp = client.get("http://dummy/")  # type: ignore
+    assert mock_request.call_count == 2
+    assert resp.status_code == 200
+
+    # 404 always
+    mock_request = mock.Mock(
+        return_value=FakeResponse("http://dummy/", 404, "failure!")
+    )
+
+    # four tries
+    with mock.patch.object(client, "_wrapped_request", mock_request):
+        resp = client.get("http://dummy/")
+    assert resp.status_code == 404
+    assert mock_request.call_count == 4
+    assert resp.status_code == 404
+
+
+def test_no_retry_404() -> None:
+    client = make_retry_client(attempts=3, wait_seconds=0.001, retry_on_404=False)
+
+    # On the first call return a 404, then a 200
+    mock_request = mock.Mock(
+        side_effect=[
+            FakeResponse("http://dummy/", 404, "failure!"),
+            FakeResponse("http://dummy/", 200, "success!"),
+        ]
+    )
+
+    with mock.patch.object(client, "_wrapped_request", mock_request):
+        resp = client.get("http://dummy/")  # type: ignore
+    assert mock_request.call_count == 1
+    assert resp.status_code == 404
+
+
+# def test_retry_ssl() -> None:
+#     s = make_retry_client(retry_attempts=5, retry_wait_seconds=0.001, raise_errors=False)
+
+#     # ensure SSLError is considered fatal even w/ retries
+#     with mock.patch.object(requests.Session, "request", mock_sslerror):
+#         with pytest.raises(requests.exceptions.SSLError):
+#             s.get("http://dummy/", retry_on_404=True)  # type: ignore
+#     assert mock_sslerror.call_count == 1

careful-0.1.0/tests/test_throttle.py

@@ -0,0 +1,29 @@
+from careful.httpx import make_throttled_client
+from unittest import mock
+from typing import Any
+from fakeresponse import FakeResponse
+
+
+def request_200(method: str, url: str, *args: Any, **kwargs: Any) -> FakeResponse:
+    return FakeResponse(url, 200, b"ok")
+
+
+mock_200 = mock.Mock(wraps=request_200)
+
+
+def test_request_throttling() -> None:
+    client = make_throttled_client(requests_per_minute=30)
+
+    mock_sleep = mock.Mock()
+
+    # check that sleep is called on call 2 & 3
+    with mock.patch("time.sleep", mock_sleep):
+        with mock.patch.object(client, "_wrapped_request", mock_200):
+            client.get("http://dummy/")
+            client.get("http://dummy/")
+            client.get("http://dummy/")
+            assert mock_sleep.call_count == 2
+            # should have slept for ~2 seconds to aim at 30 per min
+            assert 1.8 <= mock_sleep.call_args[0][0] <= 2.2
+
+