careful 0.2.1__tar.gz → 0.3.1__tar.gz

careful-0.3.1/.spellignore

@@ -0,0 +1,6 @@
+Adrien
+Coquet
+MemoryCache
+spiderman
+src
+img

{careful-0.2.1 → careful-0.3.1}/.woodpecker.yml

@@ -11,15 +11,15 @@ matrix:
 
 steps:
   lint:
-    image: python:3.11
+    image: python:3.13
     commands:
       - curl -LsSf https://astral.sh/uv/install.sh | sh
       - "export PATH=/root/.local/bin:$PATH"
       - uv run ruff check .
       - uv run ruff format --check .
+      - uvx ty check .
     when:
-      - matrix:
-        PYTHON_VERSION: 3.13
+      - evaluate: 'PYTHON_VERSION == "3.13"'
 
 
   test:
@@ -27,6 +27,5 @@ steps:
     commands:
       - curl -LsSf https://astral.sh/uv/install.sh | sh
       - "export PATH=/root/.local/bin:$PATH"
+      - uv sync --group httpbin
       - uv run --python ${PYTHON_VERSION} pytest
-    depends_on:
-      - lint

{careful-0.2.1 → careful-0.3.1}/Justfile

@@ -8,9 +8,10 @@ preview:
     uv run mkdocs serve
 
 publish-pypi:
+    rm dist/*
     uv build
     uv publish
 
 publish-docs:
     uv run mkdocs build
-    netlify deploy --prod -s careful-docs -d site
+    uvx trifold publish

{careful-0.2.1 → careful-0.3.1}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: careful
-Version: 0.2.1
-Summary: careful extensions to httpx: throttle, retry, cache
+Version: 0.3.1
+Summary: a small library for writing resilient, well-behaved HTTP code
 Project-URL: Repository, https://codeberg.org/jpt/careful
 Author-email: jpt <dev@jpt.sh>
 License: BSD-2-Clause
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
 Requires-Dist: httpx>=0.28.1
@@ -22,43 +23,50 @@ Description-Content-Type: text/markdown
 
 # careful
 
-<img src="https://careful.jpt.sh/carefully-3681327.svg" width=100 height=100 alt="logo of a warning sign">
+<img src="https://jpt.sh/projects/careful/carefully-3681327.svg" width=100 height=100 alt="logo of a warning sign">
+
+**careful** is a Python library for writing resilient, well-behaved HTTP clients.
 
-**careful** is a Python library for making requests to unreliable websites with `httpx`.
- 
 **Code**: <https://codeberg.org/jpt/careful>
 
-**Docs**: <https://careful.jpt.sh>
+**Docs**: <https://jpt.sh/projects/careful/>
 
+![PyPI - Version](https://img.shields.io/pypi/v/careful)
 [![status-badge](https://ci.codeberg.org/api/badges/15185/status.svg)](https://ci.codeberg.org/repos/15185)
 
-It offers enhancements to 
-[`httpx.Client`](https://www.python-httpx.org)
-useful for writing long-running scrapers & crawlers, particularly against sites that are slow or have intermittent errors.
+Call one function to enchant an
+**[httpx.Client](https://www.python-httpx.org)**, making your HTTP connections more resilient and better mannered.
 
-- **configurable retry support.** retry on timeouts or other errors, with exponential back-off.
-- **simple request throttling.** set a maximum number of requests per minute.
-- **development cache.** configurable caching aimed at reducing redundant requests made while authoring/testing web scrapers.
+- Configure **throttling** to avoid accidental Denial-of-Service / risking getting banned.
+- **Retries** help overcome intermittent failures on flaky sites or long crawls.
+- **Development caching** Cache persists between runs during development, reduces redundant requests made while iterating on your crawlers & scrapers.
 
-### example
+### Example
 
 ```python
 from httpx import Client
 from careful.httpx import make_careful_client
 
+# the only function you need to call is make_careful_client
+# this wraps your existing `httpx.Client` with your preferred
+# careful behaviors
+
 client = make_careful_client(
-    # can configure httpx.Client however you usually would
-    client=Client(headers={'user-agent': 'careful/1.0'}),
+    client=Client(headers={'user-agent': 'spiderman/1.0'}),
+
     # retries are configurable w/ exponential back off
     retry_attempts=2,
     retry_wait_seconds=5,
+
     # can cache to process memory, filesystem, or SQLite
     cache_storage=MemoryCache(),
-    # requests will automatically be throttled to aim at this rate
+
+    # easy-to-configure throttling
     requests_per_minute=60,
 )
 
-# all normal methods on httpx.Client make use of configured enhancements
+# methods on client are called as they always are
+# configured behaviors occur without further code changes
 client.get("https://example.com")
 ```
 

careful-0.3.1/README.md

@@ -0,0 +1,53 @@
+# careful
+
+<img src="https://jpt.sh/projects/careful/carefully-3681327.svg" width=100 height=100 alt="logo of a warning sign">
+
+**careful** is a Python library for writing resilient, well-behaved HTTP clients.
+
+**Code**: <https://codeberg.org/jpt/careful>
+
+**Docs**: <https://jpt.sh/projects/careful/>
+
+![PyPI - Version](https://img.shields.io/pypi/v/careful)
+[![status-badge](https://ci.codeberg.org/api/badges/15185/status.svg)](https://ci.codeberg.org/repos/15185)
+
+Call one function to enchant an
+**[httpx.Client](https://www.python-httpx.org)**, making your HTTP connections more resilient and better mannered.
+
+- Configure **throttling** to avoid accidental Denial-of-Service / risking getting banned.
+- **Retries** help overcome intermittent failures on flaky sites or long crawls.
+- **Development caching** Cache persists between runs during development, reduces redundant requests made while iterating on your crawlers & scrapers.
+
+### Example
+
+```python
+from httpx import Client
+from careful.httpx import make_careful_client
+
+# the only function you need to call is make_careful_client
+# this wraps your existing `httpx.Client` with your preferred
+# careful behaviors
+
+client = make_careful_client(
+    client=Client(headers={'user-agent': 'spiderman/1.0'}),
+
+    # retries are configurable w/ exponential back off
+    retry_attempts=2,
+    retry_wait_seconds=5,
+
+    # can cache to process memory, filesystem, or SQLite
+    cache_storage=MemoryCache(),
+
+    # easy-to-configure throttling
+    requests_per_minute=60,
+)
+
+# methods on client are called as they always are
+# configured behaviors occur without further code changes
+client.get("https://example.com")
+```
+
+
+---
+
+Logo licensed from [Adrien Coquet via Noun Project](https://thenounproject.com/icon/carefully-3681327/)

{careful-0.2.1 → careful-0.3.1}/docs/changelog.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.3.1
+
+- Add `make_careful_client_from_env`
+
+## 0.3.0 - 8 September 2025
+
+- Increased consistency across interfaces with parameter names, etc.
+- Added support for robots.txt protocol.
+- Documentation overhaul.
+
 ## 0.2.0 - 6 September 2025
 
 - Initial release, mostly a port of `scrapelib` functionality.

careful-0.3.1/docs/design.md

@@ -0,0 +1,224 @@
+# Design Decisions
+
+!!! note
+
+    This section is not necessary for users of the library to understand.
+
+    If you are looking to contribute, or curious about why things work the way they do, read on.
+
+## Design Goals
+
+In order of importance:
+
+1. Work as a drop-in replacement for `httpx.Client`, so existing code can gain the benefits of the library without being changed.
+2. Preserve the 100% typed interface of `httpx`.
+3. Make writing & testing new augmentations easier.
+4. Organize code in a way that'll make augmenting future libraries (or `httpx.AsyncClient`) easier. (Based on limitations in porting `scrapelib` that made a partial rewrite easier.)
+
+## How it works
+
+Each "suite" of behavior: throttling, retries, caching consists of a function that
+[monkey patches](https://en.wikipedia.org/wiki/Monkey_patch) `httpx.Client` to add this behavior
+to the all-important `request` method. (The other common methods like `get` and `post` call this method.)
+
+If you'd like to follow along, `throttle.py` is the simplest of them, at around 50 lines long.
+
+You'll see two functions (ignore the class for now):
+
+- `_throttle_request` - this acts as a sort of decorator for `httpx.Client.request`
+- `make_throttled_client` - a psuedo-constructor for our monkey patched client.
+
+Each feature is implemented in a similar way.
+
+The recommended [make_careful_client][careful.httpx.make_careful_client] entrypoint is
+just a convinient combination of these `make_ZZZ_ciient` functions.
+
+If you don't need to be convinced that monkey patching was a reasonable choice here, you can skip to [Our patch pattern](#our-patch-pattern)
+
+## Why not inheritance?
+
+For 15 years `scrapelib` used a class hierarchy to a similar end, it'd certainly work here too.
+
+The equivalent to a careful client in `scrapelib` is a `scrapelib.Scraper`.
+It has a long inheritance hierarchy:
+
+`scrapelib.Scraper` -> `CachingSession` -> `ThrottledSession`  -> `RetrySession` -> `requests.Session`
+
+This hierarchy means that there is no such thing as a `CachingSession` that doesn't use throttling,
+and adding new behavior means considering exactly where it works best in the chain and then setting that in stone.
+
+There's arguably no benefit derived from having things set up this way.
+It is just too annoying to mix & match behaviors, or add new ones, a single class would have been easier to maintain over the years.
+
+## Why not mixins?
+
+We don't want to just give up and go with a single monolithic class. (See design goals 3 and 4.)
+
+It is worth revisiting why those classes weren't [mixins](https://en.wikipedia.org/wiki/Mixin).
+
+It seems like we could have `ThrottledMixin`, `RetryMixin`, `DevCacheMixin`, etc.
+
+Then to use retry & cache together someone would:
+
+```python
+import httpx
+from careful.hypothetical import ThrottledMixin, RetryMixin, DevCacheMixin
+
+class CustomClient(RetryMixin, DevCacheMixin, Client):
+  pass
+
+client = CustomClient()
+```
+
+Honestly, not a great start: having to declare an empty class, and to carefully think about method resolution order rules in ordering them.
+
+But there's a bigger problem lurking here... configuration.
+
+Assume each mixin is configured through its constructor:
+
+```python
+
+class RetryMixin:
+  def __init__(self, num_retries=2, retry_delay_seconds=10, **kwargs):
+    ...
+
+
+class DevCacheMixin:
+  def __init__(self, cache_backend=..., should_cache=..., **kwargs):
+    ...
+```
+
+To make this work properly, our custom class needs a constructor too.
+It'd wind up looking like:
+
+```python
+  def __init__(self, num_retries, retry_delay_seconds, cache_backend, should_cache, ...)
+      # Initialize mixins explicitly
+      RetryMixin.__init__(self,  num_retries=num_retries,
+                          retry_delay_seconds=retry_delay_seconds)
+
+      DevCacheMixin.__init__(self, cache_backend=cache_backend,
+                             should_cache=should_cache)
+
+      Client.__init__(self, **kwargs)
+```
+
+This makes working with the mixins frustrating, since any new combination requires modifying a repetitive constructor.
+
+## Preserving type signatures
+
+One of the most annoying parts of maintaining `scrapelib` has been keeping its function signatures in sync with small `requests` changes.
+
+Design goal #1 is that someone's existing usage of `httpx.Client` is unimpeded.
+The most important method, and where we need to hook in our overrides, is `Client.request`.
+The method takes a whopping 13 parameters, and of course `httpx` is also fully type-annotaed.
+
+To replace `request` we have three options:
+
+1. Give our new class a `request` method which takes `*args, and **kwargs` and passes them up the chain.
+2. Give each class a `request` method take the exact same 13 parameters, and be careful to keep them in sync.
+3. Use `functools.wraps` to replace the function but leave existing annotations & docstrings in place.
+
+\#1 reduces type safety and leads to a worse DX overall since language servers can no longer offer suggestions. **This won't work for us.**
+
+\#2 is the approach that `scrapelib` took. It was annoying and conflicts with goals 3 and 4.
+
+\#3 is the approach taken by `careful`, our actual monkey patch. Each `make_ZZZ_client` winds up with code resembling:
+
+```python
+    tclient._no_throttle_request = tclient.request
+    tclient.request = types.MethodType(
+        functools.wraps(client.request)(_throttle_request), client
+    )
+```
+
+The first line tucks away the pre-patch request method for use within the decorated function. It uses a unique name since it'll be sharing a namespace with other patches.
+
+The second line does two neat things:
+
+  - `_throttle_request` is given the signature of `client.request` via `functools.wraps`
+  - `types.MethodType` rebinds the member function (so `self` is correctly handled as the first parameter)
+
+## Our patch pattern
+
+Augmenting `Client` is done in two steps:
+
+- a request function that acts as a decorator for`Client.request`, this is where the actual logic for the augmentation lives
+- a patch function that:
+    - writes any private state needed for the new behavior to a `Client` instance
+    - replaces `Client.request` with our patched request
+
+All of the files in `careful.httpx` follow this structure.
+
+### Protocol-typing the internal interface
+
+After considering some of the issues above, I was considering that I'd probably have to have type ignore statements everywhere to get the monkey patching to work with a type checker.
+
+While this might have been an option, after all the end user experience is the priority, it'd be nice to keep the benefits of type checking for myself and other authors of extensions.
+
+It turns out, as long as we consider the patches fully internal to the `Client`, there is a way to make this work.
+
+The key challenge presented is that we add new variables to the `Client` during augmentation:
+
+```python
+    # this is that internal state we store on a client
+    tclient._last_request = 0.0
+    tclient._requests_per_minute = requests_per_minute
+    tclient._request_frequency = 60.0 / requests_per_minute
+```
+
+These set off type checker alarms, and if we simply ignore them then when they are used in the request wrapper
+they'll set off alarms again!
+
+It'd be nice to at least have a consistency check between these, so the wrapped `request` doesn't accidentally use the wrong name, I typed `_requests_per_second` at least once while writing.
+
+The answer here is a `Protocol` and a `cast`.
+
+Each augmentation now comes with a `typing.Protocol`:
+
+```python
+class Throttled(Protocol):
+    _last_request: float
+    _requests_per_minute: float
+    _request_frequency: float
+    _no_throttle_request: Callable
+    request: Callable
+```
+
+This defines all of the hidden state for the augmentation, as well as a placeholder for our overriden `request`.
+
+Then, our decorator looks like this:
+
+```python
+def _throttle_request(client: Throttled, *args, **kwargs) -> Response:
+```
+
+Which satisfies the type checker when it comes to internal use of those new attributes.
+
+The final change comes in where we initialize the attributes in the wrapper functions:
+
+```python
+    # a cast is made to the new type, allowing assignment
+    tclient = cast(ThrottledClient, client)
+
+    tclient._last_request = 0.0
+    tclient._requests_per_minute = requests_per_minute
+    tclient._request_frequency = 60.0 / requests_per_minute
+
+    tclient._no_throttle_request = client.request
+    tclient.request = types.MethodType(
+        functools.wraps(client.request)(_throttle_request), client
+    )
+    # the original client can be returned, of type `Client`
+    return client
+```
+
+
+### Closing thoughts
+
+With this approach, users do not know at any point they have a `ThrottledClient` or a `CachedClient`, etc.
+Not having the final type change is not ideal, but the compromise made for today.
+
+It would be nice to be able to expose an extra method or two, but this approach leans on only having private attributes & therefore being able to safely treat an augmented client as a `Client`.
+
+There's almost certainly room for improvement, but I'm fairly happy with the trade-offs for now.

careful-0.3.1/docs/index.md

@@ -0,0 +1,56 @@
+# careful
+
+<img src="carefully-3681327.svg" width=100 height=100 alt="logo of a warning sign">
+
+**careful** is a Python library for writing resilient, well-behaved HTTP clients.
+
+**Code**: <https://codeberg.org/jpt/careful>
+
+**Docs**: <https://jpt.sh/projects/careful/>
+
+![PyPI - Version](https://img.shields.io/pypi/v/careful)
+[![status-badge](https://ci.codeberg.org/api/badges/15185/status.svg)](https://ci.codeberg.org/repos/15185)
+
+Call one function to enchant an
+**[httpx.Client](https://www.python-httpx.org)**, making your HTTP connections more resilient and better mannered.
+
+- Configure **throttling** to avoid accidental Denial-of-Service / risking getting banned.
+- **Retries** help overcome intermittent failures on flaky sites or long crawls.
+- **Development caching** Cache persists between runs during development, reduces redundant requests made while iterating on your crawlers & scrapers.
+
+Additionally the library is fully typed, thorougly tested, and [based on code that powered production web scrapers for 15+ years](changelog#scrapelib).
+
+### Example
+
+```python
+from httpx import Client
+from careful.httpx import make_careful_client
+
+# the only function you need to call is make_careful_client
+# this wraps your existing `httpx.Client` with your preferred
+# careful behaviors
+
+client = make_careful_client(
+    client=Client(headers={'user-agent': 'spiderman/1.0'}),
+
+    # retries are configurable w/ exponential back off
+    retry_attempts=2,
+    retry_wait_seconds=5,
+
+    # can cache to process memory, filesystem, or SQLite
+    cache_storage=MemoryCache(),
+
+    # easy-to-configure throttling
+    requests_per_minute=60,
+)
+
+# methods on client are called as they always are
+# configured behaviors occur without further code changes
+client.get("https://example.com")
+```
+
+
+---
+
+Logo licensed from [Adrien Coquet via Noun Project](https://thenounproject.com/icon/carefully-3681327/)
+

careful-0.3.1/docs/reference.md

@@ -0,0 +1,139 @@
+# Usage
+
+Most users will only need to call `careful.httpx.make_careful_client`.
+
+::: careful.httpx.make_careful_client
+    options:
+      annotations_path: brief
+      show_signature: false
+      show_root_heading: true
+
+## Throttling
+
+If `requests_per_minute` is set, standard (non-retry) requests will automatically
+sleep for a short period to target the given rate.
+
+For example, at 30rpm, the sleep time on a fast request will be close to 2 seconds.
+
+```python
+client = make_careful_client(requests_per_minute=20)
+
+for page in range(10):
+  # will sleep ~3 seconds each time
+  client.get(f"https://example.com?page={page}")
+```
+
+## Retries
+
+If `retry_attempts` is set, responses will be passed to `should_retry`.
+Responses that are rejected (return `True`) will be retried after a wait based on
+`retry_wait_seconds`.
+Each retry will wait twice as long as the one before.
+
+```python
+client = make_careful_client(retry_attempts=2, retry_wait_seconds=30)
+
+# will try, wait 30s, try again, wait 60s, try again, then give up & return the 500
+client.get("https://httpbin.org/status/500")
+```
+
+There are a few simple pre-written `should_retry` predicates you can use:
+
+::: careful.httpx.retries.retry_default_rule
+    options:
+       heading_level: 3
+       show_root_heading: true
+::: careful.httpx.retries.retry_only_500s
+    options:
+       heading_level: 3
+       show_root_heading: true
+::: careful.httpx.retries.retry_all_400s_500s
+    options:
+       heading_level: 3
+       show_root_heading: true
+
+## Development Caching
+
+!!! warning "Why _development_ caching?"
+
+    This feature is named as a reminder that **this is not true HTTP caching**, which
+    should take various headers into account. Look at libraries like [hishel](https://hishel.com) if that's what you are after.
+
+The purpose of this feature is to allow you to cache all of your HTTP requests during development.
+Often when writing a scraper or crawler, you wind up hitting the site you are working on more often than you'd like-- each time you iterate on your code you're likely making redundant requests to pages that haven't changed.
+
+By caching all successful requests (configurable with the `should_cache` parameter),
+you can easily re-run scrapers without making redundant HTTP requests.
+This means much faster development & happier upstream servers.
+
+To enable development caching, assign a [`MemoryCache`][careful.httpx.MemoryCache],
+[`FileCache`][careful.httpx.FileCache], or [`SqliteCache`][careful.httpx.SqliteCache] to
+the `cache_storage` property of a `scrapelib.Scraper`.
+
+```python
+client = make_careful_client(
+  cache_storage=FileStorage("_cache")
+)
+
+# only one HTTP request is made
+client.get("https://example.com")
+client.get("https://example.com")
+client.get("https://example.com")
+client.get("https://example.com")
+# on subsequent runs, zero will be made until _cache is cleared
+```
+
+## robots.txt
+
+If `check_robots_txt` is set to `True`, then each request will be checked
+against that domain's `robots.txt`.
+(Each domain's `robots.txt` will be fetched once and cached.)
+
+Evaluating `robots.txt` requires a *user agent*. `careful` will use the user agent
+set on the `client` at the time of creation by default, but sometimes it is preferable
+to use a custom user agent for these checks. To do so, set the `robots_txt_user_agent` parameter.
+
+The default behavior when checking is enabled is that any attempt to fetch a denied page will
+raise a `RobotExclusionError`. This behavior can be overriden by passing a function to `robots_txt_on_reject` that takes two parameters: `(url: str, rfp: RobotFileParser)`.
+
+::: careful.httpx.RobotExclusionError
+    options:
+       heading_level: 3
+       members: False
+       show_root_heading: true
+
+::: careful.httpx.robots.warn_robots_txt
+    options:
+       heading_level: 3
+       members: False
+       show_root_heading: true
+
+### Note about multiple enhancements
+
+When multiple features are applied, the order of wrapping ensures that:
+
+- the cache is checked first, and bypasses throttling if hit
+- retries use their own delays, but are not throttled separately
+
+## Cache Storage Options
+
+These options are available for `cache_storage`:
+
+::: careful.httpx.MemoryCache
+    options:
+       heading_level: 3
+       members: False
+       show_root_heading: true
+
+::: careful.httpx.FileCache
+    options:
+       heading_level: 3
+       members: False
+       show_root_heading: true
+
+::: careful.httpx.SqliteCache
+    options:
+       heading_level: 3
+       members: False
+       show_root_heading: true
+

{careful-0.2.1 → careful-0.3.1}/mkdocs.yml

@@ -1,5 +1,5 @@
 site_name: careful
-site_url: https://careful.jpt.sh/
+site_url: https://jpt.sh/projects/careful/
 site_author: James Turk
 site_description: A library for making requests to unreliable sites with httpx.
 copyright: Copyright © 2025 James Turk
@@ -49,4 +49,5 @@ watch:
 nav:
   - 'index.md'
   - 'reference.md'
+  - 'design.md'
   - 'changelog.md'

{careful-0.2.1 → careful-0.3.1}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "careful"
-version = "0.2.1"
-description = "careful extensions to httpx: throttle, retry, cache"
+version = "0.3.1"
+description = "a small library for writing resilient, well-behaved HTTP code"
 readme = "README.md"
 authors = [
     { name = "jpt", email = "dev@jpt.sh" }
@@ -18,6 +18,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [