careful 0.3.1__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {careful-0.3.1 → careful-0.3.2}/PKG-INFO +1 -1
- {careful-0.3.1 → careful-0.3.2}/docs/changelog.md +6 -1
- {careful-0.3.1 → careful-0.3.2}/pyproject.toml +1 -1
- {careful-0.3.1 → careful-0.3.2}/src/careful/httpx/__init__.py +19 -2
- {careful-0.3.1 → careful-0.3.2}/src/careful/httpx/dev_cache.py +14 -2
- {careful-0.3.1 → careful-0.3.2}/.gitignore +0 -0
- {careful-0.3.1 → careful-0.3.2}/.pre-commit-config.yaml +0 -0
- {careful-0.3.1 → careful-0.3.2}/.spellignore +0 -0
- {careful-0.3.1 → careful-0.3.2}/.woodpecker.yml +0 -0
- {careful-0.3.1 → careful-0.3.2}/Justfile +0 -0
- {careful-0.3.1 → careful-0.3.2}/LICENSE +0 -0
- {careful-0.3.1 → careful-0.3.2}/README.md +0 -0
- {careful-0.3.1 → careful-0.3.2}/docs/carefully-3681327.svg +0 -0
- {careful-0.3.1 → careful-0.3.2}/docs/design.md +0 -0
- {careful-0.3.1 → careful-0.3.2}/docs/index.md +0 -0
- {careful-0.3.1 → careful-0.3.2}/docs/reference.md +0 -0
- {careful-0.3.1 → careful-0.3.2}/mkdocs.yml +0 -0
- {careful-0.3.1 → careful-0.3.2}/src/careful/__init__.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/src/careful/httpx/_types.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/src/careful/httpx/py.typed +0 -0
- {careful-0.3.1 → careful-0.3.2}/src/careful/httpx/retries.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/src/careful/httpx/robots.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/src/careful/httpx/throttle.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/tests/fakeresponse.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/tests/test_cache.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/tests/test_careful.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/tests/test_retries.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/tests/test_robots_txt.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/tests/test_throttle.py +0 -0
- {careful-0.3.1 → careful-0.3.2}/trifold.toml +0 -0
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
|
+
import logging
|
|
3
|
+
import pathlib
|
|
2
4
|
from urllib.parse import urlparse
|
|
3
5
|
from .retries import make_retry_client, retry_default_rule
|
|
4
6
|
from .throttle import make_throttled_client
|
|
@@ -21,6 +23,8 @@ from .dev_cache import (
|
|
|
21
23
|
from ._types import ResponsePredicate, CacheKeyfunc
|
|
22
24
|
from httpx import Client
|
|
23
25
|
|
|
26
|
+
log = logging.getLogger("careful")
|
|
27
|
+
|
|
24
28
|
|
|
25
29
|
def make_careful_client(
|
|
26
30
|
*,
|
|
@@ -120,12 +124,25 @@ def _cache_env(var_name: str, default: CacheStorage | None) -> CacheStorage | No
|
|
|
120
124
|
if not cache_str:
|
|
121
125
|
return default
|
|
122
126
|
parsed = urlparse(cache_str)
|
|
127
|
+
# urlparse always starts with a / (var needs :/// to skip netloc -> into path)
|
|
128
|
+
true_path = parsed.path[1:] if parsed.path.startswith("/") else parsed.path
|
|
129
|
+
# if it starts with a //// then it is an absolute path
|
|
130
|
+
if true_path.startswith("/"):
|
|
131
|
+
path = pathlib.Path(true_path)
|
|
132
|
+
else:
|
|
133
|
+
path = pathlib.Path.cwd() / true_path
|
|
123
134
|
if parsed.scheme == "memory":
|
|
135
|
+
log.info("cache from env %s => MemoryCache", var_name)
|
|
124
136
|
return MemoryCache()
|
|
125
137
|
elif parsed.scheme == "file":
|
|
126
|
-
|
|
138
|
+
log.info("cache from env %s => FileCache(%s)", var_name, path)
|
|
139
|
+
return FileCache(path)
|
|
127
140
|
elif parsed.scheme == "sqlite":
|
|
128
|
-
|
|
141
|
+
log.info("cache from env %s => SqliteCache(%s)", var_name, path)
|
|
142
|
+
return SqliteCache(path)
|
|
143
|
+
else:
|
|
144
|
+
log.warning("invalid cache %s=%s", var_name, cache_str)
|
|
145
|
+
return default
|
|
129
146
|
|
|
130
147
|
|
|
131
148
|
def make_careful_client_from_env(
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import types
|
|
2
2
|
import functools
|
|
3
|
+
import pathlib
|
|
3
4
|
import logging
|
|
4
5
|
import re
|
|
5
6
|
import os
|
|
@@ -238,6 +239,7 @@ class FileCache(CacheStorage):
|
|
|
238
239
|
content=resp_content,
|
|
239
240
|
default_encoding=resp_headers.pop("encoding"),
|
|
240
241
|
headers=resp_headers,
|
|
242
|
+
request=Request("GET", key), # not perfect, but it'll do
|
|
241
243
|
)
|
|
242
244
|
return resp
|
|
243
245
|
except IOError:
|
|
@@ -254,6 +256,8 @@ class FileCache(CacheStorage):
|
|
|
254
256
|
encoding_str = "encoding: {0}\n".format(response.encoding)
|
|
255
257
|
f.write(encoding_str.encode("utf8"))
|
|
256
258
|
for h, v in response.headers.items():
|
|
259
|
+
if h.lower() in ("content-encoding", "content-length"):
|
|
260
|
+
continue
|
|
257
261
|
# header: value\n
|
|
258
262
|
f.write(h.encode("utf8"))
|
|
259
263
|
f.write(b": ")
|
|
@@ -286,12 +290,17 @@ class SqliteCache(CacheStorage):
|
|
|
286
290
|
|
|
287
291
|
_columns = ["key", "status", "modified", "encoding", "data", "headers"]
|
|
288
292
|
|
|
289
|
-
def __init__(
|
|
293
|
+
def __init__(
|
|
294
|
+
self, cache_path: str | pathlib.Path, check_last_modified: bool = False
|
|
295
|
+
):
|
|
290
296
|
self.cache_path = cache_path
|
|
291
297
|
self.check_last_modified = check_last_modified
|
|
292
|
-
self._conn = sqlite3.connect(cache_path)
|
|
298
|
+
self._conn = sqlite3.connect(str(cache_path))
|
|
293
299
|
self._conn.text_factory = str
|
|
294
300
|
self._build_table()
|
|
301
|
+
# self._conn.execute("PRAGMA journal_mode=WAL;")
|
|
302
|
+
# self._conn.execute("PRAGMA synchronous=1;")
|
|
303
|
+
# self._conn.isolation_level = None
|
|
295
304
|
|
|
296
305
|
def _build_table(self) -> None:
|
|
297
306
|
"""Create table for storing request information and response."""
|
|
@@ -304,6 +313,8 @@ class SqliteCache(CacheStorage):
|
|
|
304
313
|
def set(self, key: str, response: Response) -> None:
|
|
305
314
|
"""Set cache entry for key with contents of response."""
|
|
306
315
|
mod = response.headers.pop("last-modified", None)
|
|
316
|
+
response.headers.pop("content-encoding", None)
|
|
317
|
+
response.headers.pop("content-length", None)
|
|
307
318
|
status = int(response.status_code)
|
|
308
319
|
rec = (
|
|
309
320
|
key,
|
|
@@ -340,6 +351,7 @@ class SqliteCache(CacheStorage):
|
|
|
340
351
|
content=rec["data"],
|
|
341
352
|
default_encoding=rec["encoding"],
|
|
342
353
|
headers=json.loads(rec["headers"]),
|
|
354
|
+
request=Request("GET", key), # not perfect, but it'll do
|
|
343
355
|
)
|
|
344
356
|
return resp
|
|
345
357
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|