nab-index 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nab_index-0.0.1/LICENSE +21 -0
- nab_index-0.0.1/PKG-INFO +55 -0
- nab_index-0.0.1/README.md +26 -0
- nab_index-0.0.1/pyproject.toml +40 -0
- nab_index-0.0.1/src/nab_index/__init__.py +1 -0
- nab_index-0.0.1/src/nab_index/_naming.py +25 -0
- nab_index-0.0.1/src/nab_index/cache.py +277 -0
- nab_index-0.0.1/src/nab_index/cached_client.py +236 -0
- nab_index-0.0.1/src/nab_index/client.py +362 -0
- nab_index-0.0.1/src/nab_index/httpx_async_transport.py +41 -0
- nab_index-0.0.1/src/nab_index/local_index.py +276 -0
- nab_index-0.0.1/src/nab_index/multi_index.py +254 -0
- nab_index-0.0.1/src/nab_index/niquests_async_transport.py +47 -0
- nab_index-0.0.1/src/nab_index/py.typed +0 -0
- nab_index-0.0.1/src/nab_index/transport.py +69 -0
- nab_index-0.0.1/src/nab_index/urllib3_async_transport.py +85 -0
- nab_index-0.0.1/src/nab_index/vcs.py +261 -0
nab_index-0.0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Damian Shaw
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
nab_index-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nab-index
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: PyPI Simple-API client and on-disk cache for nab
|
|
5
|
+
Project-URL: Homepage, https://github.com/notatallshaw/nab
|
|
6
|
+
Project-URL: Documentation, https://nab.readthedocs.io/
|
|
7
|
+
Project-URL: Issues, https://github.com/notatallshaw/nab/issues
|
|
8
|
+
Project-URL: Source, https://github.com/notatallshaw/nab
|
|
9
|
+
Project-URL: Changelog, https://github.com/notatallshaw/nab/blob/main/CHANGELOG.md
|
|
10
|
+
Author-email: Damian Shaw <damian.peter.shaw@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: packaging>=24.0
|
|
22
|
+
Requires-Dist: typing-extensions>=4.6
|
|
23
|
+
Requires-Dist: urllib3>=2.0
|
|
24
|
+
Provides-Extra: httpx
|
|
25
|
+
Requires-Dist: httpx[http2]>=0.28; extra == 'httpx'
|
|
26
|
+
Provides-Extra: niquests
|
|
27
|
+
Requires-Dist: niquests>=3.18; extra == 'niquests'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# nab-index
|
|
31
|
+
|
|
32
|
+
PyPI Simple-API client and on-disk cache used by
|
|
33
|
+
[`nab-python`](https://pypi.org/project/nab-python/) and
|
|
34
|
+
[`nab`](https://pypi.org/project/nab/).
|
|
35
|
+
|
|
36
|
+
It provides:
|
|
37
|
+
|
|
38
|
+
- A small async transport interface with three drop-in backends:
|
|
39
|
+
- `urllib3` (default; pulled in by the base install).
|
|
40
|
+
- `httpx` (extra: `nab-index[httpx]`).
|
|
41
|
+
- `niquests` (extra: `nab-index[niquests]`).
|
|
42
|
+
- A Simple-API client with JSON and HTML decoders.
|
|
43
|
+
- A disk cache for project listings and file metadata responses.
|
|
44
|
+
- A multi-index router (ordered named indexes plus per-package
|
|
45
|
+
overrides guarded by PEP 508 markers).
|
|
46
|
+
- A small VCS clone helper used by the higher-level VCS policy.
|
|
47
|
+
|
|
48
|
+
## When to use it
|
|
49
|
+
|
|
50
|
+
Use `nab-index` if you need a typed PyPI Simple-API client with an
|
|
51
|
+
on-disk cache. Most users want
|
|
52
|
+
[`nab`](https://pypi.org/project/nab/) instead.
|
|
53
|
+
|
|
54
|
+
The API is currently under rapid experimentation, if you use it
|
|
55
|
+
pin to an exact version.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# nab-index
|
|
2
|
+
|
|
3
|
+
PyPI Simple-API client and on-disk cache used by
|
|
4
|
+
[`nab-python`](https://pypi.org/project/nab-python/) and
|
|
5
|
+
[`nab`](https://pypi.org/project/nab/).
|
|
6
|
+
|
|
7
|
+
It provides:
|
|
8
|
+
|
|
9
|
+
- A small async transport interface with three drop-in backends:
|
|
10
|
+
- `urllib3` (default; pulled in by the base install).
|
|
11
|
+
- `httpx` (extra: `nab-index[httpx]`).
|
|
12
|
+
- `niquests` (extra: `nab-index[niquests]`).
|
|
13
|
+
- A Simple-API client with JSON and HTML decoders.
|
|
14
|
+
- A disk cache for project listings and file metadata responses.
|
|
15
|
+
- A multi-index router (ordered named indexes plus per-package
|
|
16
|
+
overrides guarded by PEP 508 markers).
|
|
17
|
+
- A small VCS clone helper used by the higher-level VCS policy.
|
|
18
|
+
|
|
19
|
+
## When to use it
|
|
20
|
+
|
|
21
|
+
Use `nab-index` if you need a typed PyPI Simple-API client with an
|
|
22
|
+
on-disk cache. Most users want
|
|
23
|
+
[`nab`](https://pypi.org/project/nab/) instead.
|
|
24
|
+
|
|
25
|
+
The API is currently under rapid experimentation, if you use it
|
|
26
|
+
pin to an exact version.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "nab-index"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "PyPI Simple-API client and on-disk cache for nab"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Damian Shaw", email = "damian.peter.shaw@gmail.com" },
|
|
10
|
+
]
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 4 - Beta",
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Programming Language :: Python :: 3.10",
|
|
16
|
+
"Programming Language :: Python :: 3.11",
|
|
17
|
+
"Programming Language :: Python :: 3.12",
|
|
18
|
+
"Programming Language :: Python :: 3.13",
|
|
19
|
+
"Typing :: Typed",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"packaging>=24.0",
|
|
23
|
+
"typing_extensions>=4.6",
|
|
24
|
+
"urllib3>=2.0",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
httpx = ["httpx[http2]>=0.28"]
|
|
29
|
+
niquests = ["niquests>=3.18"]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/notatallshaw/nab"
|
|
33
|
+
Documentation = "https://nab.readthedocs.io/"
|
|
34
|
+
Issues = "https://github.com/notatallshaw/nab/issues"
|
|
35
|
+
Source = "https://github.com/notatallshaw/nab"
|
|
36
|
+
Changelog = "https://github.com/notatallshaw/nab/blob/main/CHANGELOG.md"
|
|
37
|
+
|
|
38
|
+
[build-system]
|
|
39
|
+
requires = ["hatchling"]
|
|
40
|
+
build-backend = "hatchling.build"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""PyPI Simple-API client and on-disk cache."""
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""PEP 503 name canonicalisation for nab-index.
|
|
2
|
+
|
|
3
|
+
Wraps :func:`packaging.utils.canonicalize_name` so the local- and
|
|
4
|
+
multi-index modules share a single helper. ``packaging`` is already
|
|
5
|
+
a runtime dependency (see ``pyproject.toml``), so deferring to it
|
|
6
|
+
avoids the divergent regexes the helpers used to carry.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from packaging.utils import canonicalize_name
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"canonical",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def canonical(name: str) -> str:
|
|
19
|
+
"""Return ``name`` as its PEP 503 canonical form.
|
|
20
|
+
|
|
21
|
+
Lower-cases the input and collapses runs of ``-``, ``_``, ``.``
|
|
22
|
+
to a single ``-`` per PEP 503. Leading and trailing dashes are
|
|
23
|
+
preserved (consistent with :func:`packaging.utils.canonicalize_name`).
|
|
24
|
+
"""
|
|
25
|
+
return canonicalize_name(name)
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"""On-disk cache for nab-index.
|
|
2
|
+
|
|
3
|
+
Stores PEP 691 Simple API responses (raw JSON body plus a sidecar
|
|
4
|
+
cache policy file) and PEP 658 wheel metadata (raw text, treated as
|
|
5
|
+
immutable). The cache is consulted by :class:`CachedAsyncSimpleClient`
|
|
6
|
+
before any HTTP transport call.
|
|
7
|
+
|
|
8
|
+
Layout under ``root``:
|
|
9
|
+
|
|
10
|
+
simple-v0/<index>/<package>.json <- raw PyPI JSON body
|
|
11
|
+
simple-v0/<index>/<package>.policy <- {fetched_at, max_age, etag}
|
|
12
|
+
metadata-v0/<index>/<package>/<version>.metadata
|
|
13
|
+
sdist-pkginfo-v0/<index>/<package>/<version>.txt
|
|
14
|
+
|
|
15
|
+
A versioned bucket name (``simple-v0``) gives zero-cost schema
|
|
16
|
+
migration: when the on-disk format changes, bump the suffix and the
|
|
17
|
+
old directory is harmless.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import contextlib
|
|
23
|
+
import hashlib
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import tempfile
|
|
27
|
+
import time
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Protocol
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"CacheBackend",
|
|
34
|
+
"CachePolicy",
|
|
35
|
+
"NullCache",
|
|
36
|
+
"OfflineError",
|
|
37
|
+
"OnDiskCache",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
CACHE_VERSION_SIMPLE = "v0"
|
|
42
|
+
CACHE_VERSION_METADATA = "v0"
|
|
43
|
+
CACHE_VERSION_SDIST = "v0"
|
|
44
|
+
|
|
45
|
+
DEFAULT_PYPI_URLS = frozenset(
|
|
46
|
+
[
|
|
47
|
+
"https://pypi.org/simple",
|
|
48
|
+
"http://pypi.org/simple",
|
|
49
|
+
]
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class OfflineError(Exception):
|
|
54
|
+
"""Raised when offline mode is set and a needed entry is not cached."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True, slots=True)
|
|
58
|
+
class CachePolicy:
|
|
59
|
+
"""RFC 9111-style freshness policy for one Simple API entry."""
|
|
60
|
+
|
|
61
|
+
fetched_at: int
|
|
62
|
+
max_age: int
|
|
63
|
+
etag: str | None
|
|
64
|
+
|
|
65
|
+
def is_fresh(self, now: int | None = None) -> bool:
|
|
66
|
+
"""Return True if the entry is still within its freshness window."""
|
|
67
|
+
current = int(time.time()) if now is None else now
|
|
68
|
+
return current - self.fetched_at < self.max_age
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _index_dirname(index_url: str) -> str:
|
|
72
|
+
"""Return a stable, filesystem-safe directory name for an index URL."""
|
|
73
|
+
if index_url.rstrip("/") in DEFAULT_PYPI_URLS:
|
|
74
|
+
return "pypi"
|
|
75
|
+
return hashlib.sha256(index_url.encode("utf-8")).hexdigest()[:16]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _atomic_write(path: Path, data: bytes) -> None:
|
|
79
|
+
"""Write ``data`` to ``path`` atomically via tempfile + replace.
|
|
80
|
+
|
|
81
|
+
The temp file is created in the destination directory so the rename
|
|
82
|
+
is a same-filesystem operation. A partial write or a crash leaves
|
|
83
|
+
the target file untouched.
|
|
84
|
+
"""
|
|
85
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
86
|
+
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=path.name + ".", suffix=".tmp")
|
|
87
|
+
tmp_path = Path(tmp)
|
|
88
|
+
try:
|
|
89
|
+
with os.fdopen(fd, "wb") as f:
|
|
90
|
+
f.write(data)
|
|
91
|
+
tmp_path.replace(path)
|
|
92
|
+
except BaseException:
|
|
93
|
+
with contextlib.suppress(OSError):
|
|
94
|
+
tmp_path.unlink()
|
|
95
|
+
raise
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _read_text(path: Path) -> str | None:
|
|
99
|
+
"""Return ``path``'s UTF-8 contents, or ``None`` if the file is absent."""
|
|
100
|
+
try:
|
|
101
|
+
return path.read_text(encoding="utf-8")
|
|
102
|
+
except OSError:
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class OnDiskCache:
|
|
107
|
+
"""File-per-key cache for Simple API and wheel metadata."""
|
|
108
|
+
|
|
109
|
+
def __init__(self, root: Path, index_url: str) -> None:
|
|
110
|
+
"""Create a cache rooted at ``root`` for ``index_url``."""
|
|
111
|
+
self._root = root
|
|
112
|
+
self._index = _index_dirname(index_url)
|
|
113
|
+
self._simple_dir = root / f"simple-{CACHE_VERSION_SIMPLE}" / self._index
|
|
114
|
+
self._metadata_dir = root / f"metadata-{CACHE_VERSION_METADATA}" / self._index
|
|
115
|
+
self._sdist_dir = root / f"sdist-pkginfo-{CACHE_VERSION_SDIST}" / self._index
|
|
116
|
+
self._sdist_pyproject_dir = (
|
|
117
|
+
root / f"sdist-pyproject-{CACHE_VERSION_SDIST}" / self._index
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
def _simple_paths(self, package: str) -> tuple[Path, Path]:
|
|
121
|
+
body = self._simple_dir / f"{package}.json"
|
|
122
|
+
policy = self._simple_dir / f"{package}.policy"
|
|
123
|
+
return (body, policy)
|
|
124
|
+
|
|
125
|
+
def get_simple(self, package: str) -> tuple[bytes, CachePolicy] | None:
|
|
126
|
+
"""Return ``(body_bytes, policy)`` if cached, else ``None``."""
|
|
127
|
+
body_path, policy_path = self._simple_paths(package)
|
|
128
|
+
try:
|
|
129
|
+
policy_bytes = policy_path.read_bytes()
|
|
130
|
+
body = body_path.read_bytes()
|
|
131
|
+
except OSError:
|
|
132
|
+
return None
|
|
133
|
+
try:
|
|
134
|
+
policy_doc = json.loads(policy_bytes)
|
|
135
|
+
policy = CachePolicy(
|
|
136
|
+
fetched_at=int(policy_doc["fetched_at"]),
|
|
137
|
+
max_age=int(policy_doc["max_age"]),
|
|
138
|
+
etag=policy_doc.get("etag"),
|
|
139
|
+
)
|
|
140
|
+
except (ValueError, KeyError, TypeError):
|
|
141
|
+
return None
|
|
142
|
+
return (body, policy)
|
|
143
|
+
|
|
144
|
+
def put_simple(self, package: str, body: bytes, policy: CachePolicy) -> None:
|
|
145
|
+
"""Write the body and the policy sidecar atomically."""
|
|
146
|
+
body_path, policy_path = self._simple_paths(package)
|
|
147
|
+
_atomic_write(body_path, body)
|
|
148
|
+
_atomic_write(policy_path, _encode_policy(policy))
|
|
149
|
+
|
|
150
|
+
def refresh_simple_policy(self, package: str, policy: CachePolicy) -> None:
|
|
151
|
+
"""Replace the policy sidecar without touching the body.
|
|
152
|
+
|
|
153
|
+
Called after a 304 Not Modified, where the cached body is still
|
|
154
|
+
valid but the freshness window has slid forward.
|
|
155
|
+
"""
|
|
156
|
+
_, policy_path = self._simple_paths(package)
|
|
157
|
+
_atomic_write(policy_path, _encode_policy(policy))
|
|
158
|
+
|
|
159
|
+
def get_metadata(self, package: str, version: str) -> str | None:
|
|
160
|
+
"""Return cached PEP 658 metadata text, or ``None`` on miss."""
|
|
161
|
+
return _read_text(self._metadata_dir / package / f"{version}.metadata")
|
|
162
|
+
|
|
163
|
+
def put_metadata(self, package: str, version: str, text: str) -> None:
|
|
164
|
+
"""Write PEP 658 metadata text. Treated as immutable."""
|
|
165
|
+
_atomic_write(
|
|
166
|
+
self._metadata_dir / package / f"{version}.metadata",
|
|
167
|
+
text.encode("utf-8"),
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
def get_sdist_pkginfo(self, package: str, version: str) -> str | None:
|
|
171
|
+
"""Return cached sdist PKG-INFO text, or ``None`` on miss."""
|
|
172
|
+
return _read_text(self._sdist_dir / package / f"{version}.txt")
|
|
173
|
+
|
|
174
|
+
def put_sdist_pkginfo(self, package: str, version: str, text: str) -> None:
|
|
175
|
+
"""Write sdist PKG-INFO text. Treated as immutable."""
|
|
176
|
+
_atomic_write(
|
|
177
|
+
self._sdist_dir / package / f"{version}.txt", text.encode("utf-8")
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
def get_sdist_pyproject(self, package: str, version: str) -> str | None:
|
|
181
|
+
"""Return cached sdist pyproject.toml text, or ``None`` on miss."""
|
|
182
|
+
return _read_text(self._sdist_pyproject_dir / package / f"{version}.toml")
|
|
183
|
+
|
|
184
|
+
def put_sdist_pyproject(self, package: str, version: str, text: str) -> None:
|
|
185
|
+
"""Write sdist pyproject.toml text. Treated as immutable."""
|
|
186
|
+
_atomic_write(
|
|
187
|
+
self._sdist_pyproject_dir / package / f"{version}.toml",
|
|
188
|
+
text.encode("utf-8"),
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _encode_policy(policy: CachePolicy) -> bytes:
|
|
193
|
+
return json.dumps(
|
|
194
|
+
{
|
|
195
|
+
"fetched_at": policy.fetched_at,
|
|
196
|
+
"max_age": policy.max_age,
|
|
197
|
+
"etag": policy.etag,
|
|
198
|
+
}
|
|
199
|
+
).encode("utf-8")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
class CacheBackend(Protocol):
|
|
203
|
+
"""Protocol shared by :class:`OnDiskCache` and :class:`NullCache`."""
|
|
204
|
+
|
|
205
|
+
def get_simple(self, package: str) -> tuple[bytes, CachePolicy] | None:
|
|
206
|
+
"""Return ``(body_bytes, policy)`` if cached, else ``None``."""
|
|
207
|
+
...
|
|
208
|
+
|
|
209
|
+
def put_simple(self, package: str, body: bytes, policy: CachePolicy) -> None:
|
|
210
|
+
"""Store a Simple API body and its freshness policy."""
|
|
211
|
+
...
|
|
212
|
+
|
|
213
|
+
def refresh_simple_policy(self, package: str, policy: CachePolicy) -> None:
|
|
214
|
+
"""Update the policy for an existing entry without rewriting the body."""
|
|
215
|
+
...
|
|
216
|
+
|
|
217
|
+
def get_metadata(self, package: str, version: str) -> str | None:
|
|
218
|
+
"""Return cached PEP 658 metadata text, or ``None`` on miss."""
|
|
219
|
+
...
|
|
220
|
+
|
|
221
|
+
def put_metadata(self, package: str, version: str, text: str) -> None:
|
|
222
|
+
"""Store PEP 658 metadata text. Treated as immutable."""
|
|
223
|
+
...
|
|
224
|
+
|
|
225
|
+
def get_sdist_pkginfo(self, package: str, version: str) -> str | None:
|
|
226
|
+
"""Return cached sdist PKG-INFO text, or ``None`` on miss."""
|
|
227
|
+
...
|
|
228
|
+
|
|
229
|
+
def put_sdist_pkginfo(self, package: str, version: str, text: str) -> None:
|
|
230
|
+
"""Store sdist PKG-INFO text. Treated as immutable."""
|
|
231
|
+
...
|
|
232
|
+
|
|
233
|
+
def get_sdist_pyproject(self, package: str, version: str) -> str | None:
|
|
234
|
+
"""Return cached sdist pyproject.toml text, or ``None`` on miss."""
|
|
235
|
+
...
|
|
236
|
+
|
|
237
|
+
def put_sdist_pyproject(self, package: str, version: str, text: str) -> None:
|
|
238
|
+
"""Store sdist pyproject.toml text. Treated as immutable."""
|
|
239
|
+
...
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
class NullCache:
|
|
243
|
+
"""No-op cache backend used when persistence is disabled.
|
|
244
|
+
|
|
245
|
+
Lets :class:`CachedAsyncSimpleClient` be used unconditionally so
|
|
246
|
+
the call site does not branch on whether a cache is configured.
|
|
247
|
+
Each method is a docstring-only stub: gets implicitly return
|
|
248
|
+
``None`` (a permanent miss) and puts implicitly do nothing.
|
|
249
|
+
Argument names match :class:`CacheBackend` for Protocol conformance.
|
|
250
|
+
"""
|
|
251
|
+
|
|
252
|
+
def get_simple(self, package: str) -> tuple[bytes, CachePolicy] | None:
|
|
253
|
+
"""Return ``None`` (always a miss)."""
|
|
254
|
+
|
|
255
|
+
def put_simple(self, package: str, body: bytes, policy: CachePolicy) -> None:
|
|
256
|
+
"""Discard the entry."""
|
|
257
|
+
|
|
258
|
+
def refresh_simple_policy(self, package: str, policy: CachePolicy) -> None:
|
|
259
|
+
"""Discard the policy refresh."""
|
|
260
|
+
|
|
261
|
+
def get_metadata(self, package: str, version: str) -> str | None:
|
|
262
|
+
"""Return ``None`` (always a miss)."""
|
|
263
|
+
|
|
264
|
+
def put_metadata(self, package: str, version: str, text: str) -> None:
|
|
265
|
+
"""Discard the entry."""
|
|
266
|
+
|
|
267
|
+
def get_sdist_pkginfo(self, package: str, version: str) -> str | None:
|
|
268
|
+
"""Return ``None`` (always a miss)."""
|
|
269
|
+
|
|
270
|
+
def put_sdist_pkginfo(self, package: str, version: str, text: str) -> None:
|
|
271
|
+
"""Discard the entry."""
|
|
272
|
+
|
|
273
|
+
def get_sdist_pyproject(self, package: str, version: str) -> str | None:
|
|
274
|
+
"""Return ``None`` (always a miss)."""
|
|
275
|
+
|
|
276
|
+
def put_sdist_pyproject(self, package: str, version: str, text: str) -> None:
|
|
277
|
+
"""Discard the entry."""
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Disk-cached PyPI Simple API client.
|
|
2
|
+
|
|
3
|
+
Wraps :class:`AsyncSimpleClient`. Consults an :class:`OnDiskCache`
|
|
4
|
+
before any HTTP transport call. Honors a small subset of RFC 9111:
|
|
5
|
+
fresh entries are served directly, stale entries are revalidated with
|
|
6
|
+
``If-None-Match``, and PEP 658 metadata + sdist PKG-INFO are treated
|
|
7
|
+
as immutable (cached forever; never revalidated).
|
|
8
|
+
|
|
9
|
+
Cache hits short-circuit before any HTTP code runs. With a fully
|
|
10
|
+
populated cache the client never opens a socket.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import time
|
|
18
|
+
from typing import TYPE_CHECKING
|
|
19
|
+
|
|
20
|
+
from .cache import CacheBackend, CachePolicy, OfflineError
|
|
21
|
+
from .client import (
|
|
22
|
+
DEFAULT_INDEX,
|
|
23
|
+
SdistFile,
|
|
24
|
+
WheelFile,
|
|
25
|
+
_extract_sdist_files,
|
|
26
|
+
_parse_files,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from typing_extensions import Self
|
|
31
|
+
|
|
32
|
+
from .transport import AsyncHttpTransport, HttpResponse
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"CachedAsyncSimpleClient",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
_JSON_ACCEPT = "application/vnd.pypi.simple.v1+json"
|
|
40
|
+
_DEFAULT_MAX_AGE = 600
|
|
41
|
+
_HTTP_NOT_MODIFIED = 304
|
|
42
|
+
_MAX_AGE_RE = re.compile(r"max-age\s*=\s*(\d+)")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _parse_max_age(cache_control: str | None) -> int:
|
|
46
|
+
if cache_control is None:
|
|
47
|
+
return _DEFAULT_MAX_AGE
|
|
48
|
+
match = _MAX_AGE_RE.search(cache_control)
|
|
49
|
+
if match is None:
|
|
50
|
+
return _DEFAULT_MAX_AGE
|
|
51
|
+
return int(match.group(1))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _header(response: HttpResponse, key: str) -> str | None:
|
|
55
|
+
"""Case-insensitive header lookup.
|
|
56
|
+
|
|
57
|
+
The :class:`HttpResponse` Protocol only promises a plain
|
|
58
|
+
:class:`Mapping`. All real transports (httpx, niquests, urllib3)
|
|
59
|
+
return case-insensitive header containers, but we don't rely on
|
|
60
|
+
that here so a plain-dict fake also works.
|
|
61
|
+
"""
|
|
62
|
+
headers = response.headers
|
|
63
|
+
target = key.lower()
|
|
64
|
+
for name, value in headers.items():
|
|
65
|
+
if name.lower() == target:
|
|
66
|
+
return value
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class CachedAsyncSimpleClient:
|
|
71
|
+
"""Async PyPI Simple API client with on-disk caching.
|
|
72
|
+
|
|
73
|
+
Distinct from :class:`AsyncSimpleClient`: the metadata methods
|
|
74
|
+
take ``(package, version)`` so the cache can key by package
|
|
75
|
+
coordinate rather than URL. The interface mirrors what
|
|
76
|
+
:class:`FetchCoordinator` actually needs.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
def __init__(
|
|
80
|
+
self,
|
|
81
|
+
transport: AsyncHttpTransport,
|
|
82
|
+
cache: CacheBackend,
|
|
83
|
+
index_url: str = DEFAULT_INDEX,
|
|
84
|
+
*,
|
|
85
|
+
offline: bool = False,
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Create a cached client wrapping ``transport``."""
|
|
88
|
+
self._transport = transport
|
|
89
|
+
self._cache = cache
|
|
90
|
+
self._index_url = index_url.rstrip("/") + "/"
|
|
91
|
+
self._offline = offline
|
|
92
|
+
|
|
93
|
+
async def aclose(self) -> None:
|
|
94
|
+
"""Close the underlying transport."""
|
|
95
|
+
await self._transport.aclose()
|
|
96
|
+
|
|
97
|
+
async def __aenter__(self) -> Self:
|
|
98
|
+
"""Enter the async context manager."""
|
|
99
|
+
return self
|
|
100
|
+
|
|
101
|
+
async def __aexit__(self, *args: object) -> None:
|
|
102
|
+
"""Exit the async context manager and close the transport."""
|
|
103
|
+
await self.aclose()
|
|
104
|
+
|
|
105
|
+
async def get_files(self, package: str) -> list[WheelFile | SdistFile]:
|
|
106
|
+
"""Return parsed Simple API file list for ``package``.
|
|
107
|
+
|
|
108
|
+
Cache hit + fresh: parses cached body, no network.
|
|
109
|
+
Cache hit + stale + online: conditional revalidation; on 304
|
|
110
|
+
the body is reused, on 200 the body is replaced.
|
|
111
|
+
Cache hit + offline: cached body is returned regardless of age.
|
|
112
|
+
Cache miss + offline: raises :class:`OfflineError`.
|
|
113
|
+
Cache miss + online: fetches, caches, returns.
|
|
114
|
+
"""
|
|
115
|
+
cached = self._cache.get_simple(package)
|
|
116
|
+
if cached is not None:
|
|
117
|
+
body, policy = cached
|
|
118
|
+
if policy.is_fresh() or self._offline:
|
|
119
|
+
return _parse_files(json.loads(body), self._index_url, package)
|
|
120
|
+
return await self._revalidate_simple(package, body, policy)
|
|
121
|
+
|
|
122
|
+
if self._offline:
|
|
123
|
+
msg = f"No cached listing for {package} (offline mode)"
|
|
124
|
+
raise OfflineError(msg)
|
|
125
|
+
return await self._fetch_simple(package)
|
|
126
|
+
|
|
127
|
+
async def _revalidate_simple(
|
|
128
|
+
self, package: str, body: bytes, policy: CachePolicy
|
|
129
|
+
) -> list[WheelFile | SdistFile]:
|
|
130
|
+
url = f"{self._index_url}{package}/"
|
|
131
|
+
headers = {"Accept": _JSON_ACCEPT}
|
|
132
|
+
if policy.etag is not None:
|
|
133
|
+
headers["If-None-Match"] = policy.etag
|
|
134
|
+
response = await self._transport.get(url, headers=headers)
|
|
135
|
+
if response.status_code == _HTTP_NOT_MODIFIED:
|
|
136
|
+
new_policy = CachePolicy(
|
|
137
|
+
fetched_at=int(time.time()),
|
|
138
|
+
max_age=_parse_max_age(_header(response, "cache-control")),
|
|
139
|
+
etag=_header(response, "etag") or policy.etag,
|
|
140
|
+
)
|
|
141
|
+
self._cache.refresh_simple_policy(package, new_policy)
|
|
142
|
+
return _parse_files(json.loads(body), self._index_url, package)
|
|
143
|
+
|
|
144
|
+
response.raise_for_status()
|
|
145
|
+
new_body = response.content
|
|
146
|
+
new_policy = CachePolicy(
|
|
147
|
+
fetched_at=int(time.time()),
|
|
148
|
+
max_age=_parse_max_age(_header(response, "cache-control")),
|
|
149
|
+
etag=_header(response, "etag"),
|
|
150
|
+
)
|
|
151
|
+
self._cache.put_simple(package, new_body, new_policy)
|
|
152
|
+
return _parse_files(json.loads(new_body), self._index_url, package)
|
|
153
|
+
|
|
154
|
+
async def _fetch_simple(self, package: str) -> list[WheelFile | SdistFile]:
|
|
155
|
+
url = f"{self._index_url}{package}/"
|
|
156
|
+
response = await self._transport.get(url, headers={"Accept": _JSON_ACCEPT})
|
|
157
|
+
response.raise_for_status()
|
|
158
|
+
body = response.content
|
|
159
|
+
policy = CachePolicy(
|
|
160
|
+
fetched_at=int(time.time()),
|
|
161
|
+
max_age=_parse_max_age(_header(response, "cache-control")),
|
|
162
|
+
etag=_header(response, "etag"),
|
|
163
|
+
)
|
|
164
|
+
self._cache.put_simple(package, body, policy)
|
|
165
|
+
return _parse_files(json.loads(body), self._index_url, package)
|
|
166
|
+
|
|
167
|
+
async def get_metadata_text(
|
|
168
|
+
self, package: str, version: str, metadata_url: str
|
|
169
|
+
) -> str:
|
|
170
|
+
"""Return PEP 658 metadata text for ``(package, version)``.
|
|
171
|
+
|
|
172
|
+
Treated as immutable: cached forever, never revalidated.
|
|
173
|
+
Cache miss + offline raises :class:`OfflineError`.
|
|
174
|
+
"""
|
|
175
|
+
cached = self._cache.get_metadata(package, version)
|
|
176
|
+
if cached is not None:
|
|
177
|
+
return cached
|
|
178
|
+
|
|
179
|
+
if self._offline:
|
|
180
|
+
msg = f"No cached metadata for {package}=={version} (offline mode)"
|
|
181
|
+
raise OfflineError(msg)
|
|
182
|
+
|
|
183
|
+
response = await self._transport.get(metadata_url)
|
|
184
|
+
response.raise_for_status()
|
|
185
|
+
text = response.text
|
|
186
|
+
self._cache.put_metadata(package, version, text)
|
|
187
|
+
return text
|
|
188
|
+
|
|
189
|
+
async def get_sdist_files(
|
|
190
|
+
self, package: str, version: str, sdist_url: str
|
|
191
|
+
) -> tuple[str | None, str | None]:
|
|
192
|
+
"""Return ``(pkg_info, pyproject_toml)`` for an sdist, caching both.
|
|
193
|
+
|
|
194
|
+
Cache miss + offline raises :class:`OfflineError`. Either
|
|
195
|
+
element may be ``None`` if the corresponding file is absent
|
|
196
|
+
from the archive (or the archive cannot be parsed). Both are
|
|
197
|
+
treated as immutable: cached forever, never revalidated.
|
|
198
|
+
"""
|
|
199
|
+
pkg_info = self._cache.get_sdist_pkginfo(package, version)
|
|
200
|
+
pyproject_toml = self._cache.get_sdist_pyproject(package, version)
|
|
201
|
+
if pkg_info is not None:
|
|
202
|
+
return (pkg_info, pyproject_toml)
|
|
203
|
+
|
|
204
|
+
if self._offline:
|
|
205
|
+
msg = f"No cached sdist PKG-INFO for {package}=={version} (offline mode)"
|
|
206
|
+
raise OfflineError(msg)
|
|
207
|
+
|
|
208
|
+
response = await self._transport.get(sdist_url)
|
|
209
|
+
response.raise_for_status()
|
|
210
|
+
pkg_info, pyproject_toml = _extract_sdist_files(response.content)
|
|
211
|
+
|
|
212
|
+
if pkg_info is not None:
|
|
213
|
+
self._cache.put_sdist_pkginfo(package, version, pkg_info)
|
|
214
|
+
if pyproject_toml is not None:
|
|
215
|
+
self._cache.put_sdist_pyproject(package, version, pyproject_toml)
|
|
216
|
+
|
|
217
|
+
return (pkg_info, pyproject_toml)
|
|
218
|
+
|
|
219
|
+
async def get_sdist_archive(
|
|
220
|
+
self, package: str, version: str, sdist_url: str
|
|
221
|
+
) -> bytes:
|
|
222
|
+
"""Return the raw bytes of an sdist archive.
|
|
223
|
+
|
|
224
|
+
Used by the ``BUILD_REMOTE`` path when a real backend invocation
|
|
225
|
+
is required. No on-disk caching is performed: archives are
|
|
226
|
+
large, builds are rare, and the in-memory index already
|
|
227
|
+
deduplicates within a single resolve. Offline mode raises
|
|
228
|
+
:class:`OfflineError` because there is no slot to read from.
|
|
229
|
+
"""
|
|
230
|
+
del package, version # offline check below is the only use
|
|
231
|
+
if self._offline:
|
|
232
|
+
msg = f"sdist archive fetch unavailable in offline mode ({sdist_url})"
|
|
233
|
+
raise OfflineError(msg)
|
|
234
|
+
response = await self._transport.get(sdist_url)
|
|
235
|
+
response.raise_for_status()
|
|
236
|
+
return response.content
|