chtypes 2.0.0.dev6__py3-none-any.whl → 2.0.0.dev8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chtypes/__init__.py +1 -2
- chtypes/__main__.py +115 -10
- chtypes/_abi2/_errors.py +1 -1
- chtypes/_codes.py +23 -0
- chtypes/_decode.py +9 -2
- chtypes/_guard.py +1 -1
- chtypes/_input.py +1 -1
- chtypes/_ocifetch/__init__.py +12 -1
- chtypes/_ocifetch/_dsse.py +12 -2
- chtypes/_ocifetch/_ensure.py +89 -12
- chtypes/_ocifetch/_errors.py +13 -11
- chtypes/_ocifetch/_hold.py +133 -0
- chtypes/_ocifetch/_prune.py +282 -0
- chtypes/_ocifetch/_resolve.py +309 -0
- chtypes/_setup.py +1 -1
- chtypes/errors.py +20 -34
- chtypes/registry.py +132 -15
- {chtypes-2.0.0.dev6.dist-info → chtypes-2.0.0.dev8.dist-info}/METADATA +2 -2
- {chtypes-2.0.0.dev6.dist-info → chtypes-2.0.0.dev8.dist-info}/RECORD +22 -18
- {chtypes-2.0.0.dev6.dist-info → chtypes-2.0.0.dev8.dist-info}/WHEEL +0 -0
- {chtypes-2.0.0.dev6.dist-info → chtypes-2.0.0.dev8.dist-info}/entry_points.txt +0 -0
- {chtypes-2.0.0.dev6.dist-info → chtypes-2.0.0.dev8.dist-info}/licenses/LICENSE +0 -0
chtypes/__init__.py
CHANGED
|
@@ -98,7 +98,7 @@ from .errors import (
|
|
|
98
98
|
UsageError,
|
|
99
99
|
)
|
|
100
100
|
from .library import Block, Filter, Library, Schema, Server, open_unverified
|
|
101
|
-
from .registry import FetchOptions, Registry, Resolved,
|
|
101
|
+
from .registry import FetchOptions, Registry, Resolved, cache_root, search_dirs
|
|
102
102
|
from .results import (
|
|
103
103
|
AtMergeEntry,
|
|
104
104
|
BatchResult,
|
|
@@ -201,7 +201,6 @@ __all__ = [
|
|
|
201
201
|
"Span",
|
|
202
202
|
"Status",
|
|
203
203
|
"Transform",
|
|
204
|
-
"TrustedKey",
|
|
205
204
|
"UnsupportedError",
|
|
206
205
|
"UsageError",
|
|
207
206
|
"Value",
|
chtypes/__main__.py
CHANGED
|
@@ -6,6 +6,10 @@
|
|
|
6
6
|
chtypes verify [--cache <dir>] [--strict] re-verify every installed build
|
|
7
7
|
chtypes list [--cache <dir>] [--offline] [--strict]
|
|
8
8
|
chtypes where [--cache <dir>] [--strict] [--all] the cache root; --all: every search dir
|
|
9
|
+
chtypes resolve <spelling> [--json] [--cache <dir>] [--offline] [--strict]
|
|
10
|
+
the build and manifest each platform resolves to, verified; installs nothing
|
|
11
|
+
chtypes prune [--line <line>] [--keep <n>] [--dry-run] [--cache <dir>] [--strict]
|
|
12
|
+
remove the builds of each line newer ones supersede, keeping the newest n
|
|
9
13
|
|
|
10
14
|
A spelling is `26.8`, `26.8.15` or `26.8.15.10`. Exit statuses come from the
|
|
11
15
|
`errors` table of spec/fetch-v1/constants.json (generated into the fetch layer
|
|
@@ -33,6 +37,7 @@ from collections.abc import Sequence
|
|
|
33
37
|
|
|
34
38
|
from . import __version__ as chtypes_version
|
|
35
39
|
from . import _ocifetch as fetch_layer
|
|
40
|
+
from ._codes import CODE_ARTIFACT_CORRUPT
|
|
36
41
|
from ._ocifetch import _channel
|
|
37
42
|
from ._ocifetch import _constants as C
|
|
38
43
|
from ._ocifetch._ensure import _translate_transport_error, detect_host_platform, missing_notes
|
|
@@ -41,6 +46,7 @@ from ._ocifetch._faults import probe_roots
|
|
|
41
46
|
from ._ocifetch._http import FetchPolicy, RetryPolicy, TransportError, fetch_from_bases
|
|
42
47
|
from ._ocifetch._layout import resolve_cache_root, search_roots
|
|
43
48
|
from ._ocifetch._lock import load_lock
|
|
49
|
+
from ._ocifetch._prune import is_line
|
|
44
50
|
from .errors import ChtypesError
|
|
45
51
|
from .registry import FetchOptions
|
|
46
52
|
|
|
@@ -50,7 +56,7 @@ EXIT_OK = 0
|
|
|
50
56
|
EXIT_USAGE = 2
|
|
51
57
|
# A failure that carries no code from the table (a lock file that cannot be
|
|
52
58
|
# read, an interrupted run): the status the table gives its verification codes.
|
|
53
|
-
_EXIT_FALLBACK = C.ERROR_EXIT_CODES[
|
|
59
|
+
_EXIT_FALLBACK = C.ERROR_EXIT_CODES[CODE_ARTIFACT_CORRUPT]
|
|
54
60
|
_EXIT_INTERRUPTED = 130
|
|
55
61
|
|
|
56
62
|
|
|
@@ -153,7 +159,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
153
159
|
listing = sub.add_parser(
|
|
154
160
|
"list",
|
|
155
161
|
help="what is installed, and what the registry publishes",
|
|
156
|
-
description="What is installed, and the
|
|
162
|
+
description="What is installed, and the lines the registry publishes.",
|
|
157
163
|
)
|
|
158
164
|
cache_option(listing)
|
|
159
165
|
listing.add_argument(
|
|
@@ -173,6 +179,55 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
173
179
|
action="store_true",
|
|
174
180
|
help="every directory searched, in order, the cache root first (one per line)",
|
|
175
181
|
)
|
|
182
|
+
|
|
183
|
+
resolving = sub.add_parser(
|
|
184
|
+
"resolve",
|
|
185
|
+
help="the build and manifest each platform resolves to, verified; installs nothing",
|
|
186
|
+
description=(
|
|
187
|
+
"What a line or version resolves to on every platform the registry offers, "
|
|
188
|
+
"each signature verified, with no layer downloaded (docs/guides/fetch-v1.md section 3)."
|
|
189
|
+
),
|
|
190
|
+
)
|
|
191
|
+
resolving.add_argument(
|
|
192
|
+
"spelling",
|
|
193
|
+
metavar="<spelling>",
|
|
194
|
+
help="26.8, 26.8.15 or 26.8.15.10 (no v prefix, no channel suffix)",
|
|
195
|
+
)
|
|
196
|
+
resolving.add_argument(
|
|
197
|
+
"--json",
|
|
198
|
+
action="store_true",
|
|
199
|
+
help="print one JSON array, one object per platform, instead of one line per platform",
|
|
200
|
+
)
|
|
201
|
+
cache_option(resolving)
|
|
202
|
+
resolving.add_argument(
|
|
203
|
+
"--offline",
|
|
204
|
+
action="store_true",
|
|
205
|
+
help="answer from the cache; no network (or set CHTYPES_OFFLINE=1)",
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
pruning = sub.add_parser(
|
|
209
|
+
"prune",
|
|
210
|
+
help="remove the builds of each line newer ones supersede, keeping the newest n",
|
|
211
|
+
description=(
|
|
212
|
+
"Remove the installed builds of each line that newer installed builds of the same line "
|
|
213
|
+
"and platform supersede, keeping the newest n, and never one a running process holds "
|
|
214
|
+
"(docs/guides/fetch-v1.md section 1)."
|
|
215
|
+
),
|
|
216
|
+
)
|
|
217
|
+
pruning.add_argument(
|
|
218
|
+
"--line", metavar="<line>", help="prune only this line, such as 26.8 (default: every line)"
|
|
219
|
+
)
|
|
220
|
+
pruning.add_argument(
|
|
221
|
+
"--keep",
|
|
222
|
+
metavar="<n>",
|
|
223
|
+
type=int,
|
|
224
|
+
default=1,
|
|
225
|
+
help="how many of each line's newest builds to keep, per platform (at least 1; default 1)",
|
|
226
|
+
)
|
|
227
|
+
pruning.add_argument(
|
|
228
|
+
"--dry-run", action="store_true", help="print what would be removed, and remove nothing"
|
|
229
|
+
)
|
|
230
|
+
cache_option(pruning)
|
|
176
231
|
return parser
|
|
177
232
|
|
|
178
233
|
|
|
@@ -198,13 +253,15 @@ def _options(args: argparse.Namespace, *, platform: str | None = None):
|
|
|
198
253
|
return fetch._to_options(platform)
|
|
199
254
|
|
|
200
255
|
|
|
201
|
-
|
|
256
|
+
# The published lines: exactly two parts, no leading zeros (Go's `lineTag`).
|
|
257
|
+
_LINE_SPELLING = re.compile(r"^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$")
|
|
202
258
|
|
|
203
259
|
|
|
204
260
|
def _published_tags(options) -> list[str]:
|
|
205
|
-
"""The registry's `tags/list`, kept to the
|
|
206
|
-
|
|
207
|
-
else a repository carries are
|
|
261
|
+
"""The registry's `tags/list`, kept to the published lines (two-part tags),
|
|
262
|
+
in numeric order: three- and four-part versions, aliases, the referrers
|
|
263
|
+
fallback tags (`sha256-<hex>`) and anything else a repository carries are
|
|
264
|
+
dropped (docs/guides/fetch-v1.md section 3)."""
|
|
208
265
|
try:
|
|
209
266
|
resp = fetch_from_bases(
|
|
210
267
|
options.resolved_bases(),
|
|
@@ -221,7 +278,7 @@ def _published_tags(options) -> list[str]:
|
|
|
221
278
|
tags = json.loads(resp.body.decode("utf-8")).get("tags") or []
|
|
222
279
|
except (ValueError, AttributeError) as exc:
|
|
223
280
|
raise ArtifactCorruptError(f"chtypes: tags/list was not a tag list ({exc})") from exc
|
|
224
|
-
keep = [t for t in tags if isinstance(t, str) and
|
|
281
|
+
keep = [t for t in tags if isinstance(t, str) and _LINE_SPELLING.fullmatch(t)]
|
|
225
282
|
return sorted(set(keep), key=lambda t: tuple(int(p) for p in t.split(".")))
|
|
226
283
|
|
|
227
284
|
|
|
@@ -250,7 +307,7 @@ def _cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
250
307
|
lock = load_lock(options.lock_path)
|
|
251
308
|
spellings = list(lock.requests)
|
|
252
309
|
else:
|
|
253
|
-
spellings =
|
|
310
|
+
spellings = _published_tags(options)
|
|
254
311
|
if not spellings:
|
|
255
312
|
_say("chtypes: the registry publishes no lines")
|
|
256
313
|
else:
|
|
@@ -297,7 +354,7 @@ def _cmd_verify(args: argparse.Namespace) -> int:
|
|
|
297
354
|
_say(f"chtypes: FAILED {r.version} {r.platform} {r.dir}: {r.detail}")
|
|
298
355
|
if failed:
|
|
299
356
|
_say(f"chtypes: {failed} of {len(results)} installed build(s) FAILED verification")
|
|
300
|
-
return C.ERROR_EXIT_CODES[
|
|
357
|
+
return C.ERROR_EXIT_CODES[CODE_ARTIFACT_CORRUPT]
|
|
301
358
|
return EXIT_OK
|
|
302
359
|
|
|
303
360
|
|
|
@@ -328,7 +385,55 @@ def _cmd_where(args: argparse.Namespace) -> int:
|
|
|
328
385
|
return EXIT_OK
|
|
329
386
|
|
|
330
387
|
|
|
331
|
-
|
|
388
|
+
def _cmd_resolve(args: argparse.Namespace) -> int:
|
|
389
|
+
options = _options(args)
|
|
390
|
+
request = fetch_layer.Request(args.spelling) # a spelling error: usage, before any I/O
|
|
391
|
+
rows, warnings = fetch_layer.resolve(request, options)
|
|
392
|
+
for warning in warnings:
|
|
393
|
+
_say(f"chtypes: warning: {warning}")
|
|
394
|
+
if args.json:
|
|
395
|
+
doc = [
|
|
396
|
+
{"platform": r.platform, "version": r.version, "build": r.build, "manifest": r.manifest}
|
|
397
|
+
for r in rows
|
|
398
|
+
]
|
|
399
|
+
sys.stdout.write(json.dumps(doc, separators=(",", ":")) + "\n")
|
|
400
|
+
else:
|
|
401
|
+
for r in rows:
|
|
402
|
+
sys.stdout.write(f"resolved {r.version} {r.platform} {r.build} {r.manifest}\n")
|
|
403
|
+
sys.stdout.flush()
|
|
404
|
+
return EXIT_OK
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _cmd_prune(args: argparse.Namespace) -> int:
|
|
408
|
+
if args.keep < 1:
|
|
409
|
+
raise ValueError(f"chtypes: --keep is at least 1, not {args.keep}")
|
|
410
|
+
if args.line is not None and not is_line(args.line):
|
|
411
|
+
raise ValueError(f"chtypes: --line takes a two-part line such as 26.8, not {args.line!r}")
|
|
412
|
+
options = _options(args)
|
|
413
|
+
results = fetch_layer.prune(options, line=args.line, keep=args.keep, dry_run=args.dry_run)
|
|
414
|
+
removed = 0
|
|
415
|
+
for r in results:
|
|
416
|
+
if r.in_use:
|
|
417
|
+
word = "in-use"
|
|
418
|
+
else:
|
|
419
|
+
removed += 1
|
|
420
|
+
word = "would-prune" if args.dry_run else "pruned"
|
|
421
|
+
sys.stdout.write(f"{word} {r.version} {r.platform} {r.dir}\n")
|
|
422
|
+
sys.stdout.flush()
|
|
423
|
+
_say_notes(options)
|
|
424
|
+
root = resolve_cache_root(args.cache)
|
|
425
|
+
_say(f"chtypes: {'would prune' if args.dry_run else 'pruned'} {removed} build(s) under {root}")
|
|
426
|
+
return EXIT_OK
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
_COMMANDS = {
|
|
430
|
+
"fetch": _cmd_fetch,
|
|
431
|
+
"verify": _cmd_verify,
|
|
432
|
+
"list": _cmd_list,
|
|
433
|
+
"where": _cmd_where,
|
|
434
|
+
"resolve": _cmd_resolve,
|
|
435
|
+
"prune": _cmd_prune,
|
|
436
|
+
}
|
|
332
437
|
|
|
333
438
|
|
|
334
439
|
def main(argv: Sequence[str] | None = None) -> int:
|
chtypes/_abi2/_errors.py
CHANGED
chtypes/_codes.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Code generated by scripts/fetch-v1/gen-constants.py from
|
|
2
|
+
# spec/fetch-v1/constants.json. DO NOT EDIT.
|
|
3
|
+
# Source sha256: 82afa8d6cba96c10b5945c6f7c1c8360a504cc8ac6231e74017cc6c20b908df6
|
|
4
|
+
"""The shared error codes (docs/guides/fetch-v1.md §8), one per entry of the
|
|
5
|
+
fetch constants' `errors`, in their order. `chtypes.errors` re-exports them
|
|
6
|
+
(docs/reference/bindings-v1.md §4, the codes)."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Final
|
|
11
|
+
|
|
12
|
+
CODE_ARTIFACT_MISSING: Final[str] = "CHTYPES_ARTIFACT_MISSING"
|
|
13
|
+
CODE_ARTIFACT_UNTRUSTED: Final[str] = "CHTYPES_ARTIFACT_UNTRUSTED"
|
|
14
|
+
CODE_ARTIFACT_CORRUPT: Final[str] = "CHTYPES_ARTIFACT_CORRUPT"
|
|
15
|
+
CODE_ARTIFACT_PINNED: Final[str] = "CHTYPES_ARTIFACT_PINNED"
|
|
16
|
+
CODE_ARTIFACT_UNPUBLISHED: Final[str] = "CHTYPES_ARTIFACT_UNPUBLISHED"
|
|
17
|
+
CODE_SOURCE_UNREACHABLE: Final[str] = "CHTYPES_SOURCE_UNREACHABLE"
|
|
18
|
+
CODE_SOURCE_UNAUTHORIZED: Final[str] = "CHTYPES_SOURCE_UNAUTHORIZED"
|
|
19
|
+
CODE_SOURCE_FORBIDDEN: Final[str] = "CHTYPES_SOURCE_FORBIDDEN"
|
|
20
|
+
CODE_SOURCE_INCOMPATIBLE: Final[str] = "CHTYPES_SOURCE_INCOMPATIBLE"
|
|
21
|
+
CODE_ARTIFACT_INCOMPATIBLE: Final[str] = "CHTYPES_ARTIFACT_INCOMPATIBLE"
|
|
22
|
+
CODE_CACHE_UNUSABLE: Final[str] = "CHTYPES_CACHE_UNUSABLE"
|
|
23
|
+
CODE_SOURCE_RETIRED: Final[str] = "CHTYPES_SOURCE_RETIRED"
|
chtypes/_decode.py
CHANGED
|
@@ -39,7 +39,8 @@ from ._abi2._vocab import (
|
|
|
39
39
|
Source,
|
|
40
40
|
Verdict,
|
|
41
41
|
)
|
|
42
|
-
from .errors import ArtifactCorruptError
|
|
42
|
+
from .errors import ArtifactCorruptError
|
|
43
|
+
from .errors import _internal as internal
|
|
43
44
|
from .results import (
|
|
44
45
|
AtMergeEntry,
|
|
45
46
|
BatchResult,
|
|
@@ -427,6 +428,12 @@ def decode_batch(raw: bytes, export: bytes | None) -> BatchResult:
|
|
|
427
428
|
framing = obj.get("framing")
|
|
428
429
|
engine_rows = None if obj.get("engine_rows") is None else _engine_rows(obj, where)
|
|
429
430
|
spans = None if obj.get("row_spans") is None else _spans(obj, "row_spans", where)
|
|
431
|
+
export_declined = _bytes(obj, "export_declined", where)
|
|
432
|
+
if export_declined:
|
|
433
|
+
# A declined export has no payload and no spans, whatever buffer the call
|
|
434
|
+
# returned: the library may hand back a zero-length one, so the document's
|
|
435
|
+
# export_declined decides, never the buffer's length.
|
|
436
|
+
export, spans = None, None
|
|
430
437
|
return BatchResult(
|
|
431
438
|
outcome=Outcome.of(_text(obj, "outcome", where)),
|
|
432
439
|
err_code=_int(obj, "code", where),
|
|
@@ -438,7 +445,7 @@ def decode_batch(raw: bytes, export: bytes | None) -> BatchResult:
|
|
|
438
445
|
engine_rows=engine_rows,
|
|
439
446
|
payload=export,
|
|
440
447
|
spans=spans,
|
|
441
|
-
export_declined=
|
|
448
|
+
export_declined=export_declined,
|
|
442
449
|
rows_passed=_u64(obj, "rows_passed", where),
|
|
443
450
|
rows_cut=_u64(obj, "rows_cut", where),
|
|
444
451
|
partition_count=_opt_u64(obj, "partition_count", where),
|
chtypes/_guard.py
CHANGED
chtypes/_input.py
CHANGED
chtypes/_ocifetch/__init__.py
CHANGED
|
@@ -6,7 +6,8 @@ logic lives in sibling modules. The seam (docs/guides/fetch-v1.md "The seam")
|
|
|
6
6
|
is the only surface anything outside this package may use: `ensure`,
|
|
7
7
|
`resolve_installed`, `list_installed`, `verify_installed` and
|
|
8
8
|
`fetch_signed`, plus the `Request`/`Options`/`Resolved`/
|
|
9
|
-
`VerifyResult` shapes they take and return
|
|
9
|
+
`VerifyResult` shapes they take and return; and beside it the CLI's `resolve`
|
|
10
|
+
and `prune` and the registry's in-use `hold` (§1, "In use"). Every other module here
|
|
10
11
|
(`_http`, `_oci`, `_referrers`, `_dsse`, `_unpack`, `_layout`,
|
|
11
12
|
`_lock`, `_errors`) is a private implementation detail of `_ensure`.
|
|
12
13
|
|
|
@@ -30,15 +31,25 @@ from chtypes._ocifetch._ensure import (
|
|
|
30
31
|
resolve_installed,
|
|
31
32
|
verify_installed,
|
|
32
33
|
)
|
|
34
|
+
from chtypes._ocifetch._hold import hold
|
|
35
|
+
from chtypes._ocifetch._prune import Superseded, prune
|
|
36
|
+
from chtypes._ocifetch._resolve import PlatformOutcome, Resolution, resolve, resolve_each
|
|
33
37
|
|
|
34
38
|
__all__ = [
|
|
35
39
|
"Options",
|
|
40
|
+
"PlatformOutcome",
|
|
36
41
|
"Request",
|
|
42
|
+
"Resolution",
|
|
37
43
|
"Resolved",
|
|
44
|
+
"Superseded",
|
|
38
45
|
"VerifyResult",
|
|
39
46
|
"ensure",
|
|
40
47
|
"fetch_signed",
|
|
48
|
+
"hold",
|
|
41
49
|
"list_installed",
|
|
50
|
+
"prune",
|
|
51
|
+
"resolve",
|
|
52
|
+
"resolve_each",
|
|
42
53
|
"resolve_installed",
|
|
43
54
|
"verify_installed",
|
|
44
55
|
]
|
chtypes/_ocifetch/_dsse.py
CHANGED
|
@@ -21,6 +21,7 @@ import base64
|
|
|
21
21
|
import hashlib
|
|
22
22
|
import json
|
|
23
23
|
import os
|
|
24
|
+
from collections.abc import Sequence
|
|
24
25
|
from dataclasses import dataclass
|
|
25
26
|
|
|
26
27
|
from chtypes import _ed25519
|
|
@@ -86,12 +87,21 @@ def env_trusted_keys() -> tuple[TrustedKey, ...] | None:
|
|
|
86
87
|
items = [part.strip() for part in raw.split(",") if part.strip()]
|
|
87
88
|
if not items:
|
|
88
89
|
return None
|
|
90
|
+
return trusted_keys_from_hex(items, C.ENV_TRUSTED_KEYS_NAME)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def trusted_keys_from_hex(items: Sequence[str], where: str) -> tuple[TrustedKey, ...]:
|
|
94
|
+
"""Raw 32-byte ed25519 public keys, each as 64 hex digits (the
|
|
95
|
+
`CHTYPES_TRUSTED_KEYS` spelling, and the `trusted_keys` option's, as in
|
|
96
|
+
every binding), as trust-list records whose key id is the generated
|
|
97
|
+
constants' algorithm, sha256-first16hex of the raw key. A bad key is a
|
|
98
|
+
ValueError naming `where`."""
|
|
89
99
|
keys = []
|
|
90
100
|
for item in items:
|
|
91
101
|
try:
|
|
92
|
-
public = _hex_to_bytes(item)
|
|
102
|
+
public = _hex_to_bytes(item.strip())
|
|
93
103
|
except ValueError as exc:
|
|
94
|
-
raise ValueError(f"chtypes: {
|
|
104
|
+
raise ValueError(f"chtypes: {where} holds a bad key: {exc}") from None
|
|
95
105
|
keys.append(TrustedKey(keyid=hashlib.sha256(public).hexdigest()[:16], public_key=public))
|
|
96
106
|
return tuple(keys)
|
|
97
107
|
|
chtypes/_ocifetch/_ensure.py
CHANGED
|
@@ -867,7 +867,13 @@ def _ensure_floating(
|
|
|
867
867
|
assert record is not None
|
|
868
868
|
|
|
869
869
|
def _add_entry(doc: dict) -> dict:
|
|
870
|
-
entries =
|
|
870
|
+
entries = list(doc.get("manifests") or [])
|
|
871
|
+
# One entry per manifest digest: a re-install, or a concurrent one that
|
|
872
|
+
# landed first, never writes a second (public issue #487).
|
|
873
|
+
if not any(
|
|
874
|
+
isinstance(e, dict) and e.get("digest") == platform_desc.digest for e in entries
|
|
875
|
+
):
|
|
876
|
+
entries.append({"digest": platform_desc.digest})
|
|
871
877
|
return {**doc, "manifests": entries}
|
|
872
878
|
|
|
873
879
|
try:
|
|
@@ -899,6 +905,82 @@ def _ensure_floating(
|
|
|
899
905
|
return resolved
|
|
900
906
|
|
|
901
907
|
|
|
908
|
+
def _installed_by_pin(
|
|
909
|
+
request: Request, platform_key: str, options: Options, pin: LockPin
|
|
910
|
+
) -> Resolved | None:
|
|
911
|
+
return _lookup_by_pin(request, platform_key, options, pin)[0]
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
def _lookup_by_pin(
|
|
915
|
+
request: Request, platform_key: str, options: Options, pin: LockPin
|
|
916
|
+
) -> tuple[Resolved | None, bool]:
|
|
917
|
+
"""The installed build a lock pin names, with no network: an unpacked
|
|
918
|
+
directory named by the pinned manifest digest, in the cache or a system
|
|
919
|
+
directory, whose `verified.json` is for this platform and records exactly
|
|
920
|
+
the pinned manifest, layer and bundle digests (public issues #414, #487).
|
|
921
|
+
`None` when no installed build matches; the flag is true when a record
|
|
922
|
+
exists under the pinned manifest but disagrees with the pin."""
|
|
923
|
+
roots = search_roots(options.cache_dir, options.system_dirs)
|
|
924
|
+
manifest_hex = parse_digest(pin.manifest)
|
|
925
|
+
mismatch = False
|
|
926
|
+
for root in roots:
|
|
927
|
+
udir = unpacked_dir_for(root, manifest_hex)
|
|
928
|
+
record = read_verified_record(udir)
|
|
929
|
+
if record is None:
|
|
930
|
+
continue
|
|
931
|
+
if (
|
|
932
|
+
record.platform == platform_key
|
|
933
|
+
and record.manifest == pin.manifest
|
|
934
|
+
and record.layer == pin.layer
|
|
935
|
+
and record.bundle == pin.bundle
|
|
936
|
+
):
|
|
937
|
+
return (
|
|
938
|
+
_record_to_resolved(
|
|
939
|
+
udir,
|
|
940
|
+
record,
|
|
941
|
+
request_spelling=request.spelling,
|
|
942
|
+
already_installed=True,
|
|
943
|
+
source=_source_for_dir(udir, roots),
|
|
944
|
+
),
|
|
945
|
+
False,
|
|
946
|
+
)
|
|
947
|
+
mismatch = True
|
|
948
|
+
return None, mismatch
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def _resolve_frozen_offline(
|
|
952
|
+
request: Request, platform_key: str, options: Options, lock: Lock | None
|
|
953
|
+
) -> Resolved:
|
|
954
|
+
"""`--frozen --offline`: verify the installed build against the lock with
|
|
955
|
+
zero network. No lock entry for the request, or an installed record that
|
|
956
|
+
does not match the pin, is `CHTYPES_ARTIFACT_PINNED`; a pinned build that
|
|
957
|
+
is not installed is `CHTYPES_ARTIFACT_MISSING`."""
|
|
958
|
+
if lock is None:
|
|
959
|
+
raise ArtifactPinnedError("chtypes: --frozen requires a lock file; none was given")
|
|
960
|
+
pin = lock.pin_for(request.spelling, platform_key)
|
|
961
|
+
if pin is None:
|
|
962
|
+
raise ArtifactPinnedError(
|
|
963
|
+
f"chtypes: --frozen: no pin for {request.spelling!r} ({platform_key}) in the lock"
|
|
964
|
+
)
|
|
965
|
+
roots = search_roots(options.cache_dir, options.system_dirs)
|
|
966
|
+
warnings = probe_roots(roots, options.resolved_strict())
|
|
967
|
+
installed, mismatch = _lookup_by_pin(request, platform_key, options, pin)
|
|
968
|
+
if installed is None:
|
|
969
|
+
if mismatch:
|
|
970
|
+
raise ArtifactPinnedError(
|
|
971
|
+
f"chtypes: --frozen --offline: the installed build for {request.spelling} "
|
|
972
|
+
f"({platform_key}) does not match the lock's pin {pin.manifest}"
|
|
973
|
+
)
|
|
974
|
+
raise ArtifactMissingError(
|
|
975
|
+
with_notes(
|
|
976
|
+
f"chtypes: --frozen --offline: the build the lock pins ({pin.manifest}) "
|
|
977
|
+
f"is not installed for {request.spelling} ({platform_key})",
|
|
978
|
+
missing_notes(options),
|
|
979
|
+
)
|
|
980
|
+
)
|
|
981
|
+
return replace(installed, warnings=(*warnings, *installed.warnings))
|
|
982
|
+
|
|
983
|
+
|
|
902
984
|
def _ensure_frozen(
|
|
903
985
|
request: Request, platform_key: str, options: Options, lock: Lock | None
|
|
904
986
|
) -> Resolved:
|
|
@@ -916,17 +998,9 @@ def _ensure_frozen(
|
|
|
916
998
|
cache_root_path = roots[0]
|
|
917
999
|
scratch_root = _scratch_root(cache_root_path)
|
|
918
1000
|
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
if existing_record is not None and existing_record.platform == platform_key:
|
|
923
|
-
return _record_to_resolved(
|
|
924
|
-
existing_dir,
|
|
925
|
-
existing_record,
|
|
926
|
-
request_spelling=request.spelling,
|
|
927
|
-
already_installed=True,
|
|
928
|
-
source="cache",
|
|
929
|
-
)
|
|
1001
|
+
installed = _installed_by_pin(request, platform_key, options, pin)
|
|
1002
|
+
if installed is not None:
|
|
1003
|
+
return installed
|
|
930
1004
|
|
|
931
1005
|
manifest_doc, _raw = fetch_manifest_by_digest(bases, pin.manifest, policy=policy, retry=retry)
|
|
932
1006
|
layer_desc = manifest_layer_descriptor(manifest_doc)
|
|
@@ -1049,6 +1123,9 @@ def _ensure(
|
|
|
1049
1123
|
request: Request, platform_key: str, options: Options, lock: Lock | None, roots: Sequence[Path]
|
|
1050
1124
|
) -> Resolved:
|
|
1051
1125
|
try:
|
|
1126
|
+
if options.resolved_offline() and options.frozen:
|
|
1127
|
+
return _resolve_frozen_offline(request, platform_key, options, lock)
|
|
1128
|
+
|
|
1052
1129
|
if options.resolved_offline():
|
|
1053
1130
|
resolved = resolve_installed(request, platform_key, options)
|
|
1054
1131
|
if resolved is None:
|
chtypes/_ocifetch/_errors.py
CHANGED
|
@@ -47,17 +47,19 @@ __all__ = [
|
|
|
47
47
|
"SourceUnreachableError",
|
|
48
48
|
]
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
CODE_ARTIFACT_PINNED
|
|
54
|
-
CODE_ARTIFACT_UNPUBLISHED
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
CODE_SOURCE_FORBIDDEN
|
|
58
|
-
CODE_SOURCE_INCOMPATIBLE
|
|
59
|
-
|
|
60
|
-
|
|
50
|
+
from chtypes._codes import (
|
|
51
|
+
CODE_ARTIFACT_CORRUPT,
|
|
52
|
+
CODE_ARTIFACT_MISSING,
|
|
53
|
+
CODE_ARTIFACT_PINNED,
|
|
54
|
+
CODE_ARTIFACT_UNPUBLISHED,
|
|
55
|
+
CODE_ARTIFACT_UNTRUSTED,
|
|
56
|
+
CODE_CACHE_UNUSABLE,
|
|
57
|
+
CODE_SOURCE_FORBIDDEN,
|
|
58
|
+
CODE_SOURCE_INCOMPATIBLE,
|
|
59
|
+
CODE_SOURCE_RETIRED,
|
|
60
|
+
CODE_SOURCE_UNAUTHORIZED,
|
|
61
|
+
CODE_SOURCE_UNREACHABLE,
|
|
62
|
+
)
|
|
61
63
|
|
|
62
64
|
_CODES_USED_HERE = (
|
|
63
65
|
CODE_ARTIFACT_MISSING,
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""The in-use signal (docs/guides/fetch-v1.md §1, "In use"; public issue #494).
|
|
2
|
+
|
|
3
|
+
A process that is about to load an installed build, or that a registry's
|
|
4
|
+
fetch handed one to, takes a SHARED advisory lock (flock) on the entry's
|
|
5
|
+
`verified.json` and keeps it until the process exits. `chtypes prune` removes
|
|
6
|
+
an entry only after it has taken the EXCLUSIVE lock without waiting, so it
|
|
7
|
+
never removes a build a live process holds. The kernel drops a dead process's
|
|
8
|
+
locks, so a crash never pins a build. flock belongs to the open file
|
|
9
|
+
description on Linux and darwin, so a hold taken by this very process counts
|
|
10
|
+
too, and so does one taken by any binding: all four lock the same file the
|
|
11
|
+
same way.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import errno
|
|
17
|
+
import os
|
|
18
|
+
import threading
|
|
19
|
+
from collections.abc import Callable
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from chtypes._ocifetch import _constants as C
|
|
23
|
+
from chtypes._ocifetch._errors import ArtifactMissingError
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
import fcntl
|
|
27
|
+
except ImportError: # not a platform chtypes serves: nothing is held, prune keeps everything
|
|
28
|
+
fcntl = None # type: ignore[assignment]
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"CLAIM_GONE",
|
|
32
|
+
"CLAIM_IN_USE",
|
|
33
|
+
"CLAIM_OWNED",
|
|
34
|
+
"HELD",
|
|
35
|
+
"UNHELD",
|
|
36
|
+
"VANISHED",
|
|
37
|
+
"claim",
|
|
38
|
+
"hold",
|
|
39
|
+
"removed_while_held",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
# What `hold` found.
|
|
43
|
+
HELD = "held" # this process holds the entry, shared, for its life
|
|
44
|
+
UNHELD = "unheld" # the record cannot be locked here: go on (prune cannot lock it either)
|
|
45
|
+
VANISHED = "vanished" # the entry is gone or replaced: a prune removed it; look again
|
|
46
|
+
|
|
47
|
+
# What `claim` found.
|
|
48
|
+
CLAIM_OWNED = "owned" # locked exclusively: no process holds it
|
|
49
|
+
CLAIM_IN_USE = "in-use" # a process holds it, or it cannot be locked: kept
|
|
50
|
+
CLAIM_GONE = "gone" # gone or replaced since it was listed: not ours to report
|
|
51
|
+
|
|
52
|
+
_GONE = (errno.ENOENT, errno.ENOTDIR)
|
|
53
|
+
|
|
54
|
+
# This process's shared locks: one open record per entry directory, kept until
|
|
55
|
+
# the process exits.
|
|
56
|
+
_holds_lock = threading.Lock()
|
|
57
|
+
_holds: dict[str, int] = {}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _same_file(fd: int, path: str) -> bool:
|
|
61
|
+
try:
|
|
62
|
+
held = os.fstat(fd)
|
|
63
|
+
now = os.stat(path)
|
|
64
|
+
except OSError:
|
|
65
|
+
return False
|
|
66
|
+
return (held.st_dev, held.st_ino) == (now.st_dev, now.st_ino)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def hold(entry_dir: str | os.PathLike[str]) -> str:
|
|
70
|
+
"""Take this process's shared hold on the installed entry `entry_dir`
|
|
71
|
+
(`<root>/unpacked/sha256/<manifest hex>`) and keep it for the life of the
|
|
72
|
+
process. It waits only while a prune holds the entry exclusively, which a
|
|
73
|
+
prune does across one rename; then the entry is gone, and this says so."""
|
|
74
|
+
key = str(entry_dir)
|
|
75
|
+
record = os.path.join(key, C.CACHE_VERIFIED_RECORD)
|
|
76
|
+
with _holds_lock:
|
|
77
|
+
fd = _holds.get(key)
|
|
78
|
+
if fd is not None:
|
|
79
|
+
if _same_file(fd, record):
|
|
80
|
+
return HELD
|
|
81
|
+
# The entry this process held was removed and installed again: the
|
|
82
|
+
# old hold protects nothing now.
|
|
83
|
+
os.close(fd)
|
|
84
|
+
del _holds[key]
|
|
85
|
+
try:
|
|
86
|
+
fd = os.open(record, os.O_RDONLY)
|
|
87
|
+
except OSError as exc:
|
|
88
|
+
return VANISHED if exc.errno in _GONE else UNHELD
|
|
89
|
+
if fcntl is None:
|
|
90
|
+
os.close(fd)
|
|
91
|
+
return UNHELD
|
|
92
|
+
try:
|
|
93
|
+
fcntl.flock(fd, fcntl.LOCK_SH) # retried on EINTR by Python itself (PEP 475)
|
|
94
|
+
except OSError:
|
|
95
|
+
os.close(fd)
|
|
96
|
+
return UNHELD
|
|
97
|
+
if not _same_file(fd, record):
|
|
98
|
+
os.close(fd)
|
|
99
|
+
return VANISHED
|
|
100
|
+
_holds[key] = fd
|
|
101
|
+
return HELD
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def claim(entry_dir: str | os.PathLike[str]) -> tuple[str, Callable[[], None] | None]:
|
|
105
|
+
"""Take the exclusive lock a prune needs on `entry_dir`, without waiting.
|
|
106
|
+
It is owned only when no process holds the entry and the record it locked
|
|
107
|
+
is still the one at the path; the returned release drops the lock."""
|
|
108
|
+
record = os.path.join(str(entry_dir), C.CACHE_VERIFIED_RECORD)
|
|
109
|
+
try:
|
|
110
|
+
fd = os.open(record, os.O_RDONLY)
|
|
111
|
+
except OSError as exc:
|
|
112
|
+
return (CLAIM_GONE if exc.errno in _GONE else CLAIM_IN_USE), None
|
|
113
|
+
if fcntl is None:
|
|
114
|
+
os.close(fd)
|
|
115
|
+
return CLAIM_IN_USE, None
|
|
116
|
+
try:
|
|
117
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
118
|
+
except OSError:
|
|
119
|
+
os.close(fd)
|
|
120
|
+
return CLAIM_IN_USE, None
|
|
121
|
+
if not _same_file(fd, record):
|
|
122
|
+
os.close(fd)
|
|
123
|
+
return CLAIM_GONE, None
|
|
124
|
+
return CLAIM_OWNED, lambda: os.close(fd)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def removed_while_held(request: str, entry_dir: Path) -> ArtifactMissingError:
|
|
128
|
+
"""`CHTYPES_ARTIFACT_MISSING` for a request whose build a concurrent prune
|
|
129
|
+
removed twice, each time between its install and this process's hold."""
|
|
130
|
+
return ArtifactMissingError(
|
|
131
|
+
f"chtypes: {entry_dir} was removed by a concurrent prune before this process "
|
|
132
|
+
f"could hold it; ask for {request} again"
|
|
133
|
+
)
|