pivot 0.3.1__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pivot-0.3.1 → pivot-0.3.3}/PKG-INFO +1 -1
- {pivot-0.3.1 → pivot-0.3.3}/pyproject.toml +1 -1
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/checkout.py +20 -8
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/commit.py +3 -1
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/track.py +22 -7
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/exceptions.py +21 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/commit.py +10 -1
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/core.py +1 -1
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/worker.py +25 -6
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/status.py +2 -1
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/cache.py +215 -33
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/restore.py +14 -8
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/store.py +18 -3
- {pivot-0.3.1 → pivot-0.3.3}/LICENSE +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/README.md +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/__init__.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/ast_utils.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/AGENTS.md +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/CLAUDE.md +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/__init__.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/_run_common.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/check_ignore.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/completion.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/config.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/console.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/dag.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/data.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/decorators.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/doctor.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/errors.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/fingerprint.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/helpers.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/history.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/import_cmd.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/init.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/list.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/metrics.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/params.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/plots.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/remote.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/repro.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/run.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/status.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/targets.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/update.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/verify.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/compose.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/config/__init__.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/config/io.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/config/models.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/dag/__init__.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/dag/render.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/decorators.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/discovery.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/__init__.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/agent_rpc.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/engine.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/graph.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/scheduler.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/sinks.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/sources.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/types.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/watch.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/worker_pool.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/AGENTS.md +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/CLAUDE.md +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/__init__.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/explain.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/fingerprint.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/git.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/ignore.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/import_artifact.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/loaders.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/metrics.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/names.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/outputs.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/parameters.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/path_policy.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/project.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/py.typed +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/registry.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/AGENTS.md +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/__init__.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/config.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/git_archive.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/github.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/storage.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/sync.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/run_history.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/__init__.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/common.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/data.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/metrics.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/params.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/plots.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/skip.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/stage_def.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/AGENTS.md +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/CLAUDE.md +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/__init__.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/artifact_lock.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/lock.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/presentation.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/state.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/track.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/types.py +0 -0
- {pivot-0.3.1 → pivot-0.3.3}/src/pivot/yaml_config.py +0 -0
|
@@ -8,9 +8,10 @@ from typing import TYPE_CHECKING, Literal
|
|
|
8
8
|
|
|
9
9
|
import click
|
|
10
10
|
|
|
11
|
-
from pivot import config, project, registry, types
|
|
11
|
+
from pivot import config, exceptions, project, registry, types
|
|
12
12
|
from pivot.cli import completion
|
|
13
13
|
from pivot.cli import decorators as cli_decorators
|
|
14
|
+
from pivot.cli import errors as cli_errors
|
|
14
15
|
from pivot.cli import helpers as cli_helpers
|
|
15
16
|
from pivot.cli import targets as cli_targets
|
|
16
17
|
from pivot.storage import cache, lock, state, track
|
|
@@ -67,11 +68,13 @@ def _get_stage_output_info() -> dict[str, HashInfo]:
|
|
|
67
68
|
return result
|
|
68
69
|
|
|
69
70
|
|
|
70
|
-
def _on_disk_matches_expected(
|
|
71
|
+
def _on_disk_matches_expected(
|
|
72
|
+
path: pathlib.Path, expected_hash: HashInfo, cache_dir: pathlib.Path
|
|
73
|
+
) -> bool:
|
|
71
74
|
if is_dir_hash(expected_hash):
|
|
72
75
|
if not path.is_dir():
|
|
73
76
|
return False
|
|
74
|
-
actual_hash, _ = cache.hash_directory(path)
|
|
77
|
+
actual_hash, _ = cache.hash_directory(path, cache_dir=cache_dir)
|
|
75
78
|
else:
|
|
76
79
|
if not path.is_file():
|
|
77
80
|
return False
|
|
@@ -119,7 +122,7 @@ def _restore_path_locked(
|
|
|
119
122
|
if path.exists():
|
|
120
123
|
match behavior:
|
|
121
124
|
case CheckoutBehavior.ERROR:
|
|
122
|
-
if _on_disk_matches_expected(path, output_hash):
|
|
125
|
+
if _on_disk_matches_expected(path, output_hash, cache_dir):
|
|
123
126
|
return ("skipped", path.name)
|
|
124
127
|
raise click.ClickException(
|
|
125
128
|
f"'{path.name}' already exists with different content. "
|
|
@@ -150,7 +153,12 @@ def _restore_path_locked(
|
|
|
150
153
|
return ("missing", path.name)
|
|
151
154
|
return ("restored" if outcome["restored"] else "skipped", path.name)
|
|
152
155
|
case CheckoutBehavior.FORCE:
|
|
153
|
-
cache.
|
|
156
|
+
missing = cache.missing_cache_objects(output_hash, cache_dir, checkout_modes)
|
|
157
|
+
if missing:
|
|
158
|
+
raise click.ClickException(
|
|
159
|
+
f"Cannot overwrite '{path.name}': cache object missing: {', '.join(missing)}. "
|
|
160
|
+
+ "The workspace output was not changed."
|
|
161
|
+
)
|
|
154
162
|
case _: # pyright: ignore[reportUnnecessaryComparison] - defensive for future enum values
|
|
155
163
|
raise ValueError(f"Unhandled checkout behavior: {behavior}") # pyright: ignore[reportUnreachable]
|
|
156
164
|
|
|
@@ -222,9 +230,13 @@ async def _checkout_files_async(
|
|
|
222
230
|
for abs_path_str, output_hash in files.items():
|
|
223
231
|
_ = tg.create_task(restore_one(abs_path_str, output_hash))
|
|
224
232
|
except* Exception as eg:
|
|
225
|
-
# Convert unexpected exceptions to friendly error message
|
|
226
|
-
|
|
227
|
-
|
|
233
|
+
# Convert unexpected exceptions to a friendly error message, rendering a PivotError's
|
|
234
|
+
# actionable suggestion the same way the top-level CLI error handler does (cli/errors.py).
|
|
235
|
+
messages = [
|
|
236
|
+
cli_errors.format_pivot_error(e) if isinstance(e, exceptions.PivotError) else str(e)
|
|
237
|
+
for e in eg.exceptions
|
|
238
|
+
]
|
|
239
|
+
raise click.ClickException("\n".join(messages)) from None
|
|
228
240
|
|
|
229
241
|
# Re-raise immediate failures (all of them, aggregated)
|
|
230
242
|
if immediate_errors:
|
|
@@ -20,7 +20,9 @@ def commit_command(ctx: click.Context, stages: tuple[str, ...]) -> None:
|
|
|
20
20
|
quiet = cli_ctx["quiet"]
|
|
21
21
|
|
|
22
22
|
stage_names = cli_helpers.resolve_stage_names(list(stages) if stages else None)
|
|
23
|
-
committed, failed = commit.commit_stages(
|
|
23
|
+
committed, failed = commit.commit_stages(
|
|
24
|
+
cli_helpers.get_pipeline(), stage_names, warn_left_out_symlinks=not quiet
|
|
25
|
+
)
|
|
24
26
|
|
|
25
27
|
if not quiet:
|
|
26
28
|
if committed:
|
|
@@ -86,9 +86,14 @@ def _track_single_path(
|
|
|
86
86
|
stage_outputs_resolved: dict[str, pathlib.Path],
|
|
87
87
|
existing_tracked: dict[str, track_mod.PvtData],
|
|
88
88
|
force: bool,
|
|
89
|
+
*,
|
|
90
|
+
warn_left_out_symlinks: bool,
|
|
89
91
|
) -> TrackResult:
|
|
90
92
|
"""Track a single file or directory.
|
|
91
93
|
|
|
94
|
+
``warn_left_out_symlinks`` logs the warning naming the symlinks a tracked directory leaves
|
|
95
|
+
out; the caller turns it off under ``--quiet``, like this command's other warning.
|
|
96
|
+
|
|
92
97
|
Returns:
|
|
93
98
|
TrackResult with optional warning and the tracked path for the caller to handle output.
|
|
94
99
|
"""
|
|
@@ -149,16 +154,20 @@ def _track_single_path(
|
|
|
149
154
|
|
|
150
155
|
# Hash and cache
|
|
151
156
|
if path.is_dir():
|
|
152
|
-
tree_hash, manifest = cache.hash_directory(
|
|
157
|
+
tree_hash, manifest = cache.hash_directory(
|
|
158
|
+
path, cache_dir=cache_dir, warn_left_out_symlinks=warn_left_out_symlinks
|
|
159
|
+
)
|
|
153
160
|
total_size = sum(e["size"] for e in manifest)
|
|
154
161
|
num_files = len(manifest)
|
|
155
162
|
|
|
156
|
-
#
|
|
163
|
+
# copy_to_cache is idempotent and already checks existence itself; letting it see
|
|
164
|
+
# every entry (not just ones missing from the cache) is what lets it republish an
|
|
165
|
+
# object `pivot track --force` finds cached non-executable by an older pivot or a
|
|
166
|
+
# cold `pivot pull`, when this entry's bytes should be executable.
|
|
157
167
|
for entry in manifest:
|
|
158
168
|
file_path = path / entry["relpath"]
|
|
159
169
|
file_cache_path = cache.get_cache_path(cache_dir, entry["hash"])
|
|
160
|
-
|
|
161
|
-
cache.copy_to_cache(file_path, file_cache_path)
|
|
170
|
+
cache.copy_to_cache(file_path, file_cache_path, executable=entry["isexec"])
|
|
162
171
|
|
|
163
172
|
pvt_data: track_mod.PvtData = {
|
|
164
173
|
"path": path.name,
|
|
@@ -171,8 +180,7 @@ def _track_single_path(
|
|
|
171
180
|
file_hash, _ = cache.hash_file(path)
|
|
172
181
|
file_size = path.stat().st_size
|
|
173
182
|
file_cache_path = cache.get_cache_path(cache_dir, file_hash)
|
|
174
|
-
|
|
175
|
-
cache.copy_to_cache(path, file_cache_path)
|
|
183
|
+
cache.copy_to_cache(path, file_cache_path)
|
|
176
184
|
|
|
177
185
|
pvt_data = {
|
|
178
186
|
"path": path.name,
|
|
@@ -212,7 +220,14 @@ def track(ctx: click.Context, paths: tuple[str, ...], force: bool) -> None:
|
|
|
212
220
|
existing_tracked = track_mod.discover_pvt_files(project_root)
|
|
213
221
|
|
|
214
222
|
for path_str in paths:
|
|
215
|
-
result = _track_single_path(
|
|
223
|
+
result = _track_single_path(
|
|
224
|
+
path_str,
|
|
225
|
+
cache_dir,
|
|
226
|
+
stage_outputs,
|
|
227
|
+
existing_tracked,
|
|
228
|
+
force,
|
|
229
|
+
warn_left_out_symlinks=not quiet,
|
|
230
|
+
)
|
|
216
231
|
if not quiet:
|
|
217
232
|
if result["warning"]:
|
|
218
233
|
click.echo(result["warning"], err=True)
|
|
@@ -214,6 +214,27 @@ class UncachedIncrementalOutputError(CacheError):
|
|
|
214
214
|
"""Raised when an incremental output exists without a cache entry."""
|
|
215
215
|
|
|
216
216
|
|
|
217
|
+
class CacheModeCollisionError(CacheError):
|
|
218
|
+
"""A HARDLINK/SYMLINK checkout shares the cache object's own permissions and found it
|
|
219
|
+
already recorded with the wrong executable bit, with no COPY fallback available.
|
|
220
|
+
|
|
221
|
+
This happens when two files with identical content disagree on their executable bit (the
|
|
222
|
+
cache key is content-only) or when a cache object was populated by a path that does not
|
|
223
|
+
know a file's intended executable bit (e.g. `pivot pull`, which downloads by hash alone).
|
|
224
|
+
"""
|
|
225
|
+
|
|
226
|
+
@override
|
|
227
|
+
def get_suggestion(self) -> str:
|
|
228
|
+
return (
|
|
229
|
+
"Add CheckoutMode.COPY to checkout_modes (cache: checkout_mode: [..., copy] in "
|
|
230
|
+
".pivot/config.yaml) so checkout can fall back to a private, correctly-permissioned "
|
|
231
|
+
"copy instead of sharing the cache object's own permissions; or, if you have the "
|
|
232
|
+
"source bytes in hand, re-running what produced them (`pivot repro --force "
|
|
233
|
+
"<stage>`, `pivot track --force <path>`) republishes the cache object with the "
|
|
234
|
+
"right bit"
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
217
238
|
class ParamsError(PivotError):
|
|
218
239
|
"""Raised when parameter validation or loading fails."""
|
|
219
240
|
|
|
@@ -24,6 +24,8 @@ def commit_stages(
|
|
|
24
24
|
pipeline: registry.PipelineLike,
|
|
25
25
|
stage_names: list[str] | None = None,
|
|
26
26
|
force: bool = False,
|
|
27
|
+
*,
|
|
28
|
+
warn_left_out_symlinks: bool = True,
|
|
27
29
|
) -> tuple[list[str], list[str]]:
|
|
28
30
|
"""Commit current workspace state for stages.
|
|
29
31
|
|
|
@@ -37,6 +39,8 @@ def commit_stages(
|
|
|
37
39
|
pipeline: The pipeline whose stages are committed.
|
|
38
40
|
stage_names: Specific stages to commit. None means all stale stages.
|
|
39
41
|
force: If True, commit even if lock file is unchanged.
|
|
42
|
+
warn_left_out_symlinks: Log the warning naming symlinks a directory output leaves
|
|
43
|
+
out. A quiet command turns it off.
|
|
40
44
|
|
|
41
45
|
Returns:
|
|
42
46
|
Tuple of (committed, failed) stage name lists.
|
|
@@ -150,7 +154,12 @@ def commit_stages(
|
|
|
150
154
|
identity_key = types.identity_key(out.identity)
|
|
151
155
|
try:
|
|
152
156
|
out_path = store.resolve_display_path(out)
|
|
153
|
-
hash_info = store.commit(
|
|
157
|
+
hash_info = store.commit(
|
|
158
|
+
out,
|
|
159
|
+
out_path,
|
|
160
|
+
state_dir=default_state_dir,
|
|
161
|
+
warn_left_out_symlinks=warn_left_out_symlinks,
|
|
162
|
+
)
|
|
154
163
|
file_hash_entries.extend(worker.output_file_hash_entries(out_path, hash_info))
|
|
155
164
|
output_hashes_by_key[identity_key] = hash_info
|
|
156
165
|
output_hashes_by_identity[out.identity] = hash_info
|
|
@@ -409,7 +409,7 @@ def verify_tracked_files(project_root: pathlib.Path, checkout_missing: bool = Fa
|
|
|
409
409
|
if path.is_file():
|
|
410
410
|
current_hash, _ = cache.hash_file(path, state_db)
|
|
411
411
|
elif path.is_dir():
|
|
412
|
-
current_hash, _ = cache.hash_directory(path, state_db)
|
|
412
|
+
current_hash, _ = cache.hash_directory(path, state_db, cache_dir=cache_dir)
|
|
413
413
|
else:
|
|
414
414
|
# Path doesn't exist
|
|
415
415
|
raise FileNotFoundError(data_path)
|
|
@@ -411,7 +411,9 @@ def execute_stage(
|
|
|
411
411
|
project_root,
|
|
412
412
|
)
|
|
413
413
|
if no_commit:
|
|
414
|
-
output_hashes = _hash_output_paths(
|
|
414
|
+
output_hashes = _hash_output_paths(
|
|
415
|
+
output_paths, cache_dir=_store_cache_dir(store)
|
|
416
|
+
)
|
|
415
417
|
else:
|
|
416
418
|
output_hashes = _commit_outputs_with_store(
|
|
417
419
|
output_paths, store, project_state_dir
|
|
@@ -556,7 +558,7 @@ def restore_outputs_to_hashes(
|
|
|
556
558
|
if expected_hash is not None:
|
|
557
559
|
output_path = store.resolve_display_path(ref)
|
|
558
560
|
try:
|
|
559
|
-
actual_hash = hash_output(output_path, state_db)
|
|
561
|
+
actual_hash = hash_output(output_path, state_db, cache_dir=store.cache_dir)
|
|
560
562
|
except OSError:
|
|
561
563
|
actual_hash = None
|
|
562
564
|
if actual_hash == expected_hash:
|
|
@@ -613,10 +615,23 @@ def _check_skip_or_run(
|
|
|
613
615
|
return "unchanged", "", input_hash, decision["run_cache_allowed"]
|
|
614
616
|
|
|
615
617
|
|
|
616
|
-
def
|
|
617
|
-
"""
|
|
618
|
+
def _store_cache_dir(store: store_mod.Store) -> pathlib.Path | None:
|
|
619
|
+
"""The cache object root whose symlinks count as directory content for ``store``."""
|
|
620
|
+
return store.cache_dir if isinstance(store, store_mod.WorkspaceStore) else None
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def hash_output(
|
|
624
|
+
path: pathlib.Path,
|
|
625
|
+
state_db: state.StateDB | None = None,
|
|
626
|
+
*,
|
|
627
|
+
cache_dir: pathlib.Path | None = None,
|
|
628
|
+
) -> HashInfo:
|
|
629
|
+
"""Compute output hash without saving to cache.
|
|
630
|
+
|
|
631
|
+
``cache_dir`` is the cache object root whose symlinks count as directory content.
|
|
632
|
+
"""
|
|
618
633
|
if path.is_dir():
|
|
619
|
-
tree_hash, manifest = cache.hash_directory(path, state_db)
|
|
634
|
+
tree_hash, manifest = cache.hash_directory(path, state_db, cache_dir=cache_dir)
|
|
620
635
|
return DirHash(hash=tree_hash, manifest=manifest)
|
|
621
636
|
file_hash, _ = cache.hash_file(path, state_db)
|
|
622
637
|
return FileHash(hash=file_hash)
|
|
@@ -1040,13 +1055,17 @@ def _run_stage_function_with_store(
|
|
|
1040
1055
|
def _hash_output_paths(
|
|
1041
1056
|
outputs_by_ref: dict[types.ArtifactRef, pathlib.Path],
|
|
1042
1057
|
state_db: state.StateDB | None = None,
|
|
1058
|
+
*,
|
|
1059
|
+
cache_dir: pathlib.Path | None = None,
|
|
1043
1060
|
) -> dict[str, HashInfo]:
|
|
1044
1061
|
output_hashes = dict[str, HashInfo]()
|
|
1045
1062
|
for ref, path in outputs_by_ref.items():
|
|
1046
1063
|
if not path.exists():
|
|
1047
1064
|
raise exceptions.OutputMissingError(f"Stage did not produce output: {path}")
|
|
1048
1065
|
if path.is_dir():
|
|
1049
|
-
tree_hash, manifest = cache.hash_directory(
|
|
1066
|
+
tree_hash, manifest = cache.hash_directory(
|
|
1067
|
+
path, state_db, cache_dir=cache_dir, warn_left_out_symlinks=True
|
|
1068
|
+
)
|
|
1050
1069
|
output_hashes[types.identity_key(ref.identity)] = DirHash(
|
|
1051
1070
|
hash=tree_hash, manifest=manifest
|
|
1052
1071
|
)
|
|
@@ -348,6 +348,7 @@ def get_tracked_files_status(
|
|
|
348
348
|
results = list[TrackedFileInfo]()
|
|
349
349
|
|
|
350
350
|
# Use state_db for hash caching (mtime-based)
|
|
351
|
+
cache_dir = config.get_cache_dir() / "files"
|
|
351
352
|
with state_mod.StateDB(config.get_state_dir()) as state_db:
|
|
352
353
|
for i, (abs_path_str, track_data) in enumerate(sorted(tracked.items()), 1):
|
|
353
354
|
path = pathlib.Path(abs_path_str)
|
|
@@ -355,7 +356,7 @@ def get_tracked_files_status(
|
|
|
355
356
|
|
|
356
357
|
try:
|
|
357
358
|
if path.is_dir():
|
|
358
|
-
current_hash, _ = cache.hash_directory(path, state_db)
|
|
359
|
+
current_hash, _ = cache.hash_directory(path, state_db, cache_dir=cache_dir)
|
|
359
360
|
else:
|
|
360
361
|
current_hash, _ = cache.hash_file(path, state_db)
|
|
361
362
|
except FileNotFoundError:
|
|
@@ -45,8 +45,8 @@ def _restore_target_token(target_path: pathlib.Path) -> str:
|
|
|
45
45
|
return hashlib.sha256(str(logical_path).encode()).hexdigest()[:32]
|
|
46
46
|
|
|
47
47
|
|
|
48
|
-
def
|
|
49
|
-
"""Return the deterministic staging path for a file
|
|
48
|
+
def restore_staging_path(path: pathlib.Path, token: str | None = None) -> pathlib.Path:
|
|
49
|
+
"""Return the deterministic staging path for a file restore."""
|
|
50
50
|
return path.parent / f"{_RESTORE_TEMP_PREFIX}{token or _restore_target_token(path)}.file"
|
|
51
51
|
|
|
52
52
|
|
|
@@ -92,26 +92,37 @@ def atomic_write_file(
|
|
|
92
92
|
dest: pathlib.Path,
|
|
93
93
|
write_fn: Callable[[int], None],
|
|
94
94
|
mode: int = 0o644,
|
|
95
|
+
*,
|
|
96
|
+
staging_path: pathlib.Path | None = None,
|
|
95
97
|
) -> None:
|
|
96
|
-
"""Atomically write to dest using
|
|
98
|
+
"""Atomically write to dest using a temporary file and replacement.
|
|
99
|
+
|
|
100
|
+
A caller holding a restore lock can supply its target-tagged staging path so a
|
|
101
|
+
later restore of the same target cleans up a process killed before replacement.
|
|
97
102
|
|
|
98
103
|
Args:
|
|
99
104
|
dest: Target file path.
|
|
100
105
|
write_fn: Function that receives the file descriptor and writes content.
|
|
101
106
|
MUST close fd (typically via os.fdopen which takes ownership).
|
|
102
107
|
mode: File permissions (default 0o644).
|
|
108
|
+
staging_path: Optional deterministic temporary path beside ``dest``.
|
|
103
109
|
"""
|
|
104
110
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
105
|
-
|
|
106
|
-
|
|
111
|
+
if staging_path is None:
|
|
112
|
+
fd, tmp_path = tempfile.mkstemp(dir=dest.parent, suffix=".tmp")
|
|
113
|
+
tmp = pathlib.Path(tmp_path)
|
|
114
|
+
else:
|
|
115
|
+
tmp = staging_path
|
|
116
|
+
_clear_path(tmp)
|
|
117
|
+
fd = os.open(tmp, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
|
107
118
|
fd_closed = False
|
|
108
119
|
try:
|
|
109
120
|
write_fn(fd)
|
|
110
121
|
fd_closed = True # write_fn took ownership via os.fdopen
|
|
111
|
-
os.chmod(
|
|
122
|
+
os.chmod(tmp, mode)
|
|
112
123
|
tmp.replace(dest)
|
|
113
124
|
except BaseException:
|
|
114
|
-
tmp
|
|
125
|
+
_clear_path(tmp)
|
|
115
126
|
raise
|
|
116
127
|
finally:
|
|
117
128
|
# Only close fd if write_fn didn't (e.g., exception before os.fdopen)
|
|
@@ -159,39 +170,83 @@ def hash_file(
|
|
|
159
170
|
return file_hash, file_stat
|
|
160
171
|
|
|
161
172
|
|
|
162
|
-
def
|
|
173
|
+
def _cache_object_key(path: pathlib.Path, cache_dir: pathlib.Path | None) -> str | None:
|
|
174
|
+
"""Return the cache key a symlink resolves to, if its target is an object in ``cache_dir``.
|
|
175
|
+
|
|
176
|
+
Cache objects live at ``<cache_dir>/XX/YYYY...`` named by their content hash; a file-by-file
|
|
177
|
+
symlink checkout restores each directory member as a link to one. A link anywhere else,
|
|
178
|
+
even to a hash-named file, is a user symlink.
|
|
179
|
+
"""
|
|
180
|
+
if cache_dir is None:
|
|
181
|
+
return None
|
|
182
|
+
key = _get_symlink_cache_hash(path, cache_dir)
|
|
183
|
+
if key is None or not is_valid_hash(key) or not path.is_file():
|
|
184
|
+
return None
|
|
185
|
+
return key
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _scandir_recursive(
|
|
189
|
+
path: pathlib.Path, cache_dir: pathlib.Path | None, left_out: list[pathlib.Path]
|
|
190
|
+
) -> Generator[os.DirEntry[str]]:
|
|
163
191
|
"""Yield every file under `path` recursively, using os.scandir() for efficiency.
|
|
164
192
|
|
|
165
193
|
A declared directory artifact is hashed and cached whole: no name is ignored, so a change
|
|
166
194
|
to any file in a directory input reruns its consumers, and every file in a directory output
|
|
167
195
|
survives cache and restore. Ignore rules belong to code fingerprinting and watch mode.
|
|
196
|
+
A symlink to an object in ``cache_dir`` (left by a symlink checkout) counts as the file it
|
|
197
|
+
points to; any other symlink is not stored and is appended to ``left_out``.
|
|
168
198
|
DirEntry objects cache stat results, avoiding redundant syscalls.
|
|
169
|
-
Symlinks are skipped to prevent loops.
|
|
170
199
|
PermissionError propagates to caller - partial hashes would be incorrect.
|
|
171
200
|
|
|
172
201
|
Args:
|
|
173
202
|
path: Directory to scan
|
|
203
|
+
cache_dir: Cache object root whose links count as content, or None for no cache
|
|
204
|
+
left_out: Collects the symlinks that are not part of the artifact
|
|
174
205
|
"""
|
|
175
206
|
with os.scandir(path) as entries:
|
|
176
207
|
for entry in entries:
|
|
177
208
|
if entry.is_symlink():
|
|
178
|
-
|
|
179
|
-
|
|
209
|
+
if _cache_object_key(pathlib.Path(entry.path), cache_dir) is not None:
|
|
210
|
+
yield entry
|
|
211
|
+
else:
|
|
212
|
+
left_out.append(pathlib.Path(entry.path))
|
|
213
|
+
elif entry.is_file():
|
|
180
214
|
yield entry
|
|
181
215
|
elif entry.is_dir():
|
|
182
|
-
yield from _scandir_recursive(pathlib.Path(entry.path))
|
|
216
|
+
yield from _scandir_recursive(pathlib.Path(entry.path), cache_dir, left_out)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _warn_left_out_symlinks(path: pathlib.Path, left_out: list[pathlib.Path]) -> None:
|
|
220
|
+
"""Log one warning naming every symlink a directory artifact does not store."""
|
|
221
|
+
absolute = path.absolute()
|
|
222
|
+
cwd = pathlib.Path.cwd()
|
|
223
|
+
shown = absolute.relative_to(cwd) if absolute.is_relative_to(cwd) else path
|
|
224
|
+
names = ", ".join(sorted(str(link.relative_to(path)) for link in left_out))
|
|
225
|
+
one = len(left_out) == 1
|
|
226
|
+
logger.warning(
|
|
227
|
+
"Directory artifact %s leaves out %s %s (a symlink inside a directory artifact is not "
|
|
228
|
+
+ "stored); replace %s with a real file or directory",
|
|
229
|
+
shown,
|
|
230
|
+
"symlink" if one else "symlinks",
|
|
231
|
+
names,
|
|
232
|
+
"it" if one else "each",
|
|
233
|
+
)
|
|
183
234
|
|
|
184
235
|
|
|
185
236
|
def hash_directory(
|
|
186
237
|
path: pathlib.Path,
|
|
187
238
|
state_db: state_mod.StateDB | None = None,
|
|
239
|
+
*,
|
|
240
|
+
cache_dir: pathlib.Path | None = None,
|
|
241
|
+
warn_left_out_symlinks: bool = False,
|
|
188
242
|
) -> tuple[str, list[DirManifestEntry]]:
|
|
189
243
|
"""Compute tree hash of directory, returning hash and manifest.
|
|
190
244
|
|
|
191
245
|
Symlink handling:
|
|
192
|
-
-
|
|
246
|
+
- A symlink to an object in ``cache_dir`` is hashed by its target's content under its own
|
|
247
|
+
relpath; the content must match the object's name
|
|
248
|
+
- Any other symlink inside the directory is left out of the manifest
|
|
193
249
|
- Base path may be symlinked (resolved for consistency)
|
|
194
|
-
- Content-based hashing only (symlink metadata excluded from fingerprints)
|
|
195
250
|
|
|
196
251
|
Note: For portability, paths are stored as normalized (symlinks preserved)
|
|
197
252
|
in lock files, but resolved here for consistent hashing.
|
|
@@ -199,20 +254,30 @@ def hash_directory(
|
|
|
199
254
|
Args:
|
|
200
255
|
path: Directory to hash
|
|
201
256
|
state_db: Optional StateDB for caching file hashes
|
|
257
|
+
cache_dir: The configured cache's object root (``<cache.dir>/files``). Pass it
|
|
258
|
+
wherever one exists, so recording and re-hashing agree on symlink checkouts;
|
|
259
|
+
None means no cache, and then no symlink is content.
|
|
260
|
+
warn_left_out_symlinks: Log one warning naming the symlinks left out. Set where a
|
|
261
|
+
manifest is recorded (a stage output, a tracked directory), not on every re-hash.
|
|
202
262
|
"""
|
|
203
263
|
_t = metrics.start()
|
|
204
264
|
manifest = list[DirManifestEntry]()
|
|
265
|
+
left_out = list[pathlib.Path]()
|
|
205
266
|
resolved_base = path.resolve()
|
|
206
267
|
|
|
207
|
-
for entry in sorted(_scandir_recursive(path), key=lambda e: e.path):
|
|
268
|
+
for entry in sorted(_scandir_recursive(path, cache_dir, left_out), key=lambda e: e.path):
|
|
208
269
|
file_path = pathlib.Path(entry.path)
|
|
209
|
-
# Verify file is still within the directory (paranoid check)
|
|
210
|
-
|
|
270
|
+
# Verify a regular file is still within the directory (paranoid check). A cache-object
|
|
271
|
+
# link resolves into the cache by design.
|
|
272
|
+
if not entry.is_symlink() and not file_path.resolve().is_relative_to(resolved_base):
|
|
211
273
|
continue
|
|
212
274
|
try:
|
|
213
275
|
rel = file_path.relative_to(path)
|
|
214
276
|
file_stat = entry.stat(follow_symlinks=True)
|
|
215
277
|
file_hash, _ = hash_file(file_path, state_db)
|
|
278
|
+
if entry.is_symlink() and file_hash != _cache_object_key(file_path, cache_dir):
|
|
279
|
+
left_out.append(file_path)
|
|
280
|
+
continue
|
|
216
281
|
manifest_entry: DirManifestEntry = {
|
|
217
282
|
"relpath": str(rel),
|
|
218
283
|
"hash": file_hash,
|
|
@@ -223,6 +288,9 @@ def hash_directory(
|
|
|
223
288
|
except FileNotFoundError:
|
|
224
289
|
continue # File deleted between scan and hash
|
|
225
290
|
|
|
291
|
+
if left_out and warn_left_out_symlinks:
|
|
292
|
+
_warn_left_out_symlinks(path, left_out)
|
|
293
|
+
|
|
226
294
|
manifest_json = json.dumps(manifest, sort_keys=True, separators=(",", ":"))
|
|
227
295
|
tree_hash = xxhash.xxh64(manifest_json.encode()).hexdigest()
|
|
228
296
|
|
|
@@ -379,9 +447,21 @@ def _get_symlink_cache_hash(path: pathlib.Path, cache_dir: pathlib.Path) -> str
|
|
|
379
447
|
return None # Includes ELOOP for circular symlinks
|
|
380
448
|
|
|
381
449
|
|
|
382
|
-
def copy_to_cache(src: pathlib.Path, cache_path: pathlib.Path) -> None:
|
|
383
|
-
"""Atomically copy file to cache with read-only permissions.
|
|
384
|
-
|
|
450
|
+
def copy_to_cache(src: pathlib.Path, cache_path: pathlib.Path, *, executable: bool = False) -> None:
|
|
451
|
+
"""Atomically copy file to cache with read-only permissions.
|
|
452
|
+
|
|
453
|
+
``executable`` records the source's executable bit on the cache object itself (0o555
|
|
454
|
+
instead of 0o444): a HARDLINK or SYMLINK checkout has no mode of its own, so it reads
|
|
455
|
+
straight through to whatever mode the cache object carries, and only COPY checkout
|
|
456
|
+
re-chmods afterwards (see ``_stage_checkout``).
|
|
457
|
+
|
|
458
|
+
An existing object missing the executable bit is republished (a fresh temp file atomically
|
|
459
|
+
replacing the old one, never a chmod in place: existing hardlinks keep the old inode and
|
|
460
|
+
its mode) when ``executable`` is True, so an object an older pivot or a remote pull cached
|
|
461
|
+
non-executable can still be repaired by a caller who has the right bytes in hand. An
|
|
462
|
+
object already executable, or a non-executable request, is left alone.
|
|
463
|
+
"""
|
|
464
|
+
if cache_path.exists() and (not executable or _cache_object_isexec(cache_path)):
|
|
385
465
|
return
|
|
386
466
|
|
|
387
467
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -390,7 +470,7 @@ def copy_to_cache(src: pathlib.Path, cache_path: pathlib.Path) -> None:
|
|
|
390
470
|
try:
|
|
391
471
|
os.close(fd)
|
|
392
472
|
shutil.copy2(src, tmp_path)
|
|
393
|
-
os.chmod(tmp_path, 0o444)
|
|
473
|
+
os.chmod(tmp_path, 0o555 if executable else 0o444)
|
|
394
474
|
tmp.replace(cache_path)
|
|
395
475
|
except BaseException:
|
|
396
476
|
tmp.unlink(missing_ok=True)
|
|
@@ -405,7 +485,7 @@ def _stage_checkout(
|
|
|
405
485
|
cache_path: pathlib.Path,
|
|
406
486
|
checkout_mode: CheckoutMode,
|
|
407
487
|
*,
|
|
408
|
-
executable: bool,
|
|
488
|
+
executable: bool | None,
|
|
409
489
|
) -> None:
|
|
410
490
|
"""Create the link or copy of cache_path at tmp_path."""
|
|
411
491
|
if checkout_mode == CheckoutMode.SYMLINK:
|
|
@@ -422,7 +502,7 @@ def _checkout_from_cache(
|
|
|
422
502
|
cache_path: pathlib.Path,
|
|
423
503
|
checkout_mode: CheckoutMode,
|
|
424
504
|
*,
|
|
425
|
-
executable: bool =
|
|
505
|
+
executable: bool | None = None,
|
|
426
506
|
staging_path: pathlib.Path | None = None,
|
|
427
507
|
) -> None:
|
|
428
508
|
"""Create link or copy at path atomically, so readers never see a partial file.
|
|
@@ -465,33 +545,105 @@ def _checkout_from_cache(
|
|
|
465
545
|
tmp_path.unlink(missing_ok=True)
|
|
466
546
|
|
|
467
547
|
|
|
548
|
+
def _cache_object_isexec(cache_path: pathlib.Path) -> bool | None:
|
|
549
|
+
"""Whether the cache object at cache_path is stored executable, or None if absent.
|
|
550
|
+
|
|
551
|
+
HARDLINK and SYMLINK checkout have no mode of their own: the workspace file shares the
|
|
552
|
+
cache object's permissions directly, so they can only ever produce whatever executable
|
|
553
|
+
bit the cache object itself carries.
|
|
554
|
+
"""
|
|
555
|
+
try:
|
|
556
|
+
return bool(cache_path.stat().st_mode & stat.S_IXUSR)
|
|
557
|
+
except FileNotFoundError:
|
|
558
|
+
return None
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def _repair_tree_dir_isexec(cache_dir_path: pathlib.Path, manifest: list[DirManifestEntry]) -> None:
|
|
562
|
+
"""Chmod any exec-marked file in a whole-directory SYMLINK cache tree that is missing
|
|
563
|
+
its executable bit, to 0o555.
|
|
564
|
+
|
|
565
|
+
A tree an older pivot built (every file chmod'd 0o444 regardless of isexec) keeps being
|
|
566
|
+
reused across an upgrade: its tree hash already incorporates isexec, so it never changes
|
|
567
|
+
and neither _store_directory's ``not cache_dir_path.exists()`` branch nor the
|
|
568
|
+
symlink_fast_path restore would otherwise ever revisit it. Chmod'ing in place is safe
|
|
569
|
+
here, unlike the single-file cache: a tree directory is a private ``copytree`` copy with
|
|
570
|
+
no other hardlinks sharing its files.
|
|
571
|
+
|
|
572
|
+
``manifest`` may come straight from a lock file on the restore path, so each relpath is
|
|
573
|
+
checked the same way the file-by-file restore branch already does: a manifest claiming
|
|
574
|
+
``../../../outside/file`` must not chmod a file outside the tree.
|
|
575
|
+
"""
|
|
576
|
+
resolved_tree = cache_dir_path.resolve()
|
|
577
|
+
for entry in manifest:
|
|
578
|
+
if not entry["isexec"]:
|
|
579
|
+
continue
|
|
580
|
+
f = cache_dir_path / entry["relpath"]
|
|
581
|
+
if not f.resolve().is_relative_to(resolved_tree):
|
|
582
|
+
raise exceptions.SecurityValidationError(
|
|
583
|
+
f"Manifest contains path traversal: {entry['relpath']!r}"
|
|
584
|
+
)
|
|
585
|
+
if _cache_object_isexec(f) is False:
|
|
586
|
+
os.chmod(f, 0o555)
|
|
587
|
+
|
|
588
|
+
|
|
468
589
|
def _checkout_with_fallback(
|
|
469
590
|
path: pathlib.Path,
|
|
470
591
|
cache_path: pathlib.Path,
|
|
471
592
|
checkout_modes: list[CheckoutMode],
|
|
472
593
|
*,
|
|
473
|
-
executable: bool =
|
|
594
|
+
executable: bool | None = None,
|
|
474
595
|
staging_path: pathlib.Path | None = None,
|
|
475
596
|
keep_existing_copy: bool = False,
|
|
597
|
+
display_path: pathlib.Path | None = None,
|
|
476
598
|
) -> None:
|
|
477
599
|
"""Try each link mode in order until one succeeds.
|
|
478
600
|
|
|
479
601
|
With ``keep_existing_copy``, reaching COPY while ``path`` still exists leaves it in place:
|
|
480
602
|
the caller guarantees it already holds the cached bytes. A failed link attempt clears
|
|
481
603
|
``path`` first, so COPY then copies from the cache as usual.
|
|
604
|
+
|
|
605
|
+
``executable`` is None for a single-file output: ``FileHash`` has no isexec field, so a
|
|
606
|
+
single file's mode is never part of any hash and there is no wrong permission to guard
|
|
607
|
+
against. For a directory entry (``executable`` is a bool), a HARDLINK or SYMLINK whose
|
|
608
|
+
cache object's recorded executable bit does not match it is skipped rather than attempted:
|
|
609
|
+
two files with identical content can disagree on the executable bit (the cache key is
|
|
610
|
+
content-only), and a cache object a remote pull populated by hash alone carries no
|
|
611
|
+
executable bit at all. Either way the shared object cannot represent two permissions at
|
|
612
|
+
once, so checkout falls through to the next mode; only COPY makes a private copy and can
|
|
613
|
+
always re-chmod it correctly.
|
|
614
|
+
|
|
615
|
+
``display_path`` names the file in a raised error when ``path`` is a restore-only staging
|
|
616
|
+
location (a caller checking out into a scratch directory before an atomic swap); it
|
|
617
|
+
defaults to ``path`` for every other caller.
|
|
482
618
|
"""
|
|
483
619
|
if not checkout_modes:
|
|
484
620
|
raise ValueError("checkout_modes cannot be empty")
|
|
621
|
+
display = display_path if display_path is not None else path
|
|
485
622
|
for i, mode in enumerate(checkout_modes):
|
|
623
|
+
is_last = i == len(checkout_modes) - 1
|
|
486
624
|
if keep_existing_copy and mode == CheckoutMode.COPY and path.exists():
|
|
487
625
|
return
|
|
626
|
+
if executable is not None and mode in (CheckoutMode.HARDLINK, CheckoutMode.SYMLINK):
|
|
627
|
+
cached_isexec = _cache_object_isexec(cache_path)
|
|
628
|
+
if cached_isexec is not None and cached_isexec != executable:
|
|
629
|
+
if is_last:
|
|
630
|
+
cached_state = "executable" if cached_isexec else "non-executable"
|
|
631
|
+
raise exceptions.CacheModeCollisionError(
|
|
632
|
+
f"Cache object {cache_path} is recorded {cached_state}, but {display} "
|
|
633
|
+
+ f"needs isexec={executable}."
|
|
634
|
+
)
|
|
635
|
+
logger.debug(
|
|
636
|
+
f"Cache object {cache_path} is executable={cached_isexec}, "
|
|
637
|
+
+ f"need {executable}; skipping {mode.value}"
|
|
638
|
+
)
|
|
639
|
+
continue
|
|
488
640
|
try:
|
|
489
641
|
_checkout_from_cache(
|
|
490
642
|
path, cache_path, mode, executable=executable, staging_path=staging_path
|
|
491
643
|
)
|
|
492
644
|
return
|
|
493
645
|
except OSError as e:
|
|
494
|
-
if e.errno not in FALLBACK_ERRNO or
|
|
646
|
+
if e.errno not in FALLBACK_ERRNO or is_last:
|
|
495
647
|
raise
|
|
496
648
|
logger.debug(f"Checkout mode {mode.value} failed ({e}), trying next mode")
|
|
497
649
|
|
|
@@ -513,6 +665,20 @@ def _resolve_checkout_modes(
|
|
|
513
665
|
return DEFAULT_CHECKOUT_MODE_ORDER.copy()
|
|
514
666
|
|
|
515
667
|
|
|
668
|
+
def missing_cache_objects(
|
|
669
|
+
output_hash: HashInfo, cache_dir: pathlib.Path, checkout_modes: list[CheckoutMode]
|
|
670
|
+
) -> list[str]:
|
|
671
|
+
"""Return hashes required to restore an output that are absent from the cache."""
|
|
672
|
+
if is_dir_hash(output_hash):
|
|
673
|
+
tree_path = get_cache_path(cache_dir, output_hash["hash"])
|
|
674
|
+
if checkout_modes[0] == CheckoutMode.SYMLINK and tree_path.exists():
|
|
675
|
+
return []
|
|
676
|
+
hashes = [entry["hash"] for entry in output_hash["manifest"]]
|
|
677
|
+
else:
|
|
678
|
+
hashes = [output_hash["hash"]]
|
|
679
|
+
return [file_hash for file_hash in hashes if not get_cache_path(cache_dir, file_hash).exists()]
|
|
680
|
+
|
|
681
|
+
|
|
516
682
|
def save_to_cache(
|
|
517
683
|
path: pathlib.Path,
|
|
518
684
|
cache_dir: pathlib.Path,
|
|
@@ -578,7 +744,7 @@ def _store_file(
|
|
|
578
744
|
cache_path = get_cache_path(cache_dir, file_hash)
|
|
579
745
|
copy_to_cache(path, cache_path)
|
|
580
746
|
with restore_lock(path, state_dir) as lock_token:
|
|
581
|
-
staging_path =
|
|
747
|
+
staging_path = restore_staging_path(path, lock_token)
|
|
582
748
|
_clear_path(staging_path)
|
|
583
749
|
_checkout_with_fallback(
|
|
584
750
|
path,
|
|
@@ -604,10 +770,10 @@ def _save_directory_to_cache(
|
|
|
604
770
|
cache_dir_path = get_cache_path(cache_dir, existing_hash)
|
|
605
771
|
if cache_dir_path.exists():
|
|
606
772
|
# Already correctly linked - compute manifest from actual content
|
|
607
|
-
_, manifest = hash_directory(path, state_db)
|
|
773
|
+
_, manifest = hash_directory(path, state_db, cache_dir=cache_dir)
|
|
608
774
|
return DirHash(hash=existing_hash, manifest=manifest)
|
|
609
775
|
|
|
610
|
-
tree_hash, manifest = hash_directory(path, state_db)
|
|
776
|
+
tree_hash, manifest = hash_directory(path, state_db, cache_dir=cache_dir)
|
|
611
777
|
output_hash = DirHash(hash=tree_hash, manifest=manifest)
|
|
612
778
|
_store_directory(
|
|
613
779
|
path,
|
|
@@ -634,7 +800,7 @@ def _store_directory(
|
|
|
634
800
|
for entry in manifest:
|
|
635
801
|
file_path = path / entry["relpath"]
|
|
636
802
|
cache_path = get_cache_path(cache_dir, entry["hash"])
|
|
637
|
-
copy_to_cache(file_path, cache_path)
|
|
803
|
+
copy_to_cache(file_path, cache_path, executable=entry["isexec"])
|
|
638
804
|
|
|
639
805
|
with restore_lock(path, state_dir):
|
|
640
806
|
if checkout_modes and checkout_modes[0] == CheckoutMode.SYMLINK:
|
|
@@ -643,12 +809,20 @@ def _store_directory(
|
|
|
643
809
|
if not cache_dir_path.exists():
|
|
644
810
|
cache_dir_path.parent.mkdir(parents=True, exist_ok=True)
|
|
645
811
|
shutil.copytree(path, cache_dir_path)
|
|
812
|
+
isexec_by_relpath = {entry["relpath"]: entry["isexec"] for entry in manifest}
|
|
646
813
|
for f in cache_dir_path.rglob("*"):
|
|
647
814
|
# Skip symlinks to avoid changing permissions on target files outside cache
|
|
648
815
|
if f.is_file() and not f.is_symlink():
|
|
649
|
-
|
|
816
|
+
relpath = str(f.relative_to(cache_dir_path))
|
|
817
|
+
executable = isexec_by_relpath.get(relpath, False)
|
|
818
|
+
os.chmod(f, 0o555 if executable else 0o444)
|
|
650
819
|
os.chmod(cache_dir_path, 0o555)
|
|
651
820
|
|
|
821
|
+
# A tree an older pivot built keeps its 0o444-everywhere files forever otherwise:
|
|
822
|
+
# the tree hash already incorporates isexec, so this branch never revisits an
|
|
823
|
+
# existing tree directory above.
|
|
824
|
+
_repair_tree_dir_isexec(cache_dir_path, manifest)
|
|
825
|
+
|
|
652
826
|
_clear_path(path)
|
|
653
827
|
path.symlink_to(cache_dir_path.resolve())
|
|
654
828
|
else:
|
|
@@ -656,7 +830,7 @@ def _store_directory(
|
|
|
656
830
|
for entry in manifest:
|
|
657
831
|
file_path = path / entry["relpath"]
|
|
658
832
|
cache_path = get_cache_path(cache_dir, entry["hash"])
|
|
659
|
-
staging_path =
|
|
833
|
+
staging_path = restore_staging_path(file_path)
|
|
660
834
|
_clear_path(staging_path)
|
|
661
835
|
_checkout_with_fallback(
|
|
662
836
|
file_path,
|
|
@@ -777,7 +951,7 @@ def _restore_file_from_cache(
|
|
|
777
951
|
lock_token=held_token,
|
|
778
952
|
)
|
|
779
953
|
|
|
780
|
-
staging_path =
|
|
954
|
+
staging_path = restore_staging_path(path, lock_token)
|
|
781
955
|
_clear_path(staging_path)
|
|
782
956
|
_checkout_with_fallback(path, cache_path, checkout_modes, staging_path=staging_path)
|
|
783
957
|
return True
|
|
@@ -820,6 +994,10 @@ def _restore_directory_from_cache(
|
|
|
820
994
|
restore_prefix = f"{_RESTORE_TEMP_PREFIX}{target_token}."
|
|
821
995
|
|
|
822
996
|
if symlink_fast_path:
|
|
997
|
+
# Same repair as _store_directory: a tree an older pivot built keeps its
|
|
998
|
+
# 0o444-everywhere files forever otherwise, since the tree hash already
|
|
999
|
+
# incorporates isexec and this path never rebuilds an existing tree directory.
|
|
1000
|
+
_repair_tree_dir_isexec(cache_dir_path, output_hash["manifest"])
|
|
823
1001
|
temp_symlink = path.parent / f"{restore_prefix}symlink_{secrets.token_hex(4)}"
|
|
824
1002
|
try:
|
|
825
1003
|
temp_symlink.symlink_to(cache_dir_path.resolve())
|
|
@@ -849,7 +1027,11 @@ def _restore_directory_from_cache(
|
|
|
849
1027
|
)
|
|
850
1028
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
851
1029
|
_checkout_with_fallback(
|
|
852
|
-
file_path,
|
|
1030
|
+
file_path,
|
|
1031
|
+
file_cache_path,
|
|
1032
|
+
checkout_modes,
|
|
1033
|
+
executable=entry["isexec"],
|
|
1034
|
+
display_path=path / entry["relpath"],
|
|
853
1035
|
)
|
|
854
1036
|
if file_count % 100 == 0:
|
|
855
1037
|
try:
|
|
@@ -237,12 +237,18 @@ def _verify_content_hash(content: bytes, expected_hash: str) -> bool:
|
|
|
237
237
|
return actual_hash == expected_hash
|
|
238
238
|
|
|
239
239
|
|
|
240
|
-
def _write_fallback_content(dest_path: pathlib.Path, content: bytes
|
|
241
|
-
"""
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
240
|
+
def _write_fallback_content(dest_path: pathlib.Path, content: bytes) -> None:
|
|
241
|
+
"""Atomically replace a cache-miss target under the cache restore lock."""
|
|
242
|
+
|
|
243
|
+
def write_content(fd: int) -> None:
|
|
244
|
+
with os.fdopen(fd, "wb") as file:
|
|
245
|
+
file.write(content)
|
|
246
|
+
|
|
247
|
+
with cache.restore_lock(dest_path, config.get_state_dir()) as lock_token:
|
|
248
|
+
staging_path = (
|
|
249
|
+
cache.restore_staging_path(dest_path, lock_token) if lock_token is not None else None
|
|
250
|
+
)
|
|
251
|
+
cache.atomic_write_file(dest_path, write_content, staging_path=staging_path)
|
|
246
252
|
|
|
247
253
|
|
|
248
254
|
def restore_file(
|
|
@@ -299,7 +305,7 @@ def restore_file(
|
|
|
299
305
|
content = git.read_file_from_revision(rel_path, rev)
|
|
300
306
|
if content is not None:
|
|
301
307
|
try:
|
|
302
|
-
_write_fallback_content(dest_path, content
|
|
308
|
+
_write_fallback_content(dest_path, content)
|
|
303
309
|
return RestoreResult(
|
|
304
310
|
status=RestoreStatus.RESTORED,
|
|
305
311
|
path=path_str,
|
|
@@ -328,7 +334,7 @@ def restore_file(
|
|
|
328
334
|
),
|
|
329
335
|
)
|
|
330
336
|
try:
|
|
331
|
-
_write_fallback_content(dest_path, remote_content
|
|
337
|
+
_write_fallback_content(dest_path, remote_content)
|
|
332
338
|
except OSError as e:
|
|
333
339
|
return RestoreResult(
|
|
334
340
|
status=RestoreStatus.ERROR,
|
|
@@ -165,6 +165,11 @@ class WorkspaceStore:
|
|
|
165
165
|
self._cache_dir = cache_dir.resolve() if cache_dir is not None else None
|
|
166
166
|
self._checkout_modes = checkout_modes
|
|
167
167
|
|
|
168
|
+
@property
|
|
169
|
+
def cache_dir(self) -> pathlib.Path | None:
|
|
170
|
+
"""The cache object root (``<cache.dir>/files``), or None when this store has no cache."""
|
|
171
|
+
return self._cache_dir
|
|
172
|
+
|
|
168
173
|
def _resolve_input_path(self, ref: types.ArtifactRef) -> pathlib.Path:
|
|
169
174
|
name = ref.identity.producer
|
|
170
175
|
if name in self._input_bindings:
|
|
@@ -216,11 +221,21 @@ class WorkspaceStore:
|
|
|
216
221
|
path: pathlib.Path,
|
|
217
222
|
*,
|
|
218
223
|
state_dir: pathlib.Path | None = None,
|
|
224
|
+
warn_left_out_symlinks: bool = True,
|
|
219
225
|
) -> types.HashInfo:
|
|
220
|
-
"""Hash a written output once and, unless it is a metric, place it in the cache.
|
|
226
|
+
"""Hash a written output once and, unless it is a metric, place it in the cache.
|
|
227
|
+
|
|
228
|
+
``warn_left_out_symlinks`` logs the warning naming the symlinks a directory output
|
|
229
|
+
leaves out; a quiet command turns it off.
|
|
230
|
+
"""
|
|
221
231
|
output_hash: types.HashInfo
|
|
222
232
|
if path.is_dir():
|
|
223
|
-
tree_hash, manifest = cache.hash_directory(
|
|
233
|
+
tree_hash, manifest = cache.hash_directory(
|
|
234
|
+
path,
|
|
235
|
+
None,
|
|
236
|
+
cache_dir=self._cache_dir,
|
|
237
|
+
warn_left_out_symlinks=warn_left_out_symlinks,
|
|
238
|
+
)
|
|
224
239
|
output_hash = types.DirHash(hash=tree_hash, manifest=manifest)
|
|
225
240
|
else:
|
|
226
241
|
file_hash, _ = cache.hash_file(path, None)
|
|
@@ -261,7 +276,7 @@ class WorkspaceStore:
|
|
|
261
276
|
) -> types.HashInfo:
|
|
262
277
|
path = self._resolve_path(ref)
|
|
263
278
|
if ref.tag == types.ArtifactTag.DIRECTORY or path.is_dir():
|
|
264
|
-
tree_hash, manifest = cache.hash_directory(path, state_db)
|
|
279
|
+
tree_hash, manifest = cache.hash_directory(path, state_db, cache_dir=self._cache_dir)
|
|
265
280
|
return types.DirHash(hash=tree_hash, manifest=manifest)
|
|
266
281
|
file_hash, _ = cache.hash_file(path, state_db)
|
|
267
282
|
return types.FileHash(hash=file_hash)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|