pivot 0.3.1__tar.gz → 0.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. {pivot-0.3.1 → pivot-0.3.3}/PKG-INFO +1 -1
  2. {pivot-0.3.1 → pivot-0.3.3}/pyproject.toml +1 -1
  3. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/checkout.py +20 -8
  4. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/commit.py +3 -1
  5. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/track.py +22 -7
  6. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/exceptions.py +21 -0
  7. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/commit.py +10 -1
  8. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/core.py +1 -1
  9. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/worker.py +25 -6
  10. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/status.py +2 -1
  11. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/cache.py +215 -33
  12. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/restore.py +14 -8
  13. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/store.py +18 -3
  14. {pivot-0.3.1 → pivot-0.3.3}/LICENSE +0 -0
  15. {pivot-0.3.1 → pivot-0.3.3}/README.md +0 -0
  16. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/__init__.py +0 -0
  17. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/ast_utils.py +0 -0
  18. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/AGENTS.md +0 -0
  19. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/CLAUDE.md +0 -0
  20. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/__init__.py +0 -0
  21. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/_run_common.py +0 -0
  22. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/check_ignore.py +0 -0
  23. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/completion.py +0 -0
  24. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/config.py +0 -0
  25. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/console.py +0 -0
  26. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/dag.py +0 -0
  27. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/data.py +0 -0
  28. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/decorators.py +0 -0
  29. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/doctor.py +0 -0
  30. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/errors.py +0 -0
  31. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/fingerprint.py +0 -0
  32. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/helpers.py +0 -0
  33. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/history.py +0 -0
  34. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/import_cmd.py +0 -0
  35. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/init.py +0 -0
  36. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/list.py +0 -0
  37. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/metrics.py +0 -0
  38. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/params.py +0 -0
  39. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/plots.py +0 -0
  40. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/remote.py +0 -0
  41. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/repro.py +0 -0
  42. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/run.py +0 -0
  43. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/status.py +0 -0
  44. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/targets.py +0 -0
  45. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/update.py +0 -0
  46. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/cli/verify.py +0 -0
  47. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/compose.py +0 -0
  48. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/config/__init__.py +0 -0
  49. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/config/io.py +0 -0
  50. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/config/models.py +0 -0
  51. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/dag/__init__.py +0 -0
  52. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/dag/render.py +0 -0
  53. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/decorators.py +0 -0
  54. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/discovery.py +0 -0
  55. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/__init__.py +0 -0
  56. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/agent_rpc.py +0 -0
  57. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/engine.py +0 -0
  58. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/graph.py +0 -0
  59. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/scheduler.py +0 -0
  60. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/sinks.py +0 -0
  61. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/sources.py +0 -0
  62. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/types.py +0 -0
  63. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/watch.py +0 -0
  64. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/engine/worker_pool.py +0 -0
  65. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/AGENTS.md +0 -0
  66. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/CLAUDE.md +0 -0
  67. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/executor/__init__.py +0 -0
  68. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/explain.py +0 -0
  69. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/fingerprint.py +0 -0
  70. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/git.py +0 -0
  71. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/ignore.py +0 -0
  72. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/import_artifact.py +0 -0
  73. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/loaders.py +0 -0
  74. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/metrics.py +0 -0
  75. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/names.py +0 -0
  76. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/outputs.py +0 -0
  77. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/parameters.py +0 -0
  78. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/path_policy.py +0 -0
  79. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/project.py +0 -0
  80. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/py.typed +0 -0
  81. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/registry.py +0 -0
  82. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/AGENTS.md +0 -0
  83. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/__init__.py +0 -0
  84. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/config.py +0 -0
  85. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/git_archive.py +0 -0
  86. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/github.py +0 -0
  87. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/storage.py +0 -0
  88. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/remote/sync.py +0 -0
  89. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/run_history.py +0 -0
  90. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/__init__.py +0 -0
  91. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/common.py +0 -0
  92. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/data.py +0 -0
  93. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/metrics.py +0 -0
  94. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/params.py +0 -0
  95. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/show/plots.py +0 -0
  96. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/skip.py +0 -0
  97. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/stage_def.py +0 -0
  98. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/AGENTS.md +0 -0
  99. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/CLAUDE.md +0 -0
  100. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/__init__.py +0 -0
  101. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/artifact_lock.py +0 -0
  102. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/lock.py +0 -0
  103. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/presentation.py +0 -0
  104. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/state.py +0 -0
  105. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/storage/track.py +0 -0
  106. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/types.py +0 -0
  107. {pivot-0.3.1 → pivot-0.3.3}/src/pivot/yaml_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pivot
3
- Version: 0.3.1
3
+ Version: 0.3.3
4
4
  Summary: High-performance Python pipeline tool with automatic code change detection
5
5
  Author: Sami Jawhar
6
6
  Author-email: Sami Jawhar <sami@metr.org>
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "pivot"
3
- version = "0.3.1"
3
+ version = "0.3.3"
4
4
  description = "High-performance Python pipeline tool with automatic code change detection"
5
5
  requires-python = ">=3.13,<3.14"
6
6
  readme = "README.md"
@@ -8,9 +8,10 @@ from typing import TYPE_CHECKING, Literal
8
8
 
9
9
  import click
10
10
 
11
- from pivot import config, project, registry, types
11
+ from pivot import config, exceptions, project, registry, types
12
12
  from pivot.cli import completion
13
13
  from pivot.cli import decorators as cli_decorators
14
+ from pivot.cli import errors as cli_errors
14
15
  from pivot.cli import helpers as cli_helpers
15
16
  from pivot.cli import targets as cli_targets
16
17
  from pivot.storage import cache, lock, state, track
@@ -67,11 +68,13 @@ def _get_stage_output_info() -> dict[str, HashInfo]:
67
68
  return result
68
69
 
69
70
 
70
- def _on_disk_matches_expected(path: pathlib.Path, expected_hash: HashInfo) -> bool:
71
+ def _on_disk_matches_expected(
72
+ path: pathlib.Path, expected_hash: HashInfo, cache_dir: pathlib.Path
73
+ ) -> bool:
71
74
  if is_dir_hash(expected_hash):
72
75
  if not path.is_dir():
73
76
  return False
74
- actual_hash, _ = cache.hash_directory(path)
77
+ actual_hash, _ = cache.hash_directory(path, cache_dir=cache_dir)
75
78
  else:
76
79
  if not path.is_file():
77
80
  return False
@@ -119,7 +122,7 @@ def _restore_path_locked(
119
122
  if path.exists():
120
123
  match behavior:
121
124
  case CheckoutBehavior.ERROR:
122
- if _on_disk_matches_expected(path, output_hash):
125
+ if _on_disk_matches_expected(path, output_hash, cache_dir):
123
126
  return ("skipped", path.name)
124
127
  raise click.ClickException(
125
128
  f"'{path.name}' already exists with different content. "
@@ -150,7 +153,12 @@ def _restore_path_locked(
150
153
  return ("missing", path.name)
151
154
  return ("restored" if outcome["restored"] else "skipped", path.name)
152
155
  case CheckoutBehavior.FORCE:
153
- cache.remove_output(path)
156
+ missing = cache.missing_cache_objects(output_hash, cache_dir, checkout_modes)
157
+ if missing:
158
+ raise click.ClickException(
159
+ f"Cannot overwrite '{path.name}': cache object missing: {', '.join(missing)}. "
160
+ + "The workspace output was not changed."
161
+ )
154
162
  case _: # pyright: ignore[reportUnnecessaryComparison] - defensive for future enum values
155
163
  raise ValueError(f"Unhandled checkout behavior: {behavior}") # pyright: ignore[reportUnreachable]
156
164
 
@@ -222,9 +230,13 @@ async def _checkout_files_async(
222
230
  for abs_path_str, output_hash in files.items():
223
231
  _ = tg.create_task(restore_one(abs_path_str, output_hash))
224
232
  except* Exception as eg:
225
- # Convert unexpected exceptions to friendly error message
226
- errors = [str(e) for e in eg.exceptions]
227
- raise click.ClickException("\n".join(errors)) from None
233
+ # Convert unexpected exceptions to a friendly error message, rendering a PivotError's
234
+ # actionable suggestion the same way the top-level CLI error handler does (cli/errors.py).
235
+ messages = [
236
+ cli_errors.format_pivot_error(e) if isinstance(e, exceptions.PivotError) else str(e)
237
+ for e in eg.exceptions
238
+ ]
239
+ raise click.ClickException("\n".join(messages)) from None
228
240
 
229
241
  # Re-raise immediate failures (all of them, aggregated)
230
242
  if immediate_errors:
@@ -20,7 +20,9 @@ def commit_command(ctx: click.Context, stages: tuple[str, ...]) -> None:
20
20
  quiet = cli_ctx["quiet"]
21
21
 
22
22
  stage_names = cli_helpers.resolve_stage_names(list(stages) if stages else None)
23
- committed, failed = commit.commit_stages(cli_helpers.get_pipeline(), stage_names)
23
+ committed, failed = commit.commit_stages(
24
+ cli_helpers.get_pipeline(), stage_names, warn_left_out_symlinks=not quiet
25
+ )
24
26
 
25
27
  if not quiet:
26
28
  if committed:
@@ -86,9 +86,14 @@ def _track_single_path(
86
86
  stage_outputs_resolved: dict[str, pathlib.Path],
87
87
  existing_tracked: dict[str, track_mod.PvtData],
88
88
  force: bool,
89
+ *,
90
+ warn_left_out_symlinks: bool,
89
91
  ) -> TrackResult:
90
92
  """Track a single file or directory.
91
93
 
94
+ ``warn_left_out_symlinks`` logs the warning naming the symlinks a tracked directory leaves
95
+ out; the caller turns it off under ``--quiet``, like this command's other warning.
96
+
92
97
  Returns:
93
98
  TrackResult with optional warning and the tracked path for the caller to handle output.
94
99
  """
@@ -149,16 +154,20 @@ def _track_single_path(
149
154
 
150
155
  # Hash and cache
151
156
  if path.is_dir():
152
- tree_hash, manifest = cache.hash_directory(path)
157
+ tree_hash, manifest = cache.hash_directory(
158
+ path, cache_dir=cache_dir, warn_left_out_symlinks=warn_left_out_symlinks
159
+ )
153
160
  total_size = sum(e["size"] for e in manifest)
154
161
  num_files = len(manifest)
155
162
 
156
- # Save each file to cache
163
+ # copy_to_cache is idempotent and already checks existence itself; letting it see
164
+ # every entry (not just ones missing from the cache) is what lets it republish an
165
+ # object `pivot track --force` finds cached non-executable by an older pivot or a
166
+ # cold `pivot pull`, when this entry's bytes should be executable.
157
167
  for entry in manifest:
158
168
  file_path = path / entry["relpath"]
159
169
  file_cache_path = cache.get_cache_path(cache_dir, entry["hash"])
160
- if not file_cache_path.exists():
161
- cache.copy_to_cache(file_path, file_cache_path)
170
+ cache.copy_to_cache(file_path, file_cache_path, executable=entry["isexec"])
162
171
 
163
172
  pvt_data: track_mod.PvtData = {
164
173
  "path": path.name,
@@ -171,8 +180,7 @@ def _track_single_path(
171
180
  file_hash, _ = cache.hash_file(path)
172
181
  file_size = path.stat().st_size
173
182
  file_cache_path = cache.get_cache_path(cache_dir, file_hash)
174
- if not file_cache_path.exists():
175
- cache.copy_to_cache(path, file_cache_path)
183
+ cache.copy_to_cache(path, file_cache_path)
176
184
 
177
185
  pvt_data = {
178
186
  "path": path.name,
@@ -212,7 +220,14 @@ def track(ctx: click.Context, paths: tuple[str, ...], force: bool) -> None:
212
220
  existing_tracked = track_mod.discover_pvt_files(project_root)
213
221
 
214
222
  for path_str in paths:
215
- result = _track_single_path(path_str, cache_dir, stage_outputs, existing_tracked, force)
223
+ result = _track_single_path(
224
+ path_str,
225
+ cache_dir,
226
+ stage_outputs,
227
+ existing_tracked,
228
+ force,
229
+ warn_left_out_symlinks=not quiet,
230
+ )
216
231
  if not quiet:
217
232
  if result["warning"]:
218
233
  click.echo(result["warning"], err=True)
@@ -214,6 +214,27 @@ class UncachedIncrementalOutputError(CacheError):
214
214
  """Raised when an incremental output exists without a cache entry."""
215
215
 
216
216
 
217
+ class CacheModeCollisionError(CacheError):
218
+ """A HARDLINK/SYMLINK checkout shares the cache object's own permissions and found it
219
+ already recorded with the wrong executable bit, with no COPY fallback available.
220
+
221
+ This happens when two files with identical content disagree on their executable bit (the
222
+ cache key is content-only) or when a cache object was populated by a path that does not
223
+ know a file's intended executable bit (e.g. `pivot pull`, which downloads by hash alone).
224
+ """
225
+
226
+ @override
227
+ def get_suggestion(self) -> str:
228
+ return (
229
+ "Add CheckoutMode.COPY to checkout_modes (cache: checkout_mode: [..., copy] in "
230
+ ".pivot/config.yaml) so checkout can fall back to a private, correctly-permissioned "
231
+ "copy instead of sharing the cache object's own permissions; or, if you have the "
232
+ "source bytes in hand, re-running what produced them (`pivot repro --force "
233
+ "<stage>`, `pivot track --force <path>`) republishes the cache object with the "
234
+ "right bit"
235
+ )
236
+
237
+
217
238
  class ParamsError(PivotError):
218
239
  """Raised when parameter validation or loading fails."""
219
240
 
@@ -24,6 +24,8 @@ def commit_stages(
24
24
  pipeline: registry.PipelineLike,
25
25
  stage_names: list[str] | None = None,
26
26
  force: bool = False,
27
+ *,
28
+ warn_left_out_symlinks: bool = True,
27
29
  ) -> tuple[list[str], list[str]]:
28
30
  """Commit current workspace state for stages.
29
31
 
@@ -37,6 +39,8 @@ def commit_stages(
37
39
  pipeline: The pipeline whose stages are committed.
38
40
  stage_names: Specific stages to commit. None means all stale stages.
39
41
  force: If True, commit even if lock file is unchanged.
42
+ warn_left_out_symlinks: Log the warning naming symlinks a directory output leaves
43
+ out. A quiet command turns it off.
40
44
 
41
45
  Returns:
42
46
  Tuple of (committed, failed) stage name lists.
@@ -150,7 +154,12 @@ def commit_stages(
150
154
  identity_key = types.identity_key(out.identity)
151
155
  try:
152
156
  out_path = store.resolve_display_path(out)
153
- hash_info = store.commit(out, out_path, state_dir=default_state_dir)
157
+ hash_info = store.commit(
158
+ out,
159
+ out_path,
160
+ state_dir=default_state_dir,
161
+ warn_left_out_symlinks=warn_left_out_symlinks,
162
+ )
154
163
  file_hash_entries.extend(worker.output_file_hash_entries(out_path, hash_info))
155
164
  output_hashes_by_key[identity_key] = hash_info
156
165
  output_hashes_by_identity[out.identity] = hash_info
@@ -409,7 +409,7 @@ def verify_tracked_files(project_root: pathlib.Path, checkout_missing: bool = Fa
409
409
  if path.is_file():
410
410
  current_hash, _ = cache.hash_file(path, state_db)
411
411
  elif path.is_dir():
412
- current_hash, _ = cache.hash_directory(path, state_db)
412
+ current_hash, _ = cache.hash_directory(path, state_db, cache_dir=cache_dir)
413
413
  else:
414
414
  # Path doesn't exist
415
415
  raise FileNotFoundError(data_path)
@@ -411,7 +411,9 @@ def execute_stage(
411
411
  project_root,
412
412
  )
413
413
  if no_commit:
414
- output_hashes = _hash_output_paths(output_paths)
414
+ output_hashes = _hash_output_paths(
415
+ output_paths, cache_dir=_store_cache_dir(store)
416
+ )
415
417
  else:
416
418
  output_hashes = _commit_outputs_with_store(
417
419
  output_paths, store, project_state_dir
@@ -556,7 +558,7 @@ def restore_outputs_to_hashes(
556
558
  if expected_hash is not None:
557
559
  output_path = store.resolve_display_path(ref)
558
560
  try:
559
- actual_hash = hash_output(output_path, state_db)
561
+ actual_hash = hash_output(output_path, state_db, cache_dir=store.cache_dir)
560
562
  except OSError:
561
563
  actual_hash = None
562
564
  if actual_hash == expected_hash:
@@ -613,10 +615,23 @@ def _check_skip_or_run(
613
615
  return "unchanged", "", input_hash, decision["run_cache_allowed"]
614
616
 
615
617
 
616
- def hash_output(path: pathlib.Path, state_db: state.StateDB | None = None) -> HashInfo:
617
- """Compute output hash without saving to cache."""
618
+ def _store_cache_dir(store: store_mod.Store) -> pathlib.Path | None:
619
+ """The cache object root whose symlinks count as directory content for ``store``."""
620
+ return store.cache_dir if isinstance(store, store_mod.WorkspaceStore) else None
621
+
622
+
623
+ def hash_output(
624
+ path: pathlib.Path,
625
+ state_db: state.StateDB | None = None,
626
+ *,
627
+ cache_dir: pathlib.Path | None = None,
628
+ ) -> HashInfo:
629
+ """Compute output hash without saving to cache.
630
+
631
+ ``cache_dir`` is the cache object root whose symlinks count as directory content.
632
+ """
618
633
  if path.is_dir():
619
- tree_hash, manifest = cache.hash_directory(path, state_db)
634
+ tree_hash, manifest = cache.hash_directory(path, state_db, cache_dir=cache_dir)
620
635
  return DirHash(hash=tree_hash, manifest=manifest)
621
636
  file_hash, _ = cache.hash_file(path, state_db)
622
637
  return FileHash(hash=file_hash)
@@ -1040,13 +1055,17 @@ def _run_stage_function_with_store(
1040
1055
  def _hash_output_paths(
1041
1056
  outputs_by_ref: dict[types.ArtifactRef, pathlib.Path],
1042
1057
  state_db: state.StateDB | None = None,
1058
+ *,
1059
+ cache_dir: pathlib.Path | None = None,
1043
1060
  ) -> dict[str, HashInfo]:
1044
1061
  output_hashes = dict[str, HashInfo]()
1045
1062
  for ref, path in outputs_by_ref.items():
1046
1063
  if not path.exists():
1047
1064
  raise exceptions.OutputMissingError(f"Stage did not produce output: {path}")
1048
1065
  if path.is_dir():
1049
- tree_hash, manifest = cache.hash_directory(path, state_db)
1066
+ tree_hash, manifest = cache.hash_directory(
1067
+ path, state_db, cache_dir=cache_dir, warn_left_out_symlinks=True
1068
+ )
1050
1069
  output_hashes[types.identity_key(ref.identity)] = DirHash(
1051
1070
  hash=tree_hash, manifest=manifest
1052
1071
  )
@@ -348,6 +348,7 @@ def get_tracked_files_status(
348
348
  results = list[TrackedFileInfo]()
349
349
 
350
350
  # Use state_db for hash caching (mtime-based)
351
+ cache_dir = config.get_cache_dir() / "files"
351
352
  with state_mod.StateDB(config.get_state_dir()) as state_db:
352
353
  for i, (abs_path_str, track_data) in enumerate(sorted(tracked.items()), 1):
353
354
  path = pathlib.Path(abs_path_str)
@@ -355,7 +356,7 @@ def get_tracked_files_status(
355
356
 
356
357
  try:
357
358
  if path.is_dir():
358
- current_hash, _ = cache.hash_directory(path, state_db)
359
+ current_hash, _ = cache.hash_directory(path, state_db, cache_dir=cache_dir)
359
360
  else:
360
361
  current_hash, _ = cache.hash_file(path, state_db)
361
362
  except FileNotFoundError:
@@ -45,8 +45,8 @@ def _restore_target_token(target_path: pathlib.Path) -> str:
45
45
  return hashlib.sha256(str(logical_path).encode()).hexdigest()[:32]
46
46
 
47
47
 
48
- def _restore_staging_path(path: pathlib.Path, token: str | None = None) -> pathlib.Path:
49
- """Return the deterministic staging path for a file checkout."""
48
+ def restore_staging_path(path: pathlib.Path, token: str | None = None) -> pathlib.Path:
49
+ """Return the deterministic staging path for a file restore."""
50
50
  return path.parent / f"{_RESTORE_TEMP_PREFIX}{token or _restore_target_token(path)}.file"
51
51
 
52
52
 
@@ -92,26 +92,37 @@ def atomic_write_file(
92
92
  dest: pathlib.Path,
93
93
  write_fn: Callable[[int], None],
94
94
  mode: int = 0o644,
95
+ *,
96
+ staging_path: pathlib.Path | None = None,
95
97
  ) -> None:
96
- """Atomically write to dest using temp file + rename pattern.
98
+ """Atomically write to dest using a temporary file and replacement.
99
+
100
+ A caller holding a restore lock can supply its target-tagged staging path so a
101
+ later restore of the same target cleans up a process killed before replacement.
97
102
 
98
103
  Args:
99
104
  dest: Target file path.
100
105
  write_fn: Function that receives the file descriptor and writes content.
101
106
  MUST close fd (typically via os.fdopen which takes ownership).
102
107
  mode: File permissions (default 0o644).
108
+ staging_path: Optional deterministic temporary path beside ``dest``.
103
109
  """
104
110
  dest.parent.mkdir(parents=True, exist_ok=True)
105
- fd, tmp_path = tempfile.mkstemp(dir=dest.parent, suffix=".tmp")
106
- tmp = pathlib.Path(tmp_path)
111
+ if staging_path is None:
112
+ fd, tmp_path = tempfile.mkstemp(dir=dest.parent, suffix=".tmp")
113
+ tmp = pathlib.Path(tmp_path)
114
+ else:
115
+ tmp = staging_path
116
+ _clear_path(tmp)
117
+ fd = os.open(tmp, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
107
118
  fd_closed = False
108
119
  try:
109
120
  write_fn(fd)
110
121
  fd_closed = True # write_fn took ownership via os.fdopen
111
- os.chmod(tmp_path, mode)
122
+ os.chmod(tmp, mode)
112
123
  tmp.replace(dest)
113
124
  except BaseException:
114
- tmp.unlink(missing_ok=True)
125
+ _clear_path(tmp)
115
126
  raise
116
127
  finally:
117
128
  # Only close fd if write_fn didn't (e.g., exception before os.fdopen)
@@ -159,39 +170,83 @@ def hash_file(
159
170
  return file_hash, file_stat
160
171
 
161
172
 
162
- def _scandir_recursive(path: pathlib.Path) -> Generator[os.DirEntry[str]]:
173
+ def _cache_object_key(path: pathlib.Path, cache_dir: pathlib.Path | None) -> str | None:
174
+ """Return the cache key a symlink resolves to, if its target is an object in ``cache_dir``.
175
+
176
+ Cache objects live at ``<cache_dir>/XX/YYYY...`` named by their content hash; a file-by-file
177
+ symlink checkout restores each directory member as a link to one. A link anywhere else,
178
+ even to a hash-named file, is a user symlink.
179
+ """
180
+ if cache_dir is None:
181
+ return None
182
+ key = _get_symlink_cache_hash(path, cache_dir)
183
+ if key is None or not is_valid_hash(key) or not path.is_file():
184
+ return None
185
+ return key
186
+
187
+
188
+ def _scandir_recursive(
189
+ path: pathlib.Path, cache_dir: pathlib.Path | None, left_out: list[pathlib.Path]
190
+ ) -> Generator[os.DirEntry[str]]:
163
191
  """Yield every file under `path` recursively, using os.scandir() for efficiency.
164
192
 
165
193
  A declared directory artifact is hashed and cached whole: no name is ignored, so a change
166
194
  to any file in a directory input reruns its consumers, and every file in a directory output
167
195
  survives cache and restore. Ignore rules belong to code fingerprinting and watch mode.
196
+ A symlink to an object in ``cache_dir`` (left by a symlink checkout) counts as the file it
197
+ points to; any other symlink is not stored and is appended to ``left_out``.
168
198
  DirEntry objects cache stat results, avoiding redundant syscalls.
169
- Symlinks are skipped to prevent loops.
170
199
  PermissionError propagates to caller - partial hashes would be incorrect.
171
200
 
172
201
  Args:
173
202
  path: Directory to scan
203
+ cache_dir: Cache object root whose links count as content, or None for no cache
204
+ left_out: Collects the symlinks that are not part of the artifact
174
205
  """
175
206
  with os.scandir(path) as entries:
176
207
  for entry in entries:
177
208
  if entry.is_symlink():
178
- continue
179
- if entry.is_file():
209
+ if _cache_object_key(pathlib.Path(entry.path), cache_dir) is not None:
210
+ yield entry
211
+ else:
212
+ left_out.append(pathlib.Path(entry.path))
213
+ elif entry.is_file():
180
214
  yield entry
181
215
  elif entry.is_dir():
182
- yield from _scandir_recursive(pathlib.Path(entry.path))
216
+ yield from _scandir_recursive(pathlib.Path(entry.path), cache_dir, left_out)
217
+
218
+
219
+ def _warn_left_out_symlinks(path: pathlib.Path, left_out: list[pathlib.Path]) -> None:
220
+ """Log one warning naming every symlink a directory artifact does not store."""
221
+ absolute = path.absolute()
222
+ cwd = pathlib.Path.cwd()
223
+ shown = absolute.relative_to(cwd) if absolute.is_relative_to(cwd) else path
224
+ names = ", ".join(sorted(str(link.relative_to(path)) for link in left_out))
225
+ one = len(left_out) == 1
226
+ logger.warning(
227
+ "Directory artifact %s leaves out %s %s (a symlink inside a directory artifact is not "
228
+ + "stored); replace %s with a real file or directory",
229
+ shown,
230
+ "symlink" if one else "symlinks",
231
+ names,
232
+ "it" if one else "each",
233
+ )
183
234
 
184
235
 
185
236
  def hash_directory(
186
237
  path: pathlib.Path,
187
238
  state_db: state_mod.StateDB | None = None,
239
+ *,
240
+ cache_dir: pathlib.Path | None = None,
241
+ warn_left_out_symlinks: bool = False,
188
242
  ) -> tuple[str, list[DirManifestEntry]]:
189
243
  """Compute tree hash of directory, returning hash and manifest.
190
244
 
191
245
  Symlink handling:
192
- - Symlinks INSIDE directories are skipped (prevents infinite loops)
246
+ - A symlink to an object in ``cache_dir`` is hashed by its target's content under its own
247
+ relpath; the content must match the object's name
248
+ - Any other symlink inside the directory is left out of the manifest
193
249
  - Base path may be symlinked (resolved for consistency)
194
- - Content-based hashing only (symlink metadata excluded from fingerprints)
195
250
 
196
251
  Note: For portability, paths are stored as normalized (symlinks preserved)
197
252
  in lock files, but resolved here for consistent hashing.
@@ -199,20 +254,30 @@ def hash_directory(
199
254
  Args:
200
255
  path: Directory to hash
201
256
  state_db: Optional StateDB for caching file hashes
257
+ cache_dir: The configured cache's object root (``<cache.dir>/files``). Pass it
258
+ wherever one exists, so recording and re-hashing agree on symlink checkouts;
259
+ None means no cache, and then no symlink is content.
260
+ warn_left_out_symlinks: Log one warning naming the symlinks left out. Set where a
261
+ manifest is recorded (a stage output, a tracked directory), not on every re-hash.
202
262
  """
203
263
  _t = metrics.start()
204
264
  manifest = list[DirManifestEntry]()
265
+ left_out = list[pathlib.Path]()
205
266
  resolved_base = path.resolve()
206
267
 
207
- for entry in sorted(_scandir_recursive(path), key=lambda e: e.path):
268
+ for entry in sorted(_scandir_recursive(path, cache_dir, left_out), key=lambda e: e.path):
208
269
  file_path = pathlib.Path(entry.path)
209
- # Verify file is still within the directory (paranoid check)
210
- if not file_path.resolve().is_relative_to(resolved_base):
270
+ # Verify a regular file is still within the directory (paranoid check). A cache-object
271
+ # link resolves into the cache by design.
272
+ if not entry.is_symlink() and not file_path.resolve().is_relative_to(resolved_base):
211
273
  continue
212
274
  try:
213
275
  rel = file_path.relative_to(path)
214
276
  file_stat = entry.stat(follow_symlinks=True)
215
277
  file_hash, _ = hash_file(file_path, state_db)
278
+ if entry.is_symlink() and file_hash != _cache_object_key(file_path, cache_dir):
279
+ left_out.append(file_path)
280
+ continue
216
281
  manifest_entry: DirManifestEntry = {
217
282
  "relpath": str(rel),
218
283
  "hash": file_hash,
@@ -223,6 +288,9 @@ def hash_directory(
223
288
  except FileNotFoundError:
224
289
  continue # File deleted between scan and hash
225
290
 
291
+ if left_out and warn_left_out_symlinks:
292
+ _warn_left_out_symlinks(path, left_out)
293
+
226
294
  manifest_json = json.dumps(manifest, sort_keys=True, separators=(",", ":"))
227
295
  tree_hash = xxhash.xxh64(manifest_json.encode()).hexdigest()
228
296
 
@@ -379,9 +447,21 @@ def _get_symlink_cache_hash(path: pathlib.Path, cache_dir: pathlib.Path) -> str
379
447
  return None # Includes ELOOP for circular symlinks
380
448
 
381
449
 
382
- def copy_to_cache(src: pathlib.Path, cache_path: pathlib.Path) -> None:
383
- """Atomically copy file to cache with read-only permissions."""
384
- if cache_path.exists():
450
+ def copy_to_cache(src: pathlib.Path, cache_path: pathlib.Path, *, executable: bool = False) -> None:
451
+ """Atomically copy file to cache with read-only permissions.
452
+
453
+ ``executable`` records the source's executable bit on the cache object itself (0o555
454
+ instead of 0o444): a HARDLINK or SYMLINK checkout has no mode of its own, so it reads
455
+ straight through to whatever mode the cache object carries, and only COPY checkout
456
+ re-chmods afterwards (see ``_stage_checkout``).
457
+
458
+ An existing object missing the executable bit is republished (a fresh temp file atomically
459
+ replacing the old one, never a chmod in place: existing hardlinks keep the old inode and
460
+ its mode) when ``executable`` is True, so an object an older pivot or a remote pull cached
461
+ non-executable can still be repaired by a caller who has the right bytes in hand. An
462
+ object already executable, or a non-executable request, is left alone.
463
+ """
464
+ if cache_path.exists() and (not executable or _cache_object_isexec(cache_path)):
385
465
  return
386
466
 
387
467
  cache_path.parent.mkdir(parents=True, exist_ok=True)
@@ -390,7 +470,7 @@ def copy_to_cache(src: pathlib.Path, cache_path: pathlib.Path) -> None:
390
470
  try:
391
471
  os.close(fd)
392
472
  shutil.copy2(src, tmp_path)
393
- os.chmod(tmp_path, 0o444)
473
+ os.chmod(tmp_path, 0o555 if executable else 0o444)
394
474
  tmp.replace(cache_path)
395
475
  except BaseException:
396
476
  tmp.unlink(missing_ok=True)
@@ -405,7 +485,7 @@ def _stage_checkout(
405
485
  cache_path: pathlib.Path,
406
486
  checkout_mode: CheckoutMode,
407
487
  *,
408
- executable: bool,
488
+ executable: bool | None,
409
489
  ) -> None:
410
490
  """Create the link or copy of cache_path at tmp_path."""
411
491
  if checkout_mode == CheckoutMode.SYMLINK:
@@ -422,7 +502,7 @@ def _checkout_from_cache(
422
502
  cache_path: pathlib.Path,
423
503
  checkout_mode: CheckoutMode,
424
504
  *,
425
- executable: bool = False,
505
+ executable: bool | None = None,
426
506
  staging_path: pathlib.Path | None = None,
427
507
  ) -> None:
428
508
  """Create link or copy at path atomically, so readers never see a partial file.
@@ -465,33 +545,105 @@ def _checkout_from_cache(
465
545
  tmp_path.unlink(missing_ok=True)
466
546
 
467
547
 
548
+ def _cache_object_isexec(cache_path: pathlib.Path) -> bool | None:
549
+ """Whether the cache object at cache_path is stored executable, or None if absent.
550
+
551
+ HARDLINK and SYMLINK checkout have no mode of their own: the workspace file shares the
552
+ cache object's permissions directly, so they can only ever produce whatever executable
553
+ bit the cache object itself carries.
554
+ """
555
+ try:
556
+ return bool(cache_path.stat().st_mode & stat.S_IXUSR)
557
+ except FileNotFoundError:
558
+ return None
559
+
560
+
561
+ def _repair_tree_dir_isexec(cache_dir_path: pathlib.Path, manifest: list[DirManifestEntry]) -> None:
562
+ """Chmod any exec-marked file in a whole-directory SYMLINK cache tree that is missing
563
+ its executable bit, to 0o555.
564
+
565
+ A tree an older pivot built (every file chmod'd 0o444 regardless of isexec) keeps being
566
+ reused across an upgrade: its tree hash already incorporates isexec, so it never changes
567
+ and neither _store_directory's ``not cache_dir_path.exists()`` branch nor the
568
+ symlink_fast_path restore would otherwise ever revisit it. Chmod'ing in place is safe
569
+ here, unlike the single-file cache: a tree directory is a private ``copytree`` copy with
570
+ no other hardlinks sharing its files.
571
+
572
+ ``manifest`` may come straight from a lock file on the restore path, so each relpath is
573
+ checked the same way the file-by-file restore branch already does: a manifest claiming
574
+ ``../../../outside/file`` must not chmod a file outside the tree.
575
+ """
576
+ resolved_tree = cache_dir_path.resolve()
577
+ for entry in manifest:
578
+ if not entry["isexec"]:
579
+ continue
580
+ f = cache_dir_path / entry["relpath"]
581
+ if not f.resolve().is_relative_to(resolved_tree):
582
+ raise exceptions.SecurityValidationError(
583
+ f"Manifest contains path traversal: {entry['relpath']!r}"
584
+ )
585
+ if _cache_object_isexec(f) is False:
586
+ os.chmod(f, 0o555)
587
+
588
+
468
589
  def _checkout_with_fallback(
469
590
  path: pathlib.Path,
470
591
  cache_path: pathlib.Path,
471
592
  checkout_modes: list[CheckoutMode],
472
593
  *,
473
- executable: bool = False,
594
+ executable: bool | None = None,
474
595
  staging_path: pathlib.Path | None = None,
475
596
  keep_existing_copy: bool = False,
597
+ display_path: pathlib.Path | None = None,
476
598
  ) -> None:
477
599
  """Try each link mode in order until one succeeds.
478
600
 
479
601
  With ``keep_existing_copy``, reaching COPY while ``path`` still exists leaves it in place:
480
602
  the caller guarantees it already holds the cached bytes. A failed link attempt clears
481
603
  ``path`` first, so COPY then copies from the cache as usual.
604
+
605
+ ``executable`` is None for a single-file output: ``FileHash`` has no isexec field, so a
606
+ single file's mode is never part of any hash and there is no wrong permission to guard
607
+ against. For a directory entry (``executable`` is a bool), a HARDLINK or SYMLINK whose
608
+ cache object's recorded executable bit does not match it is skipped rather than attempted:
609
+ two files with identical content can disagree on the executable bit (the cache key is
610
+ content-only), and a cache object a remote pull populated by hash alone carries no
611
+ executable bit at all. Either way the shared object cannot represent two permissions at
612
+ once, so checkout falls through to the next mode; only COPY makes a private copy and can
613
+ always re-chmod it correctly.
614
+
615
+ ``display_path`` names the file in a raised error when ``path`` is a restore-only staging
616
+ location (a caller checking out into a scratch directory before an atomic swap); it
617
+ defaults to ``path`` for every other caller.
482
618
  """
483
619
  if not checkout_modes:
484
620
  raise ValueError("checkout_modes cannot be empty")
621
+ display = display_path if display_path is not None else path
485
622
  for i, mode in enumerate(checkout_modes):
623
+ is_last = i == len(checkout_modes) - 1
486
624
  if keep_existing_copy and mode == CheckoutMode.COPY and path.exists():
487
625
  return
626
+ if executable is not None and mode in (CheckoutMode.HARDLINK, CheckoutMode.SYMLINK):
627
+ cached_isexec = _cache_object_isexec(cache_path)
628
+ if cached_isexec is not None and cached_isexec != executable:
629
+ if is_last:
630
+ cached_state = "executable" if cached_isexec else "non-executable"
631
+ raise exceptions.CacheModeCollisionError(
632
+ f"Cache object {cache_path} is recorded {cached_state}, but {display} "
633
+ + f"needs isexec={executable}."
634
+ )
635
+ logger.debug(
636
+ f"Cache object {cache_path} is executable={cached_isexec}, "
637
+ + f"need {executable}; skipping {mode.value}"
638
+ )
639
+ continue
488
640
  try:
489
641
  _checkout_from_cache(
490
642
  path, cache_path, mode, executable=executable, staging_path=staging_path
491
643
  )
492
644
  return
493
645
  except OSError as e:
494
- if e.errno not in FALLBACK_ERRNO or i == len(checkout_modes) - 1:
646
+ if e.errno not in FALLBACK_ERRNO or is_last:
495
647
  raise
496
648
  logger.debug(f"Checkout mode {mode.value} failed ({e}), trying next mode")
497
649
 
@@ -513,6 +665,20 @@ def _resolve_checkout_modes(
513
665
  return DEFAULT_CHECKOUT_MODE_ORDER.copy()
514
666
 
515
667
 
668
+ def missing_cache_objects(
669
+ output_hash: HashInfo, cache_dir: pathlib.Path, checkout_modes: list[CheckoutMode]
670
+ ) -> list[str]:
671
+ """Return hashes required to restore an output that are absent from the cache."""
672
+ if is_dir_hash(output_hash):
673
+ tree_path = get_cache_path(cache_dir, output_hash["hash"])
674
+ if checkout_modes[0] == CheckoutMode.SYMLINK and tree_path.exists():
675
+ return []
676
+ hashes = [entry["hash"] for entry in output_hash["manifest"]]
677
+ else:
678
+ hashes = [output_hash["hash"]]
679
+ return [file_hash for file_hash in hashes if not get_cache_path(cache_dir, file_hash).exists()]
680
+
681
+
516
682
  def save_to_cache(
517
683
  path: pathlib.Path,
518
684
  cache_dir: pathlib.Path,
@@ -578,7 +744,7 @@ def _store_file(
578
744
  cache_path = get_cache_path(cache_dir, file_hash)
579
745
  copy_to_cache(path, cache_path)
580
746
  with restore_lock(path, state_dir) as lock_token:
581
- staging_path = _restore_staging_path(path, lock_token)
747
+ staging_path = restore_staging_path(path, lock_token)
582
748
  _clear_path(staging_path)
583
749
  _checkout_with_fallback(
584
750
  path,
@@ -604,10 +770,10 @@ def _save_directory_to_cache(
604
770
  cache_dir_path = get_cache_path(cache_dir, existing_hash)
605
771
  if cache_dir_path.exists():
606
772
  # Already correctly linked - compute manifest from actual content
607
- _, manifest = hash_directory(path, state_db)
773
+ _, manifest = hash_directory(path, state_db, cache_dir=cache_dir)
608
774
  return DirHash(hash=existing_hash, manifest=manifest)
609
775
 
610
- tree_hash, manifest = hash_directory(path, state_db)
776
+ tree_hash, manifest = hash_directory(path, state_db, cache_dir=cache_dir)
611
777
  output_hash = DirHash(hash=tree_hash, manifest=manifest)
612
778
  _store_directory(
613
779
  path,
@@ -634,7 +800,7 @@ def _store_directory(
634
800
  for entry in manifest:
635
801
  file_path = path / entry["relpath"]
636
802
  cache_path = get_cache_path(cache_dir, entry["hash"])
637
- copy_to_cache(file_path, cache_path)
803
+ copy_to_cache(file_path, cache_path, executable=entry["isexec"])
638
804
 
639
805
  with restore_lock(path, state_dir):
640
806
  if checkout_modes and checkout_modes[0] == CheckoutMode.SYMLINK:
@@ -643,12 +809,20 @@ def _store_directory(
643
809
  if not cache_dir_path.exists():
644
810
  cache_dir_path.parent.mkdir(parents=True, exist_ok=True)
645
811
  shutil.copytree(path, cache_dir_path)
812
+ isexec_by_relpath = {entry["relpath"]: entry["isexec"] for entry in manifest}
646
813
  for f in cache_dir_path.rglob("*"):
647
814
  # Skip symlinks to avoid changing permissions on target files outside cache
648
815
  if f.is_file() and not f.is_symlink():
649
- os.chmod(f, 0o444)
816
+ relpath = str(f.relative_to(cache_dir_path))
817
+ executable = isexec_by_relpath.get(relpath, False)
818
+ os.chmod(f, 0o555 if executable else 0o444)
650
819
  os.chmod(cache_dir_path, 0o555)
651
820
 
821
+ # A tree an older pivot built keeps its 0o444-everywhere files forever otherwise:
822
+ # the tree hash already incorporates isexec, so this branch never revisits an
823
+ # existing tree directory above.
824
+ _repair_tree_dir_isexec(cache_dir_path, manifest)
825
+
652
826
  _clear_path(path)
653
827
  path.symlink_to(cache_dir_path.resolve())
654
828
  else:
@@ -656,7 +830,7 @@ def _store_directory(
656
830
  for entry in manifest:
657
831
  file_path = path / entry["relpath"]
658
832
  cache_path = get_cache_path(cache_dir, entry["hash"])
659
- staging_path = _restore_staging_path(file_path)
833
+ staging_path = restore_staging_path(file_path)
660
834
  _clear_path(staging_path)
661
835
  _checkout_with_fallback(
662
836
  file_path,
@@ -777,7 +951,7 @@ def _restore_file_from_cache(
777
951
  lock_token=held_token,
778
952
  )
779
953
 
780
- staging_path = _restore_staging_path(path, lock_token)
954
+ staging_path = restore_staging_path(path, lock_token)
781
955
  _clear_path(staging_path)
782
956
  _checkout_with_fallback(path, cache_path, checkout_modes, staging_path=staging_path)
783
957
  return True
@@ -820,6 +994,10 @@ def _restore_directory_from_cache(
820
994
  restore_prefix = f"{_RESTORE_TEMP_PREFIX}{target_token}."
821
995
 
822
996
  if symlink_fast_path:
997
+ # Same repair as _store_directory: a tree an older pivot built keeps its
998
+ # 0o444-everywhere files forever otherwise, since the tree hash already
999
+ # incorporates isexec and this path never rebuilds an existing tree directory.
1000
+ _repair_tree_dir_isexec(cache_dir_path, output_hash["manifest"])
823
1001
  temp_symlink = path.parent / f"{restore_prefix}symlink_{secrets.token_hex(4)}"
824
1002
  try:
825
1003
  temp_symlink.symlink_to(cache_dir_path.resolve())
@@ -849,7 +1027,11 @@ def _restore_directory_from_cache(
849
1027
  )
850
1028
  file_path.parent.mkdir(parents=True, exist_ok=True)
851
1029
  _checkout_with_fallback(
852
- file_path, file_cache_path, checkout_modes, executable=entry["isexec"]
1030
+ file_path,
1031
+ file_cache_path,
1032
+ checkout_modes,
1033
+ executable=entry["isexec"],
1034
+ display_path=path / entry["relpath"],
853
1035
  )
854
1036
  if file_count % 100 == 0:
855
1037
  try:
@@ -237,12 +237,18 @@ def _verify_content_hash(content: bytes, expected_hash: str) -> bool:
237
237
  return actual_hash == expected_hash
238
238
 
239
239
 
240
- def _write_fallback_content(dest_path: pathlib.Path, content: bytes, force: bool) -> None:
241
- """Replace a cache-miss target while holding the same lock as cache restores."""
242
- with cache.restore_lock(dest_path, config.get_state_dir()):
243
- if (force or dest_path.is_symlink()) and (dest_path.exists() or dest_path.is_symlink()):
244
- cache.remove_output(dest_path)
245
- dest_path.write_bytes(content)
240
+ def _write_fallback_content(dest_path: pathlib.Path, content: bytes) -> None:
241
+ """Atomically replace a cache-miss target under the cache restore lock."""
242
+
243
+ def write_content(fd: int) -> None:
244
+ with os.fdopen(fd, "wb") as file:
245
+ file.write(content)
246
+
247
+ with cache.restore_lock(dest_path, config.get_state_dir()) as lock_token:
248
+ staging_path = (
249
+ cache.restore_staging_path(dest_path, lock_token) if lock_token is not None else None
250
+ )
251
+ cache.atomic_write_file(dest_path, write_content, staging_path=staging_path)
246
252
 
247
253
 
248
254
  def restore_file(
@@ -299,7 +305,7 @@ def restore_file(
299
305
  content = git.read_file_from_revision(rel_path, rev)
300
306
  if content is not None:
301
307
  try:
302
- _write_fallback_content(dest_path, content, force)
308
+ _write_fallback_content(dest_path, content)
303
309
  return RestoreResult(
304
310
  status=RestoreStatus.RESTORED,
305
311
  path=path_str,
@@ -328,7 +334,7 @@ def restore_file(
328
334
  ),
329
335
  )
330
336
  try:
331
- _write_fallback_content(dest_path, remote_content, force)
337
+ _write_fallback_content(dest_path, remote_content)
332
338
  except OSError as e:
333
339
  return RestoreResult(
334
340
  status=RestoreStatus.ERROR,
@@ -165,6 +165,11 @@ class WorkspaceStore:
165
165
  self._cache_dir = cache_dir.resolve() if cache_dir is not None else None
166
166
  self._checkout_modes = checkout_modes
167
167
 
168
+ @property
169
+ def cache_dir(self) -> pathlib.Path | None:
170
+ """The cache object root (``<cache.dir>/files``), or None when this store has no cache."""
171
+ return self._cache_dir
172
+
168
173
  def _resolve_input_path(self, ref: types.ArtifactRef) -> pathlib.Path:
169
174
  name = ref.identity.producer
170
175
  if name in self._input_bindings:
@@ -216,11 +221,21 @@ class WorkspaceStore:
216
221
  path: pathlib.Path,
217
222
  *,
218
223
  state_dir: pathlib.Path | None = None,
224
+ warn_left_out_symlinks: bool = True,
219
225
  ) -> types.HashInfo:
220
- """Hash a written output once and, unless it is a metric, place it in the cache."""
226
+ """Hash a written output once and, unless it is a metric, place it in the cache.
227
+
228
+ ``warn_left_out_symlinks`` logs the warning naming the symlinks a directory output
229
+ leaves out; a quiet command turns it off.
230
+ """
221
231
  output_hash: types.HashInfo
222
232
  if path.is_dir():
223
- tree_hash, manifest = cache.hash_directory(path, None)
233
+ tree_hash, manifest = cache.hash_directory(
234
+ path,
235
+ None,
236
+ cache_dir=self._cache_dir,
237
+ warn_left_out_symlinks=warn_left_out_symlinks,
238
+ )
224
239
  output_hash = types.DirHash(hash=tree_hash, manifest=manifest)
225
240
  else:
226
241
  file_hash, _ = cache.hash_file(path, None)
@@ -261,7 +276,7 @@ class WorkspaceStore:
261
276
  ) -> types.HashInfo:
262
277
  path = self._resolve_path(ref)
263
278
  if ref.tag == types.ArtifactTag.DIRECTORY or path.is_dir():
264
- tree_hash, manifest = cache.hash_directory(path, state_db)
279
+ tree_hash, manifest = cache.hash_directory(path, state_db, cache_dir=self._cache_dir)
265
280
  return types.DirHash(hash=tree_hash, manifest=manifest)
266
281
  file_hash, _ = cache.hash_file(path, state_db)
267
282
  return types.FileHash(hash=file_hash)
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes