pivot 0.2.4__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. {pivot-0.2.4 → pivot-0.3.1}/PKG-INFO +1 -1
  2. {pivot-0.2.4 → pivot-0.3.1}/pyproject.toml +1 -1
  3. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/checkout.py +4 -10
  4. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/console.py +6 -2
  5. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/doctor.py +32 -2
  6. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/engine.py +4 -2
  7. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/watch.py +78 -7
  8. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/fingerprint.py +43 -11
  9. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/skip.py +1 -1
  10. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/cache.py +16 -128
  11. {pivot-0.2.4 → pivot-0.3.1}/LICENSE +0 -0
  12. {pivot-0.2.4 → pivot-0.3.1}/README.md +0 -0
  13. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/__init__.py +0 -0
  14. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/ast_utils.py +0 -0
  15. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/AGENTS.md +0 -0
  16. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/CLAUDE.md +0 -0
  17. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/__init__.py +0 -0
  18. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/_run_common.py +0 -0
  19. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/check_ignore.py +0 -0
  20. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/commit.py +0 -0
  21. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/completion.py +0 -0
  22. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/config.py +0 -0
  23. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/dag.py +0 -0
  24. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/data.py +0 -0
  25. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/decorators.py +0 -0
  26. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/errors.py +0 -0
  27. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/fingerprint.py +0 -0
  28. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/helpers.py +0 -0
  29. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/history.py +0 -0
  30. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/import_cmd.py +0 -0
  31. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/init.py +0 -0
  32. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/list.py +0 -0
  33. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/metrics.py +0 -0
  34. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/params.py +0 -0
  35. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/plots.py +0 -0
  36. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/remote.py +0 -0
  37. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/repro.py +0 -0
  38. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/run.py +0 -0
  39. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/status.py +0 -0
  40. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/targets.py +0 -0
  41. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/track.py +0 -0
  42. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/update.py +0 -0
  43. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/verify.py +0 -0
  44. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/compose.py +0 -0
  45. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/config/__init__.py +0 -0
  46. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/config/io.py +0 -0
  47. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/config/models.py +0 -0
  48. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/dag/__init__.py +0 -0
  49. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/dag/render.py +0 -0
  50. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/decorators.py +0 -0
  51. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/discovery.py +0 -0
  52. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/__init__.py +0 -0
  53. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/agent_rpc.py +0 -0
  54. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/graph.py +0 -0
  55. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/scheduler.py +0 -0
  56. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/sinks.py +0 -0
  57. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/sources.py +0 -0
  58. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/types.py +0 -0
  59. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/worker_pool.py +0 -0
  60. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/exceptions.py +0 -0
  61. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/AGENTS.md +0 -0
  62. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/CLAUDE.md +0 -0
  63. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/__init__.py +0 -0
  64. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/commit.py +0 -0
  65. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/core.py +0 -0
  66. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/worker.py +0 -0
  67. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/explain.py +0 -0
  68. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/git.py +0 -0
  69. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/ignore.py +0 -0
  70. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/import_artifact.py +0 -0
  71. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/loaders.py +0 -0
  72. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/metrics.py +0 -0
  73. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/names.py +0 -0
  74. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/outputs.py +0 -0
  75. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/parameters.py +0 -0
  76. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/path_policy.py +0 -0
  77. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/project.py +0 -0
  78. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/py.typed +0 -0
  79. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/registry.py +0 -0
  80. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/AGENTS.md +0 -0
  81. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/__init__.py +0 -0
  82. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/config.py +0 -0
  83. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/git_archive.py +0 -0
  84. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/github.py +0 -0
  85. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/storage.py +0 -0
  86. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/sync.py +0 -0
  87. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/run_history.py +0 -0
  88. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/__init__.py +0 -0
  89. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/common.py +0 -0
  90. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/data.py +0 -0
  91. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/metrics.py +0 -0
  92. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/params.py +0 -0
  93. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/plots.py +0 -0
  94. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/stage_def.py +0 -0
  95. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/status.py +0 -0
  96. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/AGENTS.md +0 -0
  97. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/CLAUDE.md +0 -0
  98. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/__init__.py +0 -0
  99. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/artifact_lock.py +0 -0
  100. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/lock.py +0 -0
  101. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/presentation.py +0 -0
  102. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/restore.py +0 -0
  103. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/state.py +0 -0
  104. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/store.py +0 -0
  105. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/track.py +0 -0
  106. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/types.py +0 -0
  107. {pivot-0.2.4 → pivot-0.3.1}/src/pivot/yaml_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pivot
3
- Version: 0.2.4
3
+ Version: 0.3.1
4
4
  Summary: High-performance Python pipeline tool with automatic code change detection
5
5
  Author: Sami Jawhar
6
6
  Author-email: Sami Jawhar <sami@metr.org>
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "pivot"
3
- version = "0.2.4"
3
+ version = "0.3.1"
4
4
  description = "High-performance Python pipeline tool with automatic code change detection"
5
5
  requires-python = ">=3.13,<3.14"
6
6
  readme = "README.md"
@@ -116,7 +116,6 @@ def _restore_path_locked(
116
116
  lock_token: str | None,
117
117
  ) -> tuple[RestoreResult, str]:
118
118
  """Body of _restore_path_sync; the caller holds the restore lock for path."""
119
- keep_ignored = False
120
119
  if path.exists():
121
120
  match behavior:
122
121
  case CheckoutBehavior.ERROR:
@@ -127,10 +126,9 @@ def _restore_path_locked(
127
126
  + "Use --force to overwrite or --only-missing to skip existing files."
128
127
  )
129
128
  case CheckoutBehavior.SKIP_EXISTING:
130
- # Existing file: skip. Existing real directory: fill in its missing files, or
131
- # swap in the target when every differing file is pivot's own cached output,
132
- # keeping the entries the directory hash skips. Never overwrite content the
133
- # cache lacks (a local change).
129
+ # Existing real directories can fill missing files in place, or swap to the
130
+ # target when every differing or extra file is pivot's own cached output.
131
+ # Never overwrite content the cache lacks (a local change).
134
132
  if not is_dir_hash(output_hash) or path.is_symlink() or not path.is_dir():
135
133
  return ("skipped", path.name)
136
134
  # Reuse hashes `pivot status` and `pivot track` recorded; never create the DB.
@@ -147,13 +145,10 @@ def _restore_path_locked(
147
145
  f"'{path.name}' has local changes that are not in the cache: "
148
146
  + f"{', '.join(outcome['local_changes'])}. Use --force to overwrite."
149
147
  )
150
- # An interrupted restore may have staged kept entries beside the directory;
151
- # the restore below puts them back, so take it even when nothing else differs.
152
- if not outcome["replace"] and cache.staged_kept_entries(path) is None:
148
+ if not outcome["replace"]:
153
149
  if outcome["unavailable"]:
154
150
  return ("missing", path.name)
155
151
  return ("restored" if outcome["restored"] else "skipped", path.name)
156
- keep_ignored = True
157
152
  case CheckoutBehavior.FORCE:
158
153
  cache.remove_output(path)
159
154
  case _: # pyright: ignore[reportUnnecessaryComparison] - defensive for future enum values
@@ -166,7 +161,6 @@ def _restore_path_locked(
166
161
  checkout_modes=checkout_modes,
167
162
  state_dir=state_dir,
168
163
  lock_token=lock_token,
169
- keep_ignored=keep_ignored,
170
164
  )
171
165
  if not success:
172
166
  return ("missing", path.name)
@@ -117,8 +117,12 @@ class Console:
117
117
  self._echo(f" {key}")
118
118
 
119
119
  if change_type == "modified":
120
- self._echo(f" Old: {self._color(str(old_val) if old_val else 'N/A', 'red')}")
121
- self._echo(f" New: {self._color(str(new_val) if new_val else 'N/A', 'green')}")
120
+ self._echo(
121
+ f" Old: {self._color(str(old_val) if old_val is not None else 'N/A', 'red')}"
122
+ )
123
+ self._echo(
124
+ f" New: {self._color(str(new_val) if new_val is not None else 'N/A', 'green')}"
125
+ )
122
126
  elif change_type == "added":
123
127
  self._echo(f" {self._color('(added)', 'green')} {new_val}")
124
128
  else:
@@ -4,10 +4,11 @@ import enum
4
4
  import os
5
5
  import subprocess
6
6
  import sys
7
- from typing import TYPE_CHECKING, Literal, TypedDict
7
+ from typing import TYPE_CHECKING, Literal, TypedDict, cast
8
8
  from urllib.parse import urlparse, urlunparse
9
9
 
10
10
  import click
11
+ import pydantic
11
12
 
12
13
  from pivot.cli import decorators as cli_decorators
13
14
  from pivot.cli import helpers as cli_helpers
@@ -87,6 +88,32 @@ def _sanitize_url(url: str) -> str:
87
88
  return urlunparse(parsed._replace(netloc=netloc))
88
89
 
89
90
 
91
+ def _find_invalid_remote(
92
+ remotes: dict[object, object],
93
+ ) -> tuple[object, object, str]:
94
+ """Return the first remote entry that fails the configuration validator."""
95
+ from pivot.config import models
96
+
97
+ for name, remote in remotes.items():
98
+ try:
99
+ models.PivotConfig.model_validate({"remotes": {name: remote}})
100
+ except pydantic.ValidationError as error:
101
+ detail = error.errors()[0]
102
+ return name, remote, detail["msg"].removeprefix("Value error, ")
103
+ raise AssertionError("invalid remotes mapping contains no invalid entry")
104
+
105
+
106
+ def _format_config_validation_error(error: pydantic.ValidationError) -> str:
107
+ """Format a config validation error for the doctor command's user-facing boundary."""
108
+ detail = error.errors()[0]
109
+ location = ".".join(str(part) for part in detail["loc"])
110
+ value = detail["input"]
111
+ if location == "remotes" and isinstance(value, dict) and value:
112
+ name, remote, message = _find_invalid_remote(cast("dict[object, object]", value))
113
+ return f"Invalid config remotes.{name}: {remote!r}; {message}"
114
+ return f"Invalid config {location}: {value!r}; expected {detail['msg'].removeprefix('Value error, ')}"
115
+
116
+
90
117
  def _relative_path(path: pathlib.Path, project_root: pathlib.Path) -> str:
91
118
  """Return path relative to project root, or absolute if outside."""
92
119
  try:
@@ -316,7 +343,10 @@ def doctor(ctx: click.Context, as_json: bool, check_remote: bool) -> None:
316
343
  checks.append(project_check)
317
344
 
318
345
  checks.append(_check_pipeline_config(project_root))
319
- checks.append(_check_cache_directory(project_root))
346
+ try:
347
+ checks.append(_check_cache_directory(project_root))
348
+ except pydantic.ValidationError as e:
349
+ raise click.ClickException(_format_config_validation_error(e)) from e
320
350
  checks.append(_check_git_repository(project_root))
321
351
 
322
352
  if check_remote:
@@ -1787,9 +1787,11 @@ class Engine:
1787
1787
  return lock_service.acquire_many(lock_requests)
1788
1788
 
1789
1789
  lock_handle = await anyio.to_thread.run_sync(_acquire_artifact_locks)
1790
- await self._set_stage_state(stage_name, StageExecutionState.PREPARING)
1791
-
1792
1790
  try:
1791
+ lock_data = await anyio.to_thread.run_sync(_read_lock)
1792
+ if lock_data is None:
1793
+ return False
1794
+ await self._set_stage_state(stage_name, StageExecutionState.PREPARING)
1793
1795
  return await self._try_skip_with_locks(
1794
1796
  stage_name,
1795
1797
  stage_info,
@@ -84,6 +84,8 @@ class WatchCoordinator:
84
84
 
85
85
  _graph: nx.DiGraph[str]
86
86
  _artifact_paths: dict[pathlib.Path, pivot_types.ArtifactIdentity]
87
+ _resolved_output_paths: dict[tuple[pathlib.Path, str], list[pivot_types.ArtifactIdentity]]
88
+ _resolved_input_paths: dict[pathlib.Path, list[pivot_types.ArtifactIdentity]]
87
89
 
88
90
  def __init__(
89
91
  self,
@@ -92,6 +94,29 @@ class WatchCoordinator:
92
94
  ) -> None:
93
95
  self._graph = graph
94
96
  self._artifact_paths = artifact_paths
97
+ self._rebuild_resolved_path_indexes()
98
+
99
+ def _rebuild_resolved_path_indexes(self) -> None:
100
+ """Rebuild the input and output aliases used for physical watch paths."""
101
+ self._resolved_input_paths = dict[pathlib.Path, list[pivot_types.ArtifactIdentity]]()
102
+ self._resolved_output_paths = dict[
103
+ tuple[pathlib.Path, str], list[pivot_types.ArtifactIdentity]
104
+ ]()
105
+ for artifact_path, identity in self._artifact_paths.items():
106
+ if engine_graph.get_producer(self._graph, identity) is None:
107
+ self._resolved_input_paths.setdefault(
108
+ artifact_path.resolve(), list[pivot_types.ArtifactIdentity]()
109
+ ).append(identity)
110
+ continue
111
+ key = (artifact_path.parent.resolve(), artifact_path.name)
112
+ self._resolved_output_paths.setdefault(
113
+ key, list[pivot_types.ArtifactIdentity]()
114
+ ).append(identity)
115
+
116
+ def _refresh_resolved_path_indexes_for(self, path: pathlib.Path) -> None:
117
+ """Refresh aliases after a symlink or watched directory changes."""
118
+ if path.is_symlink() or path.is_dir():
119
+ self._rebuild_resolved_path_indexes()
95
120
 
96
121
  @property
97
122
  def graph(self) -> nx.DiGraph[str]:
@@ -104,6 +129,10 @@ class WatchCoordinator:
104
129
 
105
130
  def resolve_identity(self, path: pathlib.Path) -> pivot_types.ArtifactIdentity | None:
106
131
  """Resolve a filesystem event path to the matching artifact identity."""
132
+ self._refresh_resolved_path_indexes_for(path)
133
+ outputs = self._matching_resolved_output_identities(path)
134
+ if outputs:
135
+ return outputs[0]
107
136
  current = project.normalize_path(path)
108
137
  while True:
109
138
  identity = self._artifact_paths.get(current)
@@ -114,6 +143,38 @@ class WatchCoordinator:
114
143
  return None
115
144
  current = parent
116
145
 
146
+ def _matching_identities(self, path: pathlib.Path) -> list[pivot_types.ArtifactIdentity]:
147
+ """Return identities whose resolved paths contain a filesystem event path."""
148
+ current = project.normalize_path(path)
149
+ identities = list[pivot_types.ArtifactIdentity]()
150
+ while True:
151
+ identity = self._artifact_paths.get(current)
152
+ if identity is not None:
153
+ identities.append(identity)
154
+ parent = current.parent
155
+ if parent == current:
156
+ return identities
157
+ current = parent
158
+
159
+ def _matching_resolved_input_identities(
160
+ self, path: pathlib.Path
161
+ ) -> list[pivot_types.ArtifactIdentity]:
162
+ resolved_path = path.resolve()
163
+ identities = list[pivot_types.ArtifactIdentity]()
164
+ current = resolved_path
165
+ while True:
166
+ identities.extend(self._resolved_input_paths.get(current, []))
167
+ parent = current.parent
168
+ if parent == current:
169
+ return identities
170
+ current = parent
171
+
172
+ def _matching_resolved_output_identities(
173
+ self, path: pathlib.Path
174
+ ) -> list[pivot_types.ArtifactIdentity]:
175
+ """Return output identities by resolved parent directory and logical leaf name."""
176
+ return self._resolved_output_paths.get((path.parent.resolve(), path.name), [])
177
+
117
178
  def should_filter_path(
118
179
  self,
119
180
  path: pathlib.Path,
@@ -134,13 +195,23 @@ class WatchCoordinator:
134
195
  """Get all stages affected by the changed paths, including downstream."""
135
196
  affected = set[str]()
136
197
  for path in paths:
137
- identity = self.resolve_identity(path)
138
- if identity is None:
139
- continue
140
- consumers = engine_graph.get_consumers(self._graph, identity)
141
- affected.update(consumers)
142
- for stage in consumers:
143
- affected.update(engine_graph.get_downstream_stages(self._graph, stage))
198
+ self._refresh_resolved_path_indexes_for(path)
199
+ output_identities = set(self._matching_resolved_output_identities(path))
200
+ if output_identities:
201
+ identities = output_identities
202
+ else:
203
+ logical_identities = set(self._matching_identities(path))
204
+ identities = logical_identities.copy()
205
+ if all(
206
+ engine_graph.get_producer(self._graph, identity) is None
207
+ for identity in logical_identities
208
+ ):
209
+ identities.update(self._matching_resolved_input_identities(path))
210
+ for identity in identities:
211
+ consumers = engine_graph.get_consumers(self._graph, identity)
212
+ affected.update(consumers)
213
+ for stage in consumers:
214
+ affected.update(engine_graph.get_downstream_stages(self._graph, stage))
144
215
  return sorted(affected)
145
216
 
146
217
  def get_producer(self, path: pathlib.Path) -> str | None:
@@ -11,6 +11,7 @@ import logging
11
11
  import marshal
12
12
  import os
13
13
  import pathlib
14
+ import re
14
15
  import sys
15
16
  import textwrap
16
17
  import types
@@ -60,9 +61,12 @@ _PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
60
61
  # `ext:`), not `mod:`, and follows `settings.Config.SCALE` and `pkg.sub.helper`.
61
62
  # Version 25 records call-site artifact wiring (parameter bindings, collection kind/order,
62
63
  # and project-relative input paths) in the code manifest.
63
- _CACHE_SCHEMA_VERSION = 25
64
+ # Version 26 records tuples and frozensets of scalars and compiled regexes by value.
65
+ _CACHE_SCHEMA_VERSION = 26
64
66
  _REPR_SIZE_LIMIT = 10_000
65
67
 
68
+ _SCALAR_TYPES = (bool, int, float, str, bytes, type(None))
69
+
66
70
  _SITE_PACKAGE_PATHS = ("site-packages", "dist-packages")
67
71
 
68
72
  # Type alias for AST hash cache entry tuple.
@@ -563,6 +567,24 @@ class _ValueKind(enum.Enum):
563
567
  SCALAR = "scalar"
564
568
 
565
569
 
570
+ def _immutable_value_text(value: Any) -> str | None:
571
+ """Return the manifest text for a safely trackable immutable value, if any.
572
+
573
+ Scalars keep their existing repr. Exact tuples and frozensets are tracked only when every
574
+ member is an exact scalar type, one level deep. Compiled regexes record their full pattern
575
+ and flags because their repr truncates long patterns.
576
+ """
577
+ if isinstance(value, _SCALAR_TYPES):
578
+ return repr(value)
579
+ if type(value) in (tuple, frozenset) and all(
580
+ type(item) in _SCALAR_TYPES for item in cast("tuple[Any, ...] | frozenset[Any]", value)
581
+ ):
582
+ return _serialize_value_for_hash(value)
583
+ if isinstance(value, re.Pattern):
584
+ return _serialize_value_for_hash(value)
585
+ return None
586
+
587
+
566
588
  def _value_kind(value: Any) -> _ValueKind:
567
589
  """Classify a value a walked function reads; the walk and a cache hit share this."""
568
590
  # functools.partial fails is_user_code() (module is functools/stdlib).
@@ -576,7 +598,7 @@ def _value_kind(value: Any) -> _ValueKind:
576
598
  return _ValueKind.MODULE
577
599
  if isinstance(value, logging.Logger):
578
600
  return _ValueKind.LOGGER
579
- if isinstance(value, (bool, int, float, str, bytes, type(None))):
601
+ if _immutable_value_text(value) is not None:
580
602
  return _ValueKind.SCALAR
581
603
  return _ValueKind.REFUSED
582
604
 
@@ -598,7 +620,7 @@ def _value_token(value: Any) -> str:
598
620
  if kind is _ValueKind.MODULE:
599
621
  return f"module:{value.__name__}"
600
622
  if kind is _ValueKind.SCALAR:
601
- return f"{type(value).__name__}:{value!r}"
623
+ return f"{type(value).__name__}:{_immutable_value_text(value)}"
602
624
  if kind is _ValueKind.EXTERNAL:
603
625
  return f"external:{_library_name(value)}"
604
626
  return kind.value
@@ -1335,9 +1357,9 @@ def _check_non_scalar_capture(
1335
1357
  subject += f" as the default of parameter '{parameter}'"
1336
1358
  message = (
1337
1359
  f"Stage '{stage_name}': {subject} (type: {type(value).__name__}).\n"
1338
- + "Pivot tracks only module-level bool, int, float, str, bytes, and None values. "
1339
- + "Other values may change without a reliable fingerprint, which can cause silent wrong "
1340
- + "outputs.\n"
1360
+ + "Pivot tracks module-level bool, int, float, str, bytes, and None values, tuples and "
1361
+ + "frozensets of them, and compiled regexes. Other values may change without a reliable "
1362
+ + "fingerprint, which can cause silent wrong outputs.\n"
1341
1363
  + "Fix: move this value into the stage's StageParams, or read it from a pipeline input.\n"
1342
1364
  + "To suppress: set core.unsafe_fingerprinting=true or PIVOT_UNSAFE_FINGERPRINTING=1"
1343
1365
  )
@@ -1548,7 +1570,9 @@ def _process_captured_value(
1548
1570
  if chain is not None:
1549
1571
  _process_module_dependency(chain, value, func, manifest, visited)
1550
1572
  case _ValueKind.SCALAR:
1551
- manifest[f"const:{binding_ref}"] = repr(value)
1573
+ text = _immutable_value_text(value)
1574
+ assert text is not None
1575
+ manifest[f"const:{binding_ref}"] = text
1552
1576
  case _ValueKind.REFUSED:
1553
1577
  _check_non_scalar_capture(name, value, stage_name, parameter=parameter)
1554
1578
  case _ValueKind.EXTERNAL:
@@ -1681,7 +1705,7 @@ def _process_default_values(
1681
1705
  binding_ref = f"{func_ref}.{parameter}"
1682
1706
  step = f"d:{parameter}"
1683
1707
  _record_read(func, step)
1684
- if isinstance(value, (bool, int, float, str, bytes, type(None))):
1708
+ if isinstance(value, _SCALAR_TYPES):
1685
1709
  manifest[f"default:{binding_ref}"] = repr(value)
1686
1710
  continue
1687
1711
  if isinstance(value, stage_def.StageParams):
@@ -1695,6 +1719,10 @@ def _process_default_values(
1695
1719
  expression = expressions.get(parameter)
1696
1720
  if expression is not None and _is_literal(expression):
1697
1721
  continue
1722
+ text = _immutable_value_text(value)
1723
+ if text is not None:
1724
+ manifest[f"default:{binding_ref}"] = text
1725
+ continue
1698
1726
  source = ast.unparse(expression) if expression is not None else parameter
1699
1727
  with _walking(step):
1700
1728
  _process_captured_value(
@@ -2140,12 +2168,16 @@ def _canonicalize_value_for_hash(value: Any, active: tuple[int, ...] = ()) -> An
2140
2168
  canonical({field: value})]`, field by field, so a secret inside one reaches that leaf
2141
2169
  rather than the generated repr's mask. An object whose class keeps `object.__repr__`
2142
2170
  (which holds the address) and that has a `__dict__` becomes
2143
- `[module.qualname, canonical(vars(obj))]`.
2144
- Other leaves keep their `repr`, tagged with their type. A class is a leaf, even a pydantic
2145
- model class: only an instance is dumped.
2171
+ `[module.qualname, canonical(vars(obj))]`. A compiled regex becomes
2172
+ `["re.Pattern", canonical(pattern), flags]`, so its full pattern is recorded. Other leaves
2173
+ keep their `repr`, tagged with their type. A class is a leaf, even a pydantic model class:
2174
+ only an instance is dumped.
2146
2175
  """
2147
2176
  if value is None or isinstance(value, (bool, int, float, str)):
2148
2177
  return value
2178
+ if isinstance(value, re.Pattern):
2179
+ pattern = cast("re.Pattern[Any]", value)
2180
+ return ["re.Pattern", _canonicalize_value_for_hash(pattern.pattern), pattern.flags]
2149
2181
  if id(value) in active:
2150
2182
  return ["cycle", len(active) - active.index(id(value))]
2151
2183
  if hasattr(value, "model_dump") and not isinstance(value, type):
@@ -190,7 +190,7 @@ def params_equal(old: object, new: object) -> bool:
190
190
  (``parameters.get_effective_params``), so the text is the canonical encoding. Python
191
191
  equality alone would call a NaN param changed on every run, because nan != nan.
192
192
  """
193
- return old == new or json.dumps(old, sort_keys=True) == json.dumps(new, sort_keys=True)
193
+ return json.dumps(old, sort_keys=True) == json.dumps(new, sort_keys=True)
194
194
 
195
195
 
196
196
  def diff_dep_hashes(
@@ -36,7 +36,6 @@ XXHASH64_HEX_LENGTH = 16 # xxhash64 produces 64-bit hash = 16 hex characters
36
36
  _RESTORE_TEMP_PREFIX = ".pivot_restore_"
37
37
  _BACKUP_TEMP_PREFIX = ".pivot_backup_"
38
38
  _RESTORE_LOCK_PREFIX = ".pivot_restore_lock_"
39
- _KEEP_STAGING_PREFIX = ".pivot_keep_"
40
39
  _RESTORE_TEMP_MAX_AGE = 3600 # 1 hour
41
40
 
42
41
 
@@ -46,36 +45,6 @@ def _restore_target_token(target_path: pathlib.Path) -> str:
46
45
  return hashlib.sha256(str(logical_path).encode()).hexdigest()[:32]
47
46
 
48
47
 
49
- def _keep_staging_path(path: pathlib.Path) -> pathlib.Path:
50
- """Sibling that holds a directory's kept entries while it is replaced."""
51
- return path.parent / f"{_KEEP_STAGING_PREFIX}{_restore_target_token(path)}"
52
-
53
-
54
- def _move_kept_entries(source: pathlib.Path, destination: pathlib.Path) -> None:
55
- """Move every entry under source into destination, merging directories, then drop source."""
56
- with os.scandir(source) as entries:
57
- for entry in entries:
58
- target = destination / entry.name
59
- if not os.path.lexists(target):
60
- destination.mkdir(parents=True, exist_ok=True)
61
- pathlib.Path(entry.path).rename(target)
62
- elif (
63
- entry.is_dir(follow_symlinks=False) and target.is_dir() and not target.is_symlink()
64
- ):
65
- _move_kept_entries(pathlib.Path(entry.path), target)
66
- else:
67
- raise exceptions.CacheRestoreError(
68
- f"Cannot put back {entry.path}: {target}. Move the entry by hand, then retry."
69
- )
70
- source.rmdir()
71
-
72
-
73
- def staged_kept_entries(path: pathlib.Path) -> pathlib.Path | None:
74
- """Return the staging directory an interrupted keep-ignored restore left, if any."""
75
- keep_dir = _keep_staging_path(path)
76
- return keep_dir if keep_dir.exists() else None
77
-
78
-
79
48
  def _restore_staging_path(path: pathlib.Path, token: str | None = None) -> pathlib.Path:
80
49
  """Return the deterministic staging path for a file checkout."""
81
50
  return path.parent / f"{_RESTORE_TEMP_PREFIX}{token or _restore_target_token(path)}.file"
@@ -190,39 +159,12 @@ def hash_file(
190
159
  return file_hash, file_stat
191
160
 
192
161
 
193
- # Hardcoded ignore patterns for hot path - O(1) lookups only.
194
- # For full .pivotignore support, see pivot.ignore module.
195
- _IGNORE_DIRS: frozenset[str] = frozenset(
196
- {
197
- "__pycache__",
198
- ".venv",
199
- "venv",
200
- ".git",
201
- ".hg",
202
- ".svn",
203
- ".idea",
204
- ".vscode",
205
- "node_modules",
206
- ".pivot",
207
- "dist",
208
- "build",
209
- }
210
- )
211
- _IGNORE_SUFFIXES: tuple[str, ...] = (".pyc", ".pyo", ".swp", ".swo")
212
-
213
-
214
- def _should_skip_entry(entry: os.DirEntry[str]) -> bool:
215
- """Fast ignore check for hot path. O(1) lookups only."""
216
- name = entry.name
217
- if entry.is_dir(follow_symlinks=False):
218
- return name in _IGNORE_DIRS
219
- # File checks: suffix and special patterns
220
- return name.endswith(_IGNORE_SUFFIXES) or name.endswith("~") or name.startswith(".#")
221
-
222
-
223
162
  def _scandir_recursive(path: pathlib.Path) -> Generator[os.DirEntry[str]]:
224
- """Yield all files recursively using os.scandir() for efficiency.
163
+ """Yield every file under `path` recursively, using os.scandir() for efficiency.
225
164
 
165
+ A declared directory artifact is hashed and cached whole: no name is ignored, so a change
166
+ to any file in a directory input reruns its consumers, and every file in a directory output
167
+ survives cache and restore. Ignore rules belong to code fingerprinting and watch mode.
226
168
  DirEntry objects cache stat results, avoiding redundant syscalls.
227
169
  Symlinks are skipped to prevent loops.
228
170
  PermissionError propagates to caller - partial hashes would be incorrect.
@@ -234,8 +176,6 @@ def _scandir_recursive(path: pathlib.Path) -> Generator[os.DirEntry[str]]:
234
176
  for entry in entries:
235
177
  if entry.is_symlink():
236
178
  continue
237
- if _should_skip_entry(entry):
238
- continue
239
179
  if entry.is_file():
240
180
  yield entry
241
181
  elif entry.is_dir():
@@ -773,7 +713,6 @@ def restore_from_cache(
773
713
  state_dir: pathlib.Path | None = None,
774
714
  *,
775
715
  lock_token: str | None = None,
776
- keep_ignored: bool = False,
777
716
  ) -> bool:
778
717
  """Restore file or directory from cache. Returns True if successful.
779
718
 
@@ -785,8 +724,6 @@ def restore_from_cache(
785
724
  checkout_modes: Ordered list of link modes to try with fallback on failure.
786
725
  state_dir: State directory for restore locks and crash cleanup.
787
726
  lock_token: Target token yielded by an already-held restore_lock for path.
788
- keep_ignored: When replacing an existing directory, preserve the entries the directory
789
- hash skips in the restored tree.
790
727
  """
791
728
  effective_modes = _resolve_checkout_modes(checkout_mode, checkout_modes)
792
729
  if lock_token is not None and state_dir is None:
@@ -800,7 +737,6 @@ def restore_from_cache(
800
737
  effective_modes,
801
738
  state_dir,
802
739
  lock_token=lock_token,
803
- keep_ignored=keep_ignored,
804
740
  )
805
741
  return _restore_file_from_cache(
806
742
  path, output_hash, cache_dir, effective_modes, state_dir, lock_token=lock_token
@@ -855,9 +791,8 @@ def _restore_directory_from_cache(
855
791
  state_dir: pathlib.Path | None = None,
856
792
  *,
857
793
  lock_token: str | None = None,
858
- keep_ignored: bool = False,
859
794
  ) -> bool:
860
- """Restore a directory atomically, optionally carrying ignored local entries forward."""
795
+ """Restore a directory atomically."""
861
796
  cache_dir_path = get_cache_path(cache_dir, output_hash["hash"])
862
797
  symlink_fast_path = checkout_modes[0] == CheckoutMode.SYMLINK and cache_dir_path.exists()
863
798
  if not symlink_fast_path:
@@ -877,24 +812,12 @@ def _restore_directory_from_cache(
877
812
  checkout_modes,
878
813
  state_dir,
879
814
  lock_token=held_token,
880
- keep_ignored=keep_ignored,
881
815
  )
882
816
 
883
817
  target_token = lock_token or _restore_target_token(path)
884
818
  if lock_token is not None:
885
819
  _cleanup_fresh_restore_orphans(path, target_token)
886
820
  restore_prefix = f"{_RESTORE_TEMP_PREFIX}{target_token}."
887
- keep_dir = _keep_staging_path(path)
888
- keep = list[str]()
889
- if (keep_ignored or keep_dir.exists()) and path.is_dir() and not path.is_symlink():
890
- if keep_dir.exists():
891
- _move_kept_entries(keep_dir, path)
892
- _, keep = _scan_entries(path, {entry["relpath"] for entry in output_hash["manifest"]})
893
- if symlink_fast_path and keep:
894
- symlink_fast_path = False
895
- for entry in output_hash["manifest"]:
896
- if not get_cache_path(cache_dir, entry["hash"]).exists():
897
- return False
898
821
 
899
822
  if symlink_fast_path:
900
823
  temp_symlink = path.parent / f"{restore_prefix}symlink_{secrets.token_hex(4)}"
@@ -948,14 +871,6 @@ def _restore_directory_from_cache(
948
871
  except OSError as e:
949
872
  logger.debug(f"chmod failed (may be NFS with root_squash): {e}")
950
873
 
951
- for relpath in keep:
952
- source = path / relpath
953
- if not os.path.lexists(source):
954
- continue
955
- staged = keep_dir / relpath
956
- staged.parent.mkdir(parents=True, exist_ok=True)
957
- source.rename(staged)
958
-
959
874
  try:
960
875
  if path.exists() or path.is_symlink():
961
876
  path.rename(backup_path)
@@ -964,8 +879,6 @@ def _restore_directory_from_cache(
964
879
 
965
880
  temp_path.replace(path)
966
881
  swap_completed = True
967
- if keep_dir.exists():
968
- _move_kept_entries(keep_dir, path)
969
882
 
970
883
  try:
971
884
  _clear_path(backup_path)
@@ -979,14 +892,6 @@ def _restore_directory_from_cache(
979
892
  except OSError as e:
980
893
  logger.debug(f"Failed to restore backup: {e}")
981
894
  backup_restore_failed = True
982
- if not swap_completed and keep_dir.exists() and path.is_dir() and not path.is_symlink():
983
- try:
984
- _move_kept_entries(keep_dir, path)
985
- except (OSError, exceptions.CacheRestoreError) as e:
986
- logger.warning(
987
- f"Entries kept from {path} remain in {keep_dir} ({e}); "
988
- + "the next restore of it puts them back"
989
- )
990
895
  raise
991
896
  finally:
992
897
  if temp_path.exists():
@@ -1017,39 +922,25 @@ def _is_cached(cache_dir: pathlib.Path, content_hash: str | None) -> bool:
1017
922
  )
1018
923
 
1019
924
 
1020
- def _scan_entries(path: pathlib.Path, expected_paths: set[str]) -> tuple[list[str], list[str]]:
1021
- """Return (files and symlinks, entries the directory hash skips) under path, as relpaths.
925
+ def _scan_entries(path: pathlib.Path) -> list[str]:
926
+ """Return every file and symlink under ``path`` as relative paths.
1022
927
 
1023
- Symlinked directories are not followed. A skipped name that is, or holds, a manifest
1024
- path is part of the output and is scanned like any other entry.
928
+ Symlinked directories are not followed.
1025
929
  """
1026
- expected_dirs = {
1027
- str(parent)
1028
- for relpath in expected_paths
1029
- for parent in pathlib.PurePosixPath(relpath).parents
1030
- if str(parent) != "."
1031
- }
1032
930
  files = list[str]()
1033
- ignored = list[str]()
1034
931
 
1035
932
  def _walk(directory: pathlib.Path) -> None:
1036
933
  with os.scandir(directory) as entries:
1037
934
  for entry in entries:
1038
935
  entry_path = pathlib.Path(entry.path)
1039
936
  relpath = str(entry_path.relative_to(path))
1040
- if (
1041
- _should_skip_entry(entry)
1042
- and relpath not in expected_paths
1043
- and relpath not in expected_dirs
1044
- ):
1045
- ignored.append(relpath)
1046
- elif entry.is_symlink() or entry.is_file():
937
+ if entry.is_symlink() or entry.is_file():
1047
938
  files.append(relpath)
1048
939
  elif entry.is_dir():
1049
940
  _walk(entry_path)
1050
941
 
1051
942
  _walk(path)
1052
- return files, ignored
943
+ return files
1053
944
 
1054
945
 
1055
946
  def _is_same_file(file_stat: os.stat_result, cache_path: pathlib.Path) -> bool:
@@ -1069,15 +960,12 @@ def restore_missing_in_directory(
1069
960
  checkout_modes: list[CheckoutMode],
1070
961
  state_db: state_mod.StateDB | None = None,
1071
962
  ) -> MissingRestore:
1072
- """Restore a directory's missing target files without overwriting local changes.
963
+ """Restore missing target files without overwriting local changes.
1073
964
 
1074
- When every existing file matches the target, the missing files are restored in place
1075
- and nothing else is touched. When some file differs from the target or is not in it,
1076
- only a whole-directory swap brings the target. It is safe only if every file it
1077
- removes is in the cache (pivot's own earlier output): then nothing is written and
1078
- ``replace`` asks the caller to swap with ``restore_from_cache(..., keep_ignored=True)``,
1079
- which carries over the entries the directory hash skips (build/, *.swp, ...). Otherwise
1080
- nothing is written, and the files the cache lacks (local edits, extra files) are returned.
965
+ When every existing file matches the target, missing files are restored in place. When a
966
+ file differs from the target or an extra file exists, only a whole-directory swap can
967
+ restore the target. The swap is safe only when every file it removes is in the cache;
968
+ otherwise the local changes are returned without writing anything.
1081
969
 
1082
970
  A retained file is read only when nothing else proves its content: a symlink to the
1083
971
  target's cache path, a hardlink to the target's cache blob, or a matching ``state_db``
@@ -1128,7 +1016,7 @@ def restore_missing_in_directory(
1128
1016
  )
1129
1017
  missing.append((entry, file_path, file_cache_path))
1130
1018
 
1131
- files, _ = _scan_entries(path, expected_paths)
1019
+ files = _scan_entries(path)
1132
1020
  for relpath in files:
1133
1021
  if relpath in expected_paths:
1134
1022
  continue
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes