pivot 0.2.4__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pivot-0.2.4 → pivot-0.3.1}/PKG-INFO +1 -1
- {pivot-0.2.4 → pivot-0.3.1}/pyproject.toml +1 -1
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/checkout.py +4 -10
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/console.py +6 -2
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/doctor.py +32 -2
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/engine.py +4 -2
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/watch.py +78 -7
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/fingerprint.py +43 -11
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/skip.py +1 -1
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/cache.py +16 -128
- {pivot-0.2.4 → pivot-0.3.1}/LICENSE +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/README.md +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/__init__.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/ast_utils.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/AGENTS.md +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/CLAUDE.md +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/__init__.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/_run_common.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/check_ignore.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/commit.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/completion.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/config.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/dag.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/data.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/decorators.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/errors.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/fingerprint.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/helpers.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/history.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/import_cmd.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/init.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/list.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/metrics.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/params.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/plots.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/remote.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/repro.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/run.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/status.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/targets.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/track.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/update.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/cli/verify.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/compose.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/config/__init__.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/config/io.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/config/models.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/dag/__init__.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/dag/render.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/decorators.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/discovery.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/__init__.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/agent_rpc.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/graph.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/scheduler.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/sinks.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/sources.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/types.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/engine/worker_pool.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/exceptions.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/AGENTS.md +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/CLAUDE.md +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/__init__.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/commit.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/core.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/executor/worker.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/explain.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/git.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/ignore.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/import_artifact.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/loaders.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/metrics.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/names.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/outputs.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/parameters.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/path_policy.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/project.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/py.typed +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/registry.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/AGENTS.md +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/__init__.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/config.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/git_archive.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/github.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/storage.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/remote/sync.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/run_history.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/__init__.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/common.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/data.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/metrics.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/params.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/show/plots.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/stage_def.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/status.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/AGENTS.md +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/CLAUDE.md +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/__init__.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/artifact_lock.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/lock.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/presentation.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/restore.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/state.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/store.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/storage/track.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/types.py +0 -0
- {pivot-0.2.4 → pivot-0.3.1}/src/pivot/yaml_config.py +0 -0
|
@@ -116,7 +116,6 @@ def _restore_path_locked(
|
|
|
116
116
|
lock_token: str | None,
|
|
117
117
|
) -> tuple[RestoreResult, str]:
|
|
118
118
|
"""Body of _restore_path_sync; the caller holds the restore lock for path."""
|
|
119
|
-
keep_ignored = False
|
|
120
119
|
if path.exists():
|
|
121
120
|
match behavior:
|
|
122
121
|
case CheckoutBehavior.ERROR:
|
|
@@ -127,10 +126,9 @@ def _restore_path_locked(
|
|
|
127
126
|
+ "Use --force to overwrite or --only-missing to skip existing files."
|
|
128
127
|
)
|
|
129
128
|
case CheckoutBehavior.SKIP_EXISTING:
|
|
130
|
-
# Existing
|
|
131
|
-
#
|
|
132
|
-
#
|
|
133
|
-
# cache lacks (a local change).
|
|
129
|
+
# Existing real directories can fill missing files in place, or swap to the
|
|
130
|
+
# target when every differing or extra file is pivot's own cached output.
|
|
131
|
+
# Never overwrite content the cache lacks (a local change).
|
|
134
132
|
if not is_dir_hash(output_hash) or path.is_symlink() or not path.is_dir():
|
|
135
133
|
return ("skipped", path.name)
|
|
136
134
|
# Reuse hashes `pivot status` and `pivot track` recorded; never create the DB.
|
|
@@ -147,13 +145,10 @@ def _restore_path_locked(
|
|
|
147
145
|
f"'{path.name}' has local changes that are not in the cache: "
|
|
148
146
|
+ f"{', '.join(outcome['local_changes'])}. Use --force to overwrite."
|
|
149
147
|
)
|
|
150
|
-
|
|
151
|
-
# the restore below puts them back, so take it even when nothing else differs.
|
|
152
|
-
if not outcome["replace"] and cache.staged_kept_entries(path) is None:
|
|
148
|
+
if not outcome["replace"]:
|
|
153
149
|
if outcome["unavailable"]:
|
|
154
150
|
return ("missing", path.name)
|
|
155
151
|
return ("restored" if outcome["restored"] else "skipped", path.name)
|
|
156
|
-
keep_ignored = True
|
|
157
152
|
case CheckoutBehavior.FORCE:
|
|
158
153
|
cache.remove_output(path)
|
|
159
154
|
case _: # pyright: ignore[reportUnnecessaryComparison] - defensive for future enum values
|
|
@@ -166,7 +161,6 @@ def _restore_path_locked(
|
|
|
166
161
|
checkout_modes=checkout_modes,
|
|
167
162
|
state_dir=state_dir,
|
|
168
163
|
lock_token=lock_token,
|
|
169
|
-
keep_ignored=keep_ignored,
|
|
170
164
|
)
|
|
171
165
|
if not success:
|
|
172
166
|
return ("missing", path.name)
|
|
@@ -117,8 +117,12 @@ class Console:
|
|
|
117
117
|
self._echo(f" {key}")
|
|
118
118
|
|
|
119
119
|
if change_type == "modified":
|
|
120
|
-
self._echo(
|
|
121
|
-
|
|
120
|
+
self._echo(
|
|
121
|
+
f" Old: {self._color(str(old_val) if old_val is not None else 'N/A', 'red')}"
|
|
122
|
+
)
|
|
123
|
+
self._echo(
|
|
124
|
+
f" New: {self._color(str(new_val) if new_val is not None else 'N/A', 'green')}"
|
|
125
|
+
)
|
|
122
126
|
elif change_type == "added":
|
|
123
127
|
self._echo(f" {self._color('(added)', 'green')} {new_val}")
|
|
124
128
|
else:
|
|
@@ -4,10 +4,11 @@ import enum
|
|
|
4
4
|
import os
|
|
5
5
|
import subprocess
|
|
6
6
|
import sys
|
|
7
|
-
from typing import TYPE_CHECKING, Literal, TypedDict
|
|
7
|
+
from typing import TYPE_CHECKING, Literal, TypedDict, cast
|
|
8
8
|
from urllib.parse import urlparse, urlunparse
|
|
9
9
|
|
|
10
10
|
import click
|
|
11
|
+
import pydantic
|
|
11
12
|
|
|
12
13
|
from pivot.cli import decorators as cli_decorators
|
|
13
14
|
from pivot.cli import helpers as cli_helpers
|
|
@@ -87,6 +88,32 @@ def _sanitize_url(url: str) -> str:
|
|
|
87
88
|
return urlunparse(parsed._replace(netloc=netloc))
|
|
88
89
|
|
|
89
90
|
|
|
91
|
+
def _find_invalid_remote(
|
|
92
|
+
remotes: dict[object, object],
|
|
93
|
+
) -> tuple[object, object, str]:
|
|
94
|
+
"""Return the first remote entry that fails the configuration validator."""
|
|
95
|
+
from pivot.config import models
|
|
96
|
+
|
|
97
|
+
for name, remote in remotes.items():
|
|
98
|
+
try:
|
|
99
|
+
models.PivotConfig.model_validate({"remotes": {name: remote}})
|
|
100
|
+
except pydantic.ValidationError as error:
|
|
101
|
+
detail = error.errors()[0]
|
|
102
|
+
return name, remote, detail["msg"].removeprefix("Value error, ")
|
|
103
|
+
raise AssertionError("invalid remotes mapping contains no invalid entry")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _format_config_validation_error(error: pydantic.ValidationError) -> str:
|
|
107
|
+
"""Format a config validation error for the doctor command's user-facing boundary."""
|
|
108
|
+
detail = error.errors()[0]
|
|
109
|
+
location = ".".join(str(part) for part in detail["loc"])
|
|
110
|
+
value = detail["input"]
|
|
111
|
+
if location == "remotes" and isinstance(value, dict) and value:
|
|
112
|
+
name, remote, message = _find_invalid_remote(cast("dict[object, object]", value))
|
|
113
|
+
return f"Invalid config remotes.{name}: {remote!r}; {message}"
|
|
114
|
+
return f"Invalid config {location}: {value!r}; expected {detail['msg'].removeprefix('Value error, ')}"
|
|
115
|
+
|
|
116
|
+
|
|
90
117
|
def _relative_path(path: pathlib.Path, project_root: pathlib.Path) -> str:
|
|
91
118
|
"""Return path relative to project root, or absolute if outside."""
|
|
92
119
|
try:
|
|
@@ -316,7 +343,10 @@ def doctor(ctx: click.Context, as_json: bool, check_remote: bool) -> None:
|
|
|
316
343
|
checks.append(project_check)
|
|
317
344
|
|
|
318
345
|
checks.append(_check_pipeline_config(project_root))
|
|
319
|
-
|
|
346
|
+
try:
|
|
347
|
+
checks.append(_check_cache_directory(project_root))
|
|
348
|
+
except pydantic.ValidationError as e:
|
|
349
|
+
raise click.ClickException(_format_config_validation_error(e)) from e
|
|
320
350
|
checks.append(_check_git_repository(project_root))
|
|
321
351
|
|
|
322
352
|
if check_remote:
|
|
@@ -1787,9 +1787,11 @@ class Engine:
|
|
|
1787
1787
|
return lock_service.acquire_many(lock_requests)
|
|
1788
1788
|
|
|
1789
1789
|
lock_handle = await anyio.to_thread.run_sync(_acquire_artifact_locks)
|
|
1790
|
-
await self._set_stage_state(stage_name, StageExecutionState.PREPARING)
|
|
1791
|
-
|
|
1792
1790
|
try:
|
|
1791
|
+
lock_data = await anyio.to_thread.run_sync(_read_lock)
|
|
1792
|
+
if lock_data is None:
|
|
1793
|
+
return False
|
|
1794
|
+
await self._set_stage_state(stage_name, StageExecutionState.PREPARING)
|
|
1793
1795
|
return await self._try_skip_with_locks(
|
|
1794
1796
|
stage_name,
|
|
1795
1797
|
stage_info,
|
|
@@ -84,6 +84,8 @@ class WatchCoordinator:
|
|
|
84
84
|
|
|
85
85
|
_graph: nx.DiGraph[str]
|
|
86
86
|
_artifact_paths: dict[pathlib.Path, pivot_types.ArtifactIdentity]
|
|
87
|
+
_resolved_output_paths: dict[tuple[pathlib.Path, str], list[pivot_types.ArtifactIdentity]]
|
|
88
|
+
_resolved_input_paths: dict[pathlib.Path, list[pivot_types.ArtifactIdentity]]
|
|
87
89
|
|
|
88
90
|
def __init__(
|
|
89
91
|
self,
|
|
@@ -92,6 +94,29 @@ class WatchCoordinator:
|
|
|
92
94
|
) -> None:
|
|
93
95
|
self._graph = graph
|
|
94
96
|
self._artifact_paths = artifact_paths
|
|
97
|
+
self._rebuild_resolved_path_indexes()
|
|
98
|
+
|
|
99
|
+
def _rebuild_resolved_path_indexes(self) -> None:
|
|
100
|
+
"""Rebuild the input and output aliases used for physical watch paths."""
|
|
101
|
+
self._resolved_input_paths = dict[pathlib.Path, list[pivot_types.ArtifactIdentity]]()
|
|
102
|
+
self._resolved_output_paths = dict[
|
|
103
|
+
tuple[pathlib.Path, str], list[pivot_types.ArtifactIdentity]
|
|
104
|
+
]()
|
|
105
|
+
for artifact_path, identity in self._artifact_paths.items():
|
|
106
|
+
if engine_graph.get_producer(self._graph, identity) is None:
|
|
107
|
+
self._resolved_input_paths.setdefault(
|
|
108
|
+
artifact_path.resolve(), list[pivot_types.ArtifactIdentity]()
|
|
109
|
+
).append(identity)
|
|
110
|
+
continue
|
|
111
|
+
key = (artifact_path.parent.resolve(), artifact_path.name)
|
|
112
|
+
self._resolved_output_paths.setdefault(
|
|
113
|
+
key, list[pivot_types.ArtifactIdentity]()
|
|
114
|
+
).append(identity)
|
|
115
|
+
|
|
116
|
+
def _refresh_resolved_path_indexes_for(self, path: pathlib.Path) -> None:
|
|
117
|
+
"""Refresh aliases after a symlink or watched directory changes."""
|
|
118
|
+
if path.is_symlink() or path.is_dir():
|
|
119
|
+
self._rebuild_resolved_path_indexes()
|
|
95
120
|
|
|
96
121
|
@property
|
|
97
122
|
def graph(self) -> nx.DiGraph[str]:
|
|
@@ -104,6 +129,10 @@ class WatchCoordinator:
|
|
|
104
129
|
|
|
105
130
|
def resolve_identity(self, path: pathlib.Path) -> pivot_types.ArtifactIdentity | None:
|
|
106
131
|
"""Resolve a filesystem event path to the matching artifact identity."""
|
|
132
|
+
self._refresh_resolved_path_indexes_for(path)
|
|
133
|
+
outputs = self._matching_resolved_output_identities(path)
|
|
134
|
+
if outputs:
|
|
135
|
+
return outputs[0]
|
|
107
136
|
current = project.normalize_path(path)
|
|
108
137
|
while True:
|
|
109
138
|
identity = self._artifact_paths.get(current)
|
|
@@ -114,6 +143,38 @@ class WatchCoordinator:
|
|
|
114
143
|
return None
|
|
115
144
|
current = parent
|
|
116
145
|
|
|
146
|
+
def _matching_identities(self, path: pathlib.Path) -> list[pivot_types.ArtifactIdentity]:
|
|
147
|
+
"""Return identities whose resolved paths contain a filesystem event path."""
|
|
148
|
+
current = project.normalize_path(path)
|
|
149
|
+
identities = list[pivot_types.ArtifactIdentity]()
|
|
150
|
+
while True:
|
|
151
|
+
identity = self._artifact_paths.get(current)
|
|
152
|
+
if identity is not None:
|
|
153
|
+
identities.append(identity)
|
|
154
|
+
parent = current.parent
|
|
155
|
+
if parent == current:
|
|
156
|
+
return identities
|
|
157
|
+
current = parent
|
|
158
|
+
|
|
159
|
+
def _matching_resolved_input_identities(
|
|
160
|
+
self, path: pathlib.Path
|
|
161
|
+
) -> list[pivot_types.ArtifactIdentity]:
|
|
162
|
+
resolved_path = path.resolve()
|
|
163
|
+
identities = list[pivot_types.ArtifactIdentity]()
|
|
164
|
+
current = resolved_path
|
|
165
|
+
while True:
|
|
166
|
+
identities.extend(self._resolved_input_paths.get(current, []))
|
|
167
|
+
parent = current.parent
|
|
168
|
+
if parent == current:
|
|
169
|
+
return identities
|
|
170
|
+
current = parent
|
|
171
|
+
|
|
172
|
+
def _matching_resolved_output_identities(
|
|
173
|
+
self, path: pathlib.Path
|
|
174
|
+
) -> list[pivot_types.ArtifactIdentity]:
|
|
175
|
+
"""Return output identities by resolved parent directory and logical leaf name."""
|
|
176
|
+
return self._resolved_output_paths.get((path.parent.resolve(), path.name), [])
|
|
177
|
+
|
|
117
178
|
def should_filter_path(
|
|
118
179
|
self,
|
|
119
180
|
path: pathlib.Path,
|
|
@@ -134,13 +195,23 @@ class WatchCoordinator:
|
|
|
134
195
|
"""Get all stages affected by the changed paths, including downstream."""
|
|
135
196
|
affected = set[str]()
|
|
136
197
|
for path in paths:
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
198
|
+
self._refresh_resolved_path_indexes_for(path)
|
|
199
|
+
output_identities = set(self._matching_resolved_output_identities(path))
|
|
200
|
+
if output_identities:
|
|
201
|
+
identities = output_identities
|
|
202
|
+
else:
|
|
203
|
+
logical_identities = set(self._matching_identities(path))
|
|
204
|
+
identities = logical_identities.copy()
|
|
205
|
+
if all(
|
|
206
|
+
engine_graph.get_producer(self._graph, identity) is None
|
|
207
|
+
for identity in logical_identities
|
|
208
|
+
):
|
|
209
|
+
identities.update(self._matching_resolved_input_identities(path))
|
|
210
|
+
for identity in identities:
|
|
211
|
+
consumers = engine_graph.get_consumers(self._graph, identity)
|
|
212
|
+
affected.update(consumers)
|
|
213
|
+
for stage in consumers:
|
|
214
|
+
affected.update(engine_graph.get_downstream_stages(self._graph, stage))
|
|
144
215
|
return sorted(affected)
|
|
145
216
|
|
|
146
217
|
def get_producer(self, path: pathlib.Path) -> str | None:
|
|
@@ -11,6 +11,7 @@ import logging
|
|
|
11
11
|
import marshal
|
|
12
12
|
import os
|
|
13
13
|
import pathlib
|
|
14
|
+
import re
|
|
14
15
|
import sys
|
|
15
16
|
import textwrap
|
|
16
17
|
import types
|
|
@@ -60,9 +61,12 @@ _PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
|
|
|
60
61
|
# `ext:`), not `mod:`, and follows `settings.Config.SCALE` and `pkg.sub.helper`.
|
|
61
62
|
# Version 25 records call-site artifact wiring (parameter bindings, collection kind/order,
|
|
62
63
|
# and project-relative input paths) in the code manifest.
|
|
63
|
-
|
|
64
|
+
# Version 26 records tuples and frozensets of scalars and compiled regexes by value.
|
|
65
|
+
_CACHE_SCHEMA_VERSION = 26
|
|
64
66
|
_REPR_SIZE_LIMIT = 10_000
|
|
65
67
|
|
|
68
|
+
_SCALAR_TYPES = (bool, int, float, str, bytes, type(None))
|
|
69
|
+
|
|
66
70
|
_SITE_PACKAGE_PATHS = ("site-packages", "dist-packages")
|
|
67
71
|
|
|
68
72
|
# Type alias for AST hash cache entry tuple.
|
|
@@ -563,6 +567,24 @@ class _ValueKind(enum.Enum):
|
|
|
563
567
|
SCALAR = "scalar"
|
|
564
568
|
|
|
565
569
|
|
|
570
|
+
def _immutable_value_text(value: Any) -> str | None:
|
|
571
|
+
"""Return the manifest text for a safely trackable immutable value, if any.
|
|
572
|
+
|
|
573
|
+
Scalars keep their existing repr. Exact tuples and frozensets are tracked only when every
|
|
574
|
+
member is an exact scalar type, one level deep. Compiled regexes record their full pattern
|
|
575
|
+
and flags because their repr truncates long patterns.
|
|
576
|
+
"""
|
|
577
|
+
if isinstance(value, _SCALAR_TYPES):
|
|
578
|
+
return repr(value)
|
|
579
|
+
if type(value) in (tuple, frozenset) and all(
|
|
580
|
+
type(item) in _SCALAR_TYPES for item in cast("tuple[Any, ...] | frozenset[Any]", value)
|
|
581
|
+
):
|
|
582
|
+
return _serialize_value_for_hash(value)
|
|
583
|
+
if isinstance(value, re.Pattern):
|
|
584
|
+
return _serialize_value_for_hash(value)
|
|
585
|
+
return None
|
|
586
|
+
|
|
587
|
+
|
|
566
588
|
def _value_kind(value: Any) -> _ValueKind:
|
|
567
589
|
"""Classify a value a walked function reads; the walk and a cache hit share this."""
|
|
568
590
|
# functools.partial fails is_user_code() (module is functools/stdlib).
|
|
@@ -576,7 +598,7 @@ def _value_kind(value: Any) -> _ValueKind:
|
|
|
576
598
|
return _ValueKind.MODULE
|
|
577
599
|
if isinstance(value, logging.Logger):
|
|
578
600
|
return _ValueKind.LOGGER
|
|
579
|
-
if
|
|
601
|
+
if _immutable_value_text(value) is not None:
|
|
580
602
|
return _ValueKind.SCALAR
|
|
581
603
|
return _ValueKind.REFUSED
|
|
582
604
|
|
|
@@ -598,7 +620,7 @@ def _value_token(value: Any) -> str:
|
|
|
598
620
|
if kind is _ValueKind.MODULE:
|
|
599
621
|
return f"module:{value.__name__}"
|
|
600
622
|
if kind is _ValueKind.SCALAR:
|
|
601
|
-
return f"{type(value).__name__}:{value
|
|
623
|
+
return f"{type(value).__name__}:{_immutable_value_text(value)}"
|
|
602
624
|
if kind is _ValueKind.EXTERNAL:
|
|
603
625
|
return f"external:{_library_name(value)}"
|
|
604
626
|
return kind.value
|
|
@@ -1335,9 +1357,9 @@ def _check_non_scalar_capture(
|
|
|
1335
1357
|
subject += f" as the default of parameter '{parameter}'"
|
|
1336
1358
|
message = (
|
|
1337
1359
|
f"Stage '{stage_name}': {subject} (type: {type(value).__name__}).\n"
|
|
1338
|
-
+ "Pivot tracks
|
|
1339
|
-
+ "Other values may change without a reliable
|
|
1340
|
-
+ "outputs.\n"
|
|
1360
|
+
+ "Pivot tracks module-level bool, int, float, str, bytes, and None values, tuples and "
|
|
1361
|
+
+ "frozensets of them, and compiled regexes. Other values may change without a reliable "
|
|
1362
|
+
+ "fingerprint, which can cause silent wrong outputs.\n"
|
|
1341
1363
|
+ "Fix: move this value into the stage's StageParams, or read it from a pipeline input.\n"
|
|
1342
1364
|
+ "To suppress: set core.unsafe_fingerprinting=true or PIVOT_UNSAFE_FINGERPRINTING=1"
|
|
1343
1365
|
)
|
|
@@ -1548,7 +1570,9 @@ def _process_captured_value(
|
|
|
1548
1570
|
if chain is not None:
|
|
1549
1571
|
_process_module_dependency(chain, value, func, manifest, visited)
|
|
1550
1572
|
case _ValueKind.SCALAR:
|
|
1551
|
-
|
|
1573
|
+
text = _immutable_value_text(value)
|
|
1574
|
+
assert text is not None
|
|
1575
|
+
manifest[f"const:{binding_ref}"] = text
|
|
1552
1576
|
case _ValueKind.REFUSED:
|
|
1553
1577
|
_check_non_scalar_capture(name, value, stage_name, parameter=parameter)
|
|
1554
1578
|
case _ValueKind.EXTERNAL:
|
|
@@ -1681,7 +1705,7 @@ def _process_default_values(
|
|
|
1681
1705
|
binding_ref = f"{func_ref}.{parameter}"
|
|
1682
1706
|
step = f"d:{parameter}"
|
|
1683
1707
|
_record_read(func, step)
|
|
1684
|
-
if isinstance(value,
|
|
1708
|
+
if isinstance(value, _SCALAR_TYPES):
|
|
1685
1709
|
manifest[f"default:{binding_ref}"] = repr(value)
|
|
1686
1710
|
continue
|
|
1687
1711
|
if isinstance(value, stage_def.StageParams):
|
|
@@ -1695,6 +1719,10 @@ def _process_default_values(
|
|
|
1695
1719
|
expression = expressions.get(parameter)
|
|
1696
1720
|
if expression is not None and _is_literal(expression):
|
|
1697
1721
|
continue
|
|
1722
|
+
text = _immutable_value_text(value)
|
|
1723
|
+
if text is not None:
|
|
1724
|
+
manifest[f"default:{binding_ref}"] = text
|
|
1725
|
+
continue
|
|
1698
1726
|
source = ast.unparse(expression) if expression is not None else parameter
|
|
1699
1727
|
with _walking(step):
|
|
1700
1728
|
_process_captured_value(
|
|
@@ -2140,12 +2168,16 @@ def _canonicalize_value_for_hash(value: Any, active: tuple[int, ...] = ()) -> An
|
|
|
2140
2168
|
canonical({field: value})]`, field by field, so a secret inside one reaches that leaf
|
|
2141
2169
|
rather than the generated repr's mask. An object whose class keeps `object.__repr__`
|
|
2142
2170
|
(which holds the address) and that has a `__dict__` becomes
|
|
2143
|
-
`[module.qualname, canonical(vars(obj))]`.
|
|
2144
|
-
|
|
2145
|
-
|
|
2171
|
+
`[module.qualname, canonical(vars(obj))]`. A compiled regex becomes
|
|
2172
|
+
`["re.Pattern", canonical(pattern), flags]`, so its full pattern is recorded. Other leaves
|
|
2173
|
+
keep their `repr`, tagged with their type. A class is a leaf, even a pydantic model class:
|
|
2174
|
+
only an instance is dumped.
|
|
2146
2175
|
"""
|
|
2147
2176
|
if value is None or isinstance(value, (bool, int, float, str)):
|
|
2148
2177
|
return value
|
|
2178
|
+
if isinstance(value, re.Pattern):
|
|
2179
|
+
pattern = cast("re.Pattern[Any]", value)
|
|
2180
|
+
return ["re.Pattern", _canonicalize_value_for_hash(pattern.pattern), pattern.flags]
|
|
2149
2181
|
if id(value) in active:
|
|
2150
2182
|
return ["cycle", len(active) - active.index(id(value))]
|
|
2151
2183
|
if hasattr(value, "model_dump") and not isinstance(value, type):
|
|
@@ -190,7 +190,7 @@ def params_equal(old: object, new: object) -> bool:
|
|
|
190
190
|
(``parameters.get_effective_params``), so the text is the canonical encoding. Python
|
|
191
191
|
equality alone would call a NaN param changed on every run, because nan != nan.
|
|
192
192
|
"""
|
|
193
|
-
return
|
|
193
|
+
return json.dumps(old, sort_keys=True) == json.dumps(new, sort_keys=True)
|
|
194
194
|
|
|
195
195
|
|
|
196
196
|
def diff_dep_hashes(
|
|
@@ -36,7 +36,6 @@ XXHASH64_HEX_LENGTH = 16 # xxhash64 produces 64-bit hash = 16 hex characters
|
|
|
36
36
|
_RESTORE_TEMP_PREFIX = ".pivot_restore_"
|
|
37
37
|
_BACKUP_TEMP_PREFIX = ".pivot_backup_"
|
|
38
38
|
_RESTORE_LOCK_PREFIX = ".pivot_restore_lock_"
|
|
39
|
-
_KEEP_STAGING_PREFIX = ".pivot_keep_"
|
|
40
39
|
_RESTORE_TEMP_MAX_AGE = 3600 # 1 hour
|
|
41
40
|
|
|
42
41
|
|
|
@@ -46,36 +45,6 @@ def _restore_target_token(target_path: pathlib.Path) -> str:
|
|
|
46
45
|
return hashlib.sha256(str(logical_path).encode()).hexdigest()[:32]
|
|
47
46
|
|
|
48
47
|
|
|
49
|
-
def _keep_staging_path(path: pathlib.Path) -> pathlib.Path:
|
|
50
|
-
"""Sibling that holds a directory's kept entries while it is replaced."""
|
|
51
|
-
return path.parent / f"{_KEEP_STAGING_PREFIX}{_restore_target_token(path)}"
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
def _move_kept_entries(source: pathlib.Path, destination: pathlib.Path) -> None:
|
|
55
|
-
"""Move every entry under source into destination, merging directories, then drop source."""
|
|
56
|
-
with os.scandir(source) as entries:
|
|
57
|
-
for entry in entries:
|
|
58
|
-
target = destination / entry.name
|
|
59
|
-
if not os.path.lexists(target):
|
|
60
|
-
destination.mkdir(parents=True, exist_ok=True)
|
|
61
|
-
pathlib.Path(entry.path).rename(target)
|
|
62
|
-
elif (
|
|
63
|
-
entry.is_dir(follow_symlinks=False) and target.is_dir() and not target.is_symlink()
|
|
64
|
-
):
|
|
65
|
-
_move_kept_entries(pathlib.Path(entry.path), target)
|
|
66
|
-
else:
|
|
67
|
-
raise exceptions.CacheRestoreError(
|
|
68
|
-
f"Cannot put back {entry.path}: {target}. Move the entry by hand, then retry."
|
|
69
|
-
)
|
|
70
|
-
source.rmdir()
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def staged_kept_entries(path: pathlib.Path) -> pathlib.Path | None:
|
|
74
|
-
"""Return the staging directory an interrupted keep-ignored restore left, if any."""
|
|
75
|
-
keep_dir = _keep_staging_path(path)
|
|
76
|
-
return keep_dir if keep_dir.exists() else None
|
|
77
|
-
|
|
78
|
-
|
|
79
48
|
def _restore_staging_path(path: pathlib.Path, token: str | None = None) -> pathlib.Path:
|
|
80
49
|
"""Return the deterministic staging path for a file checkout."""
|
|
81
50
|
return path.parent / f"{_RESTORE_TEMP_PREFIX}{token or _restore_target_token(path)}.file"
|
|
@@ -190,39 +159,12 @@ def hash_file(
|
|
|
190
159
|
return file_hash, file_stat
|
|
191
160
|
|
|
192
161
|
|
|
193
|
-
# Hardcoded ignore patterns for hot path - O(1) lookups only.
|
|
194
|
-
# For full .pivotignore support, see pivot.ignore module.
|
|
195
|
-
_IGNORE_DIRS: frozenset[str] = frozenset(
|
|
196
|
-
{
|
|
197
|
-
"__pycache__",
|
|
198
|
-
".venv",
|
|
199
|
-
"venv",
|
|
200
|
-
".git",
|
|
201
|
-
".hg",
|
|
202
|
-
".svn",
|
|
203
|
-
".idea",
|
|
204
|
-
".vscode",
|
|
205
|
-
"node_modules",
|
|
206
|
-
".pivot",
|
|
207
|
-
"dist",
|
|
208
|
-
"build",
|
|
209
|
-
}
|
|
210
|
-
)
|
|
211
|
-
_IGNORE_SUFFIXES: tuple[str, ...] = (".pyc", ".pyo", ".swp", ".swo")
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
def _should_skip_entry(entry: os.DirEntry[str]) -> bool:
|
|
215
|
-
"""Fast ignore check for hot path. O(1) lookups only."""
|
|
216
|
-
name = entry.name
|
|
217
|
-
if entry.is_dir(follow_symlinks=False):
|
|
218
|
-
return name in _IGNORE_DIRS
|
|
219
|
-
# File checks: suffix and special patterns
|
|
220
|
-
return name.endswith(_IGNORE_SUFFIXES) or name.endswith("~") or name.startswith(".#")
|
|
221
|
-
|
|
222
|
-
|
|
223
162
|
def _scandir_recursive(path: pathlib.Path) -> Generator[os.DirEntry[str]]:
|
|
224
|
-
"""Yield
|
|
163
|
+
"""Yield every file under `path` recursively, using os.scandir() for efficiency.
|
|
225
164
|
|
|
165
|
+
A declared directory artifact is hashed and cached whole: no name is ignored, so a change
|
|
166
|
+
to any file in a directory input reruns its consumers, and every file in a directory output
|
|
167
|
+
survives cache and restore. Ignore rules belong to code fingerprinting and watch mode.
|
|
226
168
|
DirEntry objects cache stat results, avoiding redundant syscalls.
|
|
227
169
|
Symlinks are skipped to prevent loops.
|
|
228
170
|
PermissionError propagates to caller - partial hashes would be incorrect.
|
|
@@ -234,8 +176,6 @@ def _scandir_recursive(path: pathlib.Path) -> Generator[os.DirEntry[str]]:
|
|
|
234
176
|
for entry in entries:
|
|
235
177
|
if entry.is_symlink():
|
|
236
178
|
continue
|
|
237
|
-
if _should_skip_entry(entry):
|
|
238
|
-
continue
|
|
239
179
|
if entry.is_file():
|
|
240
180
|
yield entry
|
|
241
181
|
elif entry.is_dir():
|
|
@@ -773,7 +713,6 @@ def restore_from_cache(
|
|
|
773
713
|
state_dir: pathlib.Path | None = None,
|
|
774
714
|
*,
|
|
775
715
|
lock_token: str | None = None,
|
|
776
|
-
keep_ignored: bool = False,
|
|
777
716
|
) -> bool:
|
|
778
717
|
"""Restore file or directory from cache. Returns True if successful.
|
|
779
718
|
|
|
@@ -785,8 +724,6 @@ def restore_from_cache(
|
|
|
785
724
|
checkout_modes: Ordered list of link modes to try with fallback on failure.
|
|
786
725
|
state_dir: State directory for restore locks and crash cleanup.
|
|
787
726
|
lock_token: Target token yielded by an already-held restore_lock for path.
|
|
788
|
-
keep_ignored: When replacing an existing directory, preserve the entries the directory
|
|
789
|
-
hash skips in the restored tree.
|
|
790
727
|
"""
|
|
791
728
|
effective_modes = _resolve_checkout_modes(checkout_mode, checkout_modes)
|
|
792
729
|
if lock_token is not None and state_dir is None:
|
|
@@ -800,7 +737,6 @@ def restore_from_cache(
|
|
|
800
737
|
effective_modes,
|
|
801
738
|
state_dir,
|
|
802
739
|
lock_token=lock_token,
|
|
803
|
-
keep_ignored=keep_ignored,
|
|
804
740
|
)
|
|
805
741
|
return _restore_file_from_cache(
|
|
806
742
|
path, output_hash, cache_dir, effective_modes, state_dir, lock_token=lock_token
|
|
@@ -855,9 +791,8 @@ def _restore_directory_from_cache(
|
|
|
855
791
|
state_dir: pathlib.Path | None = None,
|
|
856
792
|
*,
|
|
857
793
|
lock_token: str | None = None,
|
|
858
|
-
keep_ignored: bool = False,
|
|
859
794
|
) -> bool:
|
|
860
|
-
"""Restore a directory atomically
|
|
795
|
+
"""Restore a directory atomically."""
|
|
861
796
|
cache_dir_path = get_cache_path(cache_dir, output_hash["hash"])
|
|
862
797
|
symlink_fast_path = checkout_modes[0] == CheckoutMode.SYMLINK and cache_dir_path.exists()
|
|
863
798
|
if not symlink_fast_path:
|
|
@@ -877,24 +812,12 @@ def _restore_directory_from_cache(
|
|
|
877
812
|
checkout_modes,
|
|
878
813
|
state_dir,
|
|
879
814
|
lock_token=held_token,
|
|
880
|
-
keep_ignored=keep_ignored,
|
|
881
815
|
)
|
|
882
816
|
|
|
883
817
|
target_token = lock_token or _restore_target_token(path)
|
|
884
818
|
if lock_token is not None:
|
|
885
819
|
_cleanup_fresh_restore_orphans(path, target_token)
|
|
886
820
|
restore_prefix = f"{_RESTORE_TEMP_PREFIX}{target_token}."
|
|
887
|
-
keep_dir = _keep_staging_path(path)
|
|
888
|
-
keep = list[str]()
|
|
889
|
-
if (keep_ignored or keep_dir.exists()) and path.is_dir() and not path.is_symlink():
|
|
890
|
-
if keep_dir.exists():
|
|
891
|
-
_move_kept_entries(keep_dir, path)
|
|
892
|
-
_, keep = _scan_entries(path, {entry["relpath"] for entry in output_hash["manifest"]})
|
|
893
|
-
if symlink_fast_path and keep:
|
|
894
|
-
symlink_fast_path = False
|
|
895
|
-
for entry in output_hash["manifest"]:
|
|
896
|
-
if not get_cache_path(cache_dir, entry["hash"]).exists():
|
|
897
|
-
return False
|
|
898
821
|
|
|
899
822
|
if symlink_fast_path:
|
|
900
823
|
temp_symlink = path.parent / f"{restore_prefix}symlink_{secrets.token_hex(4)}"
|
|
@@ -948,14 +871,6 @@ def _restore_directory_from_cache(
|
|
|
948
871
|
except OSError as e:
|
|
949
872
|
logger.debug(f"chmod failed (may be NFS with root_squash): {e}")
|
|
950
873
|
|
|
951
|
-
for relpath in keep:
|
|
952
|
-
source = path / relpath
|
|
953
|
-
if not os.path.lexists(source):
|
|
954
|
-
continue
|
|
955
|
-
staged = keep_dir / relpath
|
|
956
|
-
staged.parent.mkdir(parents=True, exist_ok=True)
|
|
957
|
-
source.rename(staged)
|
|
958
|
-
|
|
959
874
|
try:
|
|
960
875
|
if path.exists() or path.is_symlink():
|
|
961
876
|
path.rename(backup_path)
|
|
@@ -964,8 +879,6 @@ def _restore_directory_from_cache(
|
|
|
964
879
|
|
|
965
880
|
temp_path.replace(path)
|
|
966
881
|
swap_completed = True
|
|
967
|
-
if keep_dir.exists():
|
|
968
|
-
_move_kept_entries(keep_dir, path)
|
|
969
882
|
|
|
970
883
|
try:
|
|
971
884
|
_clear_path(backup_path)
|
|
@@ -979,14 +892,6 @@ def _restore_directory_from_cache(
|
|
|
979
892
|
except OSError as e:
|
|
980
893
|
logger.debug(f"Failed to restore backup: {e}")
|
|
981
894
|
backup_restore_failed = True
|
|
982
|
-
if not swap_completed and keep_dir.exists() and path.is_dir() and not path.is_symlink():
|
|
983
|
-
try:
|
|
984
|
-
_move_kept_entries(keep_dir, path)
|
|
985
|
-
except (OSError, exceptions.CacheRestoreError) as e:
|
|
986
|
-
logger.warning(
|
|
987
|
-
f"Entries kept from {path} remain in {keep_dir} ({e}); "
|
|
988
|
-
+ "the next restore of it puts them back"
|
|
989
|
-
)
|
|
990
895
|
raise
|
|
991
896
|
finally:
|
|
992
897
|
if temp_path.exists():
|
|
@@ -1017,39 +922,25 @@ def _is_cached(cache_dir: pathlib.Path, content_hash: str | None) -> bool:
|
|
|
1017
922
|
)
|
|
1018
923
|
|
|
1019
924
|
|
|
1020
|
-
def _scan_entries(path: pathlib.Path
|
|
1021
|
-
"""Return
|
|
925
|
+
def _scan_entries(path: pathlib.Path) -> list[str]:
|
|
926
|
+
"""Return every file and symlink under ``path`` as relative paths.
|
|
1022
927
|
|
|
1023
|
-
Symlinked directories are not followed.
|
|
1024
|
-
path is part of the output and is scanned like any other entry.
|
|
928
|
+
Symlinked directories are not followed.
|
|
1025
929
|
"""
|
|
1026
|
-
expected_dirs = {
|
|
1027
|
-
str(parent)
|
|
1028
|
-
for relpath in expected_paths
|
|
1029
|
-
for parent in pathlib.PurePosixPath(relpath).parents
|
|
1030
|
-
if str(parent) != "."
|
|
1031
|
-
}
|
|
1032
930
|
files = list[str]()
|
|
1033
|
-
ignored = list[str]()
|
|
1034
931
|
|
|
1035
932
|
def _walk(directory: pathlib.Path) -> None:
|
|
1036
933
|
with os.scandir(directory) as entries:
|
|
1037
934
|
for entry in entries:
|
|
1038
935
|
entry_path = pathlib.Path(entry.path)
|
|
1039
936
|
relpath = str(entry_path.relative_to(path))
|
|
1040
|
-
if (
|
|
1041
|
-
_should_skip_entry(entry)
|
|
1042
|
-
and relpath not in expected_paths
|
|
1043
|
-
and relpath not in expected_dirs
|
|
1044
|
-
):
|
|
1045
|
-
ignored.append(relpath)
|
|
1046
|
-
elif entry.is_symlink() or entry.is_file():
|
|
937
|
+
if entry.is_symlink() or entry.is_file():
|
|
1047
938
|
files.append(relpath)
|
|
1048
939
|
elif entry.is_dir():
|
|
1049
940
|
_walk(entry_path)
|
|
1050
941
|
|
|
1051
942
|
_walk(path)
|
|
1052
|
-
return files
|
|
943
|
+
return files
|
|
1053
944
|
|
|
1054
945
|
|
|
1055
946
|
def _is_same_file(file_stat: os.stat_result, cache_path: pathlib.Path) -> bool:
|
|
@@ -1069,15 +960,12 @@ def restore_missing_in_directory(
|
|
|
1069
960
|
checkout_modes: list[CheckoutMode],
|
|
1070
961
|
state_db: state_mod.StateDB | None = None,
|
|
1071
962
|
) -> MissingRestore:
|
|
1072
|
-
"""Restore
|
|
963
|
+
"""Restore missing target files without overwriting local changes.
|
|
1073
964
|
|
|
1074
|
-
When every existing file matches the target,
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
``replace`` asks the caller to swap with ``restore_from_cache(..., keep_ignored=True)``,
|
|
1079
|
-
which carries over the entries the directory hash skips (build/, *.swp, ...). Otherwise
|
|
1080
|
-
nothing is written, and the files the cache lacks (local edits, extra files) are returned.
|
|
965
|
+
When every existing file matches the target, missing files are restored in place. When a
|
|
966
|
+
file differs from the target or an extra file exists, only a whole-directory swap can
|
|
967
|
+
restore the target. The swap is safe only when every file it removes is in the cache;
|
|
968
|
+
otherwise the local changes are returned without writing anything.
|
|
1081
969
|
|
|
1082
970
|
A retained file is read only when nothing else proves its content: a symlink to the
|
|
1083
971
|
target's cache path, a hardlink to the target's cache blob, or a matching ``state_db``
|
|
@@ -1128,7 +1016,7 @@ def restore_missing_in_directory(
|
|
|
1128
1016
|
)
|
|
1129
1017
|
missing.append((entry, file_path, file_cache_path))
|
|
1130
1018
|
|
|
1131
|
-
files
|
|
1019
|
+
files = _scan_entries(path)
|
|
1132
1020
|
for relpath in files:
|
|
1133
1021
|
if relpath in expected_paths:
|
|
1134
1022
|
continue
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|