pivot 0.6.2__tar.gz → 0.6.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pivot-0.6.2 → pivot-0.6.4}/PKG-INFO +1 -2
- {pivot-0.6.2 → pivot-0.6.4}/pyproject.toml +1 -2
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/AGENTS.md +1 -1
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/checkout.py +5 -3
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/helpers.py +20 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/status.py +11 -2
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/track.py +48 -10
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/update.py +16 -3
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/verify.py +91 -17
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/discovery.py +14 -6
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/engine.py +26 -5
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/watch.py +53 -2
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/core.py +9 -2
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/worker.py +20 -36
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/explain.py +23 -4
- pivot-0.6.4/src/pivot/ignore.py +906 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/import_artifact.py +1 -1
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/sync.py +63 -22
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/state_dir.py +17 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/status.py +27 -10
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/AGENTS.md +17 -2
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/cache.py +61 -12
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/listing.py +35 -17
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/lock.py +215 -36
- pivot-0.6.4/src/pivot/storage/member_skip.py +163 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/ownership.py +112 -45
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/store.py +71 -9
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/track.py +34 -13
- pivot-0.6.2/src/pivot/ignore.py +0 -358
- {pivot-0.6.2 → pivot-0.6.4}/LICENSE +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/README.md +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/__init__.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/ast_utils.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/CLAUDE.md +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/__init__.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/_run_common.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/check_ignore.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/commit.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/completion.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/config.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/console.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/dag.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/data.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/decorators.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/doctor.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/errors.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/fingerprint.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/history.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/import_cmd.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/init.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/list.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/metrics.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/params.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/plots.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/remote.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/repro.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/run.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/targets.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/compose.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/config/__init__.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/config/io.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/config/models.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/config_files.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/dag/__init__.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/dag/render.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/decorators.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/display.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/__init__.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/agent_rpc.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/graph.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/scheduler.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/sinks.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/sources.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/types.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/worker_pool.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/exceptions.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/AGENTS.md +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/CLAUDE.md +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/__init__.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/commit.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/fingerprint.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/git.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/loaded_source.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/loaders.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/metrics.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/names.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/parameters.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/path_policy.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/project.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/py.typed +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/registry.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/AGENTS.md +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/__init__.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/batch.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/config.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/gcs.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/git_archive.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/github.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/keys.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/protocol.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/storage.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/run_history.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/__init__.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/common.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/data.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/metrics.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/params.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/plots.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/selection.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/skip.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/stage_def.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/stage_io.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/CLAUDE.md +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/__init__.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/artifact_lock.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/mounts.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/presentation.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/restore.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/state.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/types.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/unix_socket.py +0 -0
- {pivot-0.6.2 → pivot-0.6.4}/src/pivot/yaml_config.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pivot
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.4
|
|
4
4
|
Summary: High-performance Python pipeline tool with automatic code change detection
|
|
5
5
|
Author: Sami Jawhar
|
|
6
6
|
Author-email: Sami Jawhar <sami@metr.org>
|
|
@@ -21,7 +21,6 @@ Requires-Dist: grandalf>=0.8
|
|
|
21
21
|
Requires-Dist: lmdb>=1.4.0
|
|
22
22
|
Requires-Dist: loky>=3.5.6,<3.8
|
|
23
23
|
Requires-Dist: networkx>=3.0
|
|
24
|
-
Requires-Dist: pathspec>=0.12.0
|
|
25
24
|
Requires-Dist: pandas>=2.0
|
|
26
25
|
Requires-Dist: psutil>=7.0
|
|
27
26
|
Requires-Dist: pydantic>=2.13
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "pivot"
|
|
3
|
-
version = "0.6.
|
|
3
|
+
version = "0.6.4"
|
|
4
4
|
description = "High-performance Python pipeline tool with automatic code change detection"
|
|
5
5
|
requires-python = ">=3.13,<3.14"
|
|
6
6
|
readme = "README.md"
|
|
@@ -25,7 +25,6 @@ dependencies = [
|
|
|
25
25
|
"lmdb>=1.4.0",
|
|
26
26
|
"loky>=3.5.6,<3.8",
|
|
27
27
|
"networkx>=3.0",
|
|
28
|
-
"pathspec>=0.12.0",
|
|
29
28
|
"pandas>=2.0",
|
|
30
29
|
"psutil>=7.0",
|
|
31
30
|
"pydantic>=2.13",
|
|
@@ -37,7 +37,7 @@ Set `auto_discover=False` only for commands that don't use the stage registry:
|
|
|
37
37
|
| run | True (default) | Need registry to find stages |
|
|
38
38
|
| list, dag | True (default), allow_all=True | Need registry; --all for multi-pipeline |
|
|
39
39
|
| checkout | True (default) | Need registry; handles None pipeline gracefully |
|
|
40
|
-
| track | False | Discovers every pipeline itself (`discovery.declared_outputs`, as `--all` does) and refuses a new path overlapping any of their outputs |
|
|
40
|
+
| track | False | Discovers every pipeline itself (`discovery.declared_outputs`, as `--all` does) and refuses a new path overlapping any of their outputs, or one whose `.pvt` sidecar the lock of a stage that no longer exists records as that stage's file |
|
|
41
41
|
| status | True (default), allow_all=True | Handles None pipeline gracefully (skips stages section) |
|
|
42
42
|
| init, doctor, check-ignore | False | No stage data needed |
|
|
43
43
|
| history, show | False | Read run history only |
|
|
@@ -755,9 +755,6 @@ def checkout(
|
|
|
755
755
|
[cache.CheckoutMode(checkout_mode)] if checkout_mode else config.get_checkout_mode_order()
|
|
756
756
|
)
|
|
757
757
|
|
|
758
|
-
# Discover tracked files
|
|
759
|
-
tracked_files = track.discover_pvt_files(project_root)
|
|
760
|
-
|
|
761
758
|
# Get stage output info from lock files (cached outputs only)
|
|
762
759
|
pipeline = cli_decorators.get_pipeline_from_context()
|
|
763
760
|
stage_outputs = dict[str, Expected]()
|
|
@@ -765,6 +762,11 @@ def checkout(
|
|
|
765
762
|
if pipeline is not None:
|
|
766
763
|
stage_outputs, output_state_dirs = _get_stage_output_info(cache_dir)
|
|
767
764
|
|
|
765
|
+
# Discover tracked files: a stage's own .pvt member of its output, in any pipeline, is none
|
|
766
|
+
tracked_files = track.discover_pvt_files(
|
|
767
|
+
project_root, skip=cli_helpers.own_member_skip(cache_dir)
|
|
768
|
+
)
|
|
769
|
+
|
|
768
770
|
state_dir = config.locate_state_dir()
|
|
769
771
|
|
|
770
772
|
# Every target adds its read or restored file hashes to one thread-safe batch. Saving once
|
|
@@ -9,10 +9,13 @@ import click
|
|
|
9
9
|
from tqdm.asyncio import tqdm as async_tqdm
|
|
10
10
|
|
|
11
11
|
if TYPE_CHECKING:
|
|
12
|
+
import pathlib
|
|
13
|
+
|
|
12
14
|
from networkx import DiGraph
|
|
13
15
|
|
|
14
16
|
from pivot.cli import CliContext
|
|
15
17
|
from pivot.registry import PipelineLike, RegistryStageInfo
|
|
18
|
+
from pivot.storage import track
|
|
16
19
|
from pivot.storage.store import WorkspaceStore
|
|
17
20
|
|
|
18
21
|
from pivot import display, exceptions, names, project
|
|
@@ -83,6 +86,23 @@ def require_workspace_store() -> WorkspaceStore:
|
|
|
83
86
|
return store
|
|
84
87
|
|
|
85
88
|
|
|
89
|
+
def own_member_skip(cache_dir: pathlib.Path) -> track.PvtSkip:
|
|
90
|
+
"""The skip that leaves a stage's own ``*.pvt`` member out of what this command discovers
|
|
91
|
+
(``member_skip.own_member_skip``): the loaded pipeline's stages (none without a pipeline)
|
|
92
|
+
decide their outputs, and the directory outputs other stages' locks record decide every
|
|
93
|
+
other. ``cache_dir`` is the cache object root."""
|
|
94
|
+
from pivot import config
|
|
95
|
+
from pivot.storage import member_skip
|
|
96
|
+
|
|
97
|
+
pipeline = cli_decorators.get_pipeline_from_context()
|
|
98
|
+
return member_skip.own_member_skip(
|
|
99
|
+
project.get_project_root(),
|
|
100
|
+
get_all_stages() if pipeline is not None else {},
|
|
101
|
+
config.locate_state_dir(),
|
|
102
|
+
cache_dir,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
86
106
|
def _json_default(obj: Any) -> Any: # noqa: ANN401 - json.dumps default requires Any
|
|
87
107
|
"""Handle non-standard JSON types in JSONL output."""
|
|
88
108
|
if isinstance(obj, enum.Enum):
|
|
@@ -91,6 +91,13 @@ def status(
|
|
|
91
91
|
show_tracked = show_all or tracked_only
|
|
92
92
|
show_remote = remote_only or remote
|
|
93
93
|
|
|
94
|
+
# A stage's own .pvt member of its output, in any pipeline, is no tracked file or import
|
|
95
|
+
skip = (
|
|
96
|
+
cli_helpers.own_member_skip(resolved_cache_dir / "files")
|
|
97
|
+
if show_tracked or check_imports
|
|
98
|
+
else None
|
|
99
|
+
)
|
|
100
|
+
|
|
94
101
|
# When --explain is used, fetch detailed explanations instead of basic status
|
|
95
102
|
pipeline_status = list[PipelineStatusInfo]()
|
|
96
103
|
pipeline_explanations = list[StageExplanation]()
|
|
@@ -141,7 +148,9 @@ def status(
|
|
|
141
148
|
pbar.update(1)
|
|
142
149
|
|
|
143
150
|
try:
|
|
144
|
-
tracked_status = status_mod.get_tracked_files_status(
|
|
151
|
+
tracked_status = status_mod.get_tracked_files_status(
|
|
152
|
+
project_root, on_progress, skip=skip
|
|
153
|
+
)
|
|
145
154
|
finally:
|
|
146
155
|
if pbar is not None:
|
|
147
156
|
pbar.close()
|
|
@@ -161,7 +170,7 @@ def status(
|
|
|
161
170
|
|
|
162
171
|
import_statuses = list[ImportStatusInfo]()
|
|
163
172
|
if check_imports:
|
|
164
|
-
import_statuses = status_mod.get_import_status(project_root)
|
|
173
|
+
import_statuses = status_mod.get_import_status(project_root, skip=skip)
|
|
165
174
|
|
|
166
175
|
# Compute counts once for suggestions and output
|
|
167
176
|
if explain and pipeline_explanations:
|
|
@@ -2,16 +2,20 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
import pathlib
|
|
5
|
-
|
|
5
|
+
import shlex
|
|
6
|
+
from typing import TYPE_CHECKING, TypedDict
|
|
6
7
|
|
|
7
8
|
import click
|
|
8
9
|
|
|
9
10
|
from pivot import config, discovery, git, project
|
|
10
11
|
from pivot.cli import decorators as cli_decorators
|
|
11
12
|
from pivot.cli import helpers as cli_helpers
|
|
12
|
-
from pivot.storage import cache, listing, ownership, state
|
|
13
|
+
from pivot.storage import cache, listing, member_skip, ownership, state
|
|
13
14
|
from pivot.storage import track as track_mod
|
|
14
15
|
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from collections.abc import Callable
|
|
18
|
+
|
|
15
19
|
|
|
16
20
|
class TrackResult(TypedDict):
|
|
17
21
|
"""Result of tracking a single path."""
|
|
@@ -20,6 +24,26 @@ class TrackResult(TypedDict):
|
|
|
20
24
|
tracked_path: str
|
|
21
25
|
|
|
22
26
|
|
|
27
|
+
def _orphan_refusal(path_str: str, orphan: member_skip.OrphanedMember) -> str:
|
|
28
|
+
"""Why ``track`` refuses a path whose sidecar a deleted stage's lock records."""
|
|
29
|
+
project_root = project.get_project_root()
|
|
30
|
+
locks = [f"'{ownership.display_path(found, project_root)}'" for found in orphan.lock_files]
|
|
31
|
+
if len(locks) == 1:
|
|
32
|
+
recorded = f"the lock {locks[0]} records"
|
|
33
|
+
remedy = "remove that lock"
|
|
34
|
+
else:
|
|
35
|
+
recorded = f"the locks {' and '.join(locks)} record"
|
|
36
|
+
remedy = "remove those locks"
|
|
37
|
+
sidecar = ownership.display_path(orphan.sidecar, project_root)
|
|
38
|
+
return (
|
|
39
|
+
f"Cannot track '{path_str}': {recorded} '{sidecar}' as a file of stage "
|
|
40
|
+
+ f"'{orphan.stage_name}', a stage that no longer exists in any pipeline. Pivot reads "
|
|
41
|
+
+ "that stage's files there as its own, not as tracked files. If the stage is gone for "
|
|
42
|
+
+ f"good, {remedy}, then run `pivot track --force {shlex.quote(path_str)}`, which tracks "
|
|
43
|
+
+ "the file whatever its .pvt then says."
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
23
47
|
def _tracking_files(data_path: pathlib.Path) -> tuple[pathlib.Path, pathlib.Path]:
|
|
24
48
|
"""The files tracking ``data_path`` writes, or for a file removes: its .pvt and listing."""
|
|
25
49
|
pvt_path = track_mod.get_pvt_path(data_path)
|
|
@@ -86,7 +110,8 @@ def _foreign_listing(
|
|
|
86
110
|
def _track_single_path(
|
|
87
111
|
path_str: str,
|
|
88
112
|
cache_dir: pathlib.Path,
|
|
89
|
-
|
|
113
|
+
outputs: dict[str, ownership.DeclaredOutput],
|
|
114
|
+
orphaned_member: Callable[[pathlib.Path], member_skip.OrphanedMember | None],
|
|
90
115
|
existing_tracked: dict[str, track_mod.PvtData],
|
|
91
116
|
force: bool,
|
|
92
117
|
*,
|
|
@@ -152,7 +177,7 @@ def _track_single_path(
|
|
|
152
177
|
track_mod.validate_pvt_path_name(path.name)
|
|
153
178
|
|
|
154
179
|
# Check every pipeline output. OSError reaches the command error handler, which names path.
|
|
155
|
-
overlap = ownership.find_declared_overlap(path,
|
|
180
|
+
overlap = ownership.find_declared_overlap(path, outputs, project_root)
|
|
156
181
|
if overlap is not None:
|
|
157
182
|
if not already_tracked:
|
|
158
183
|
raise click.ClickException(
|
|
@@ -166,6 +191,11 @@ def _track_single_path(
|
|
|
166
191
|
+ "its .pvt sidecar outside every stage output (for example under data/raw/); until "
|
|
167
192
|
+ "then Pivot refuses to replace that output."
|
|
168
193
|
)
|
|
194
|
+
# A sidecar a stage that no longer exists recorded at this path: every command reads it as
|
|
195
|
+
# that stage's own file, by its lock, so a sidecar written there could be one none reads
|
|
196
|
+
orphan = orphaned_member(path)
|
|
197
|
+
if orphan is not None:
|
|
198
|
+
raise click.ClickException(_orphan_refusal(path_str, orphan))
|
|
169
199
|
warning = "\n".join(warnings) if warnings else None
|
|
170
200
|
|
|
171
201
|
# Hash and cache
|
|
@@ -247,19 +277,27 @@ def track(ctx: click.Context, paths: tuple[str, ...], force: bool) -> None:
|
|
|
247
277
|
project_root = project.get_project_root()
|
|
248
278
|
cache_dir = config.get_cache_dir() / "files"
|
|
249
279
|
|
|
250
|
-
# Every pipeline's outputs, not only the pipeline this directory selects: a file
|
|
251
|
-
# another pipeline's output would be removed when that pipeline runs.
|
|
252
|
-
|
|
280
|
+
# Every pipeline's outputs and stages, not only the pipeline this directory selects: a file
|
|
281
|
+
# tracked in another pipeline's output would be removed when that pipeline runs.
|
|
282
|
+
declared = discovery.declared_outputs(project_root)
|
|
283
|
+
state_dir = config.locate_state_dir()
|
|
284
|
+
orphaned_member = member_skip.orphaned_member(
|
|
285
|
+
project_root, state_dir, cache_dir, declared.stages
|
|
286
|
+
)
|
|
253
287
|
|
|
254
|
-
# Discover existing .pvt files
|
|
255
|
-
existing_tracked = track_mod.discover_pvt_files(
|
|
288
|
+
# Discover existing .pvt files: a stage's own .pvt member of its output tracks nothing
|
|
289
|
+
existing_tracked = track_mod.discover_pvt_files(
|
|
290
|
+
project_root,
|
|
291
|
+
skip=member_skip.own_member_skip(project_root, declared.stages, state_dir, cache_dir),
|
|
292
|
+
)
|
|
256
293
|
|
|
257
294
|
read = list[tuple[pathlib.Path, os.stat_result, str]]()
|
|
258
295
|
for path_str in paths:
|
|
259
296
|
result = _track_single_path(
|
|
260
297
|
path_str,
|
|
261
298
|
cache_dir,
|
|
262
|
-
|
|
299
|
+
declared.outputs,
|
|
300
|
+
orphaned_member,
|
|
263
301
|
existing_tracked,
|
|
264
302
|
force,
|
|
265
303
|
warn_left_out_symlinks=not quiet,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import functools
|
|
4
5
|
import logging
|
|
5
6
|
import pathlib
|
|
6
7
|
from typing import TYPE_CHECKING
|
|
@@ -49,8 +50,8 @@ def update(targets: tuple[str, ...], rev: str | None, dry_run: bool) -> None:
|
|
|
49
50
|
|
|
50
51
|
If no TARGETS specified, updates all imports found in the project.
|
|
51
52
|
"""
|
|
52
|
-
from pivot import project
|
|
53
|
-
from pivot.storage import track
|
|
53
|
+
from pivot import config, project
|
|
54
|
+
from pivot.storage import member_skip, track
|
|
54
55
|
|
|
55
56
|
root = project.get_project_root()
|
|
56
57
|
|
|
@@ -71,7 +72,19 @@ def update(targets: tuple[str, ...], rev: str | None, dry_run: bool) -> None:
|
|
|
71
72
|
continue
|
|
72
73
|
pairs.append((pvt_path, pvt_data))
|
|
73
74
|
else:
|
|
74
|
-
|
|
75
|
+
# A stage's own .pvt member of its output is no import. The command loads no pipeline,
|
|
76
|
+
# so the directory outputs the stages' locks record decide; it reads no config until an
|
|
77
|
+
# import .pvt is found.
|
|
78
|
+
@functools.cache
|
|
79
|
+
def own_member_skip() -> track.PvtSkip:
|
|
80
|
+
return member_skip.own_member_skip(
|
|
81
|
+
root, {}, config.locate_state_dir(), config.locate_cache_dir() / "files"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def skip(path: pathlib.Path, data: track.PvtData) -> bool:
|
|
85
|
+
return own_member_skip()(path, data)
|
|
86
|
+
|
|
87
|
+
imports = track.discover_import_pvt_files(root, skip=skip)
|
|
75
88
|
pairs = list[tuple[pathlib.Path, track.PvtData]]()
|
|
76
89
|
for data_path_str, pvt_data in imports.items():
|
|
77
90
|
pvt_path = track.get_pvt_path(pathlib.Path(data_path_str))
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import functools
|
|
4
5
|
import pathlib
|
|
5
6
|
from typing import TYPE_CHECKING, Literal, TypedDict
|
|
6
7
|
|
|
@@ -17,8 +18,23 @@ from pivot.storage import listing, lock, track
|
|
|
17
18
|
from pivot.types import HashInfo, LockData, PipelineStatus, PipelineStatusInfo, is_dir_hash
|
|
18
19
|
|
|
19
20
|
if TYPE_CHECKING:
|
|
21
|
+
from collections.abc import Callable
|
|
22
|
+
|
|
20
23
|
from pivot.remote import protocol
|
|
21
24
|
|
|
25
|
+
# The project's tracked files by data path (``track.discover_pvt_files``), read when first
|
|
26
|
+
# asked for, and once for all of verify's own checks (the status explanations verify reads
|
|
27
|
+
# scan once more, in ``status``)
|
|
28
|
+
TrackedFiles = Callable[[], dict[str, track.PvtData]]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _project_tracked_files(skip: track.PvtSkip | None) -> TrackedFiles:
|
|
32
|
+
"""The project's tracked files, scanned for ``.pvt`` files once, when first asked for.
|
|
33
|
+
``skip`` leaves out a stage's own ``*.pvt`` member of its output, which tracks nothing."""
|
|
34
|
+
return functools.cache(
|
|
35
|
+
functools.partial(track.discover_pvt_files, project.get_project_root(), skip=skip)
|
|
36
|
+
)
|
|
37
|
+
|
|
22
38
|
|
|
23
39
|
VerifyStatus = Literal["passed", "failed"]
|
|
24
40
|
|
|
@@ -57,7 +73,7 @@ def _object_hashes(
|
|
|
57
73
|
if not is_dir_hash(hash_info):
|
|
58
74
|
return result
|
|
59
75
|
try:
|
|
60
|
-
members = listing.read(committed, hash_info, files_dir,
|
|
76
|
+
members = listing.read(committed, hash_info, files_dir, on_stale=listing.OnStale.REFUSE)
|
|
61
77
|
except exceptions.ListingMissingError:
|
|
62
78
|
return result
|
|
63
79
|
for entry in members["files"]:
|
|
@@ -73,18 +89,22 @@ def _check_tracked_listing(
|
|
|
73
89
|
if pvt_data is None or "nfiles" not in pvt_data:
|
|
74
90
|
return
|
|
75
91
|
try:
|
|
76
|
-
track.tracked_members(data_path, pvt_data, files_dir,
|
|
92
|
+
track.tracked_members(data_path, pvt_data, files_dir, on_stale=listing.OnStale.REFUSE)
|
|
77
93
|
except exceptions.ListingMissingError:
|
|
78
94
|
return
|
|
79
95
|
|
|
80
96
|
|
|
81
|
-
def _tracked_listing_failures(
|
|
97
|
+
def _tracked_listing_failures(
|
|
98
|
+
files_dir: pathlib.Path, tracked_files: TrackedFiles
|
|
99
|
+
) -> list[StageVerifyInfo]:
|
|
82
100
|
"""A failed result, named by its path, for each tracked directory in the project whose
|
|
83
101
|
committed ``.pvt.listing`` cannot be used (``_check_tracked_listing``), whether or not a
|
|
84
|
-
stage reads it: a fresh clone would have no cache to restore it from.
|
|
102
|
+
stage reads it: a fresh clone would have no cache to restore it from. ``tracked_files``
|
|
103
|
+
leaves out a stage's own ``*.pvt`` member of its output, which tracks nothing and whose
|
|
104
|
+
directory a restore of that output reads from the lock's listing."""
|
|
85
105
|
project_root = project.get_project_root()
|
|
86
106
|
failures = list[StageVerifyInfo]()
|
|
87
|
-
for path, pvt_data in sorted(
|
|
107
|
+
for path, pvt_data in sorted(tracked_files().items()):
|
|
88
108
|
try:
|
|
89
109
|
_check_tracked_listing(pathlib.Path(path), pvt_data, files_dir)
|
|
90
110
|
except exceptions.CorruptListingError as e:
|
|
@@ -99,17 +119,57 @@ def _tracked_listing_failures(files_dir: pathlib.Path) -> list[StageVerifyInfo]:
|
|
|
99
119
|
return failures
|
|
100
120
|
|
|
101
121
|
|
|
122
|
+
def _tracked_input_objects(
|
|
123
|
+
input_path: str, tracked: dict[str, track.PvtData], files_dir: pathlib.Path
|
|
124
|
+
) -> dict[str, str]:
|
|
125
|
+
"""The cache objects that restore the tracked data holding the pipeline input at
|
|
126
|
+
``input_path`` (project-relative), by workspace path: every tracked file or directory at,
|
|
127
|
+
under or above it, a directory's listing object and each file it lists.
|
|
128
|
+
|
|
129
|
+
These are what pull fetches for the input (``sync.get_target_hashes``) and what checkout
|
|
130
|
+
restores it from. The lock's dependency hash of a directory input leaves out what
|
|
131
|
+
``.pivotignore`` names, and its listing object is written only to the cache of the machine
|
|
132
|
+
that ran the stage; it is derived from the whole listing wherever it is needed
|
|
133
|
+
(``WorkspaceStore.recorded_input_hash``), so it is never required from a remote. Imports
|
|
134
|
+
are left out, as pull leaves them out. Raises CorruptListingError for a ``.pvt.listing``
|
|
135
|
+
that cannot be used.
|
|
136
|
+
"""
|
|
137
|
+
target = pathlib.PurePosixPath(input_path)
|
|
138
|
+
objects = dict[str, str]()
|
|
139
|
+
for data_path, pvt_data in tracked.items():
|
|
140
|
+
if track.is_import(pvt_data):
|
|
141
|
+
continue
|
|
142
|
+
relpath = pathlib.PurePosixPath(project.to_relative_path(data_path))
|
|
143
|
+
if not (relpath.is_relative_to(target) or target.is_relative_to(relpath)):
|
|
144
|
+
continue
|
|
145
|
+
record = track.pvt_to_hash_info(pvt_data)
|
|
146
|
+
objects[str(relpath)] = record["hash"]
|
|
147
|
+
if not is_dir_hash(record):
|
|
148
|
+
continue
|
|
149
|
+
try:
|
|
150
|
+
members = track.tracked_members(
|
|
151
|
+
pathlib.Path(data_path), pvt_data, files_dir, on_stale=listing.OnStale.REFUSE
|
|
152
|
+
)
|
|
153
|
+
except exceptions.ListingMissingError:
|
|
154
|
+
continue
|
|
155
|
+
for entry in members["files"]:
|
|
156
|
+
objects[str(relpath / entry["relpath"])] = entry["hash"]
|
|
157
|
+
return objects
|
|
158
|
+
|
|
159
|
+
|
|
102
160
|
def _get_stage_lock_hashes(
|
|
103
|
-
stage_name: str,
|
|
161
|
+
stage_name: str, tracked_files: TrackedFiles
|
|
104
162
|
) -> tuple[dict[str, str], dict[str, str]]:
|
|
105
163
|
"""Get output and dep cache-object hashes from a stage's lock file and listings.
|
|
106
164
|
|
|
107
165
|
Returns (output_hashes, dep_hashes) where each is {path: hash}.
|
|
108
166
|
|
|
109
|
-
For both outputs and deps, a directory contributes its listing object and its files
|
|
110
|
-
|
|
111
|
-
(
|
|
112
|
-
|
|
167
|
+
For both outputs and deps, a directory contributes its listing object and its files. A
|
|
168
|
+
directory input held by tracked data contributes what restores that data instead
|
|
169
|
+
(``_tracked_input_objects``, over ``tracked_files``); a directory input `pivot import`
|
|
170
|
+
wrote has its ``.pvt.listing`` checked. Non-cached outputs (e.g. Metric with cache=False)
|
|
171
|
+
are excluded — they are git-tracked, not in cache. Raises CorruptListingError, naming the
|
|
172
|
+
file, for a listing that cannot be used.
|
|
113
173
|
"""
|
|
114
174
|
stage_info = cli_helpers.get_stage(stage_name)
|
|
115
175
|
default_state_dir = config.locate_state_dir()
|
|
@@ -174,6 +234,13 @@ def _get_stage_lock_hashes(
|
|
|
174
234
|
committed = lock.committed_listing(producer_lock, committed)
|
|
175
235
|
else:
|
|
176
236
|
dep_path = workspace_store.checkout(ref)
|
|
237
|
+
if is_dir_hash(hash_info):
|
|
238
|
+
restore_objects = _tracked_input_objects(
|
|
239
|
+
project.to_relative_path(dep_path), tracked_files(), files_dir
|
|
240
|
+
)
|
|
241
|
+
if restore_objects:
|
|
242
|
+
dep_hashes.update(restore_objects)
|
|
243
|
+
continue
|
|
177
244
|
committed = lock.committed_listing(
|
|
178
245
|
lock_data,
|
|
179
246
|
listing.input_listing_path(
|
|
@@ -195,13 +262,14 @@ def _get_stage_lock_hashes(
|
|
|
195
262
|
def _get_stage_missing_workspace_files(
|
|
196
263
|
stage_name: str,
|
|
197
264
|
project_root: pathlib.Path,
|
|
265
|
+
tracked_files: TrackedFiles,
|
|
198
266
|
) -> list[str]:
|
|
199
267
|
"""Get output workspace files that are missing from disk for a non-stale stage.
|
|
200
268
|
|
|
201
269
|
Returns list of relative paths for output files that don't exist on disk.
|
|
202
270
|
Excludes metric outputs (git-tracked, not workspace files).
|
|
203
271
|
"""
|
|
204
|
-
output_hashes, _ = _get_stage_lock_hashes(stage_name)
|
|
272
|
+
output_hashes, _ = _get_stage_lock_hashes(stage_name, tracked_files)
|
|
205
273
|
missing = list[str]()
|
|
206
274
|
for rel_path in output_hashes:
|
|
207
275
|
if not (project_root / rel_path).exists():
|
|
@@ -232,13 +300,14 @@ def _get_stage_missing_hashes(
|
|
|
232
300
|
local_hashes: set[str],
|
|
233
301
|
allow_missing: bool,
|
|
234
302
|
project_root: pathlib.Path,
|
|
303
|
+
tracked_files: TrackedFiles,
|
|
235
304
|
) -> dict[str, list[str]]:
|
|
236
305
|
"""Get hashes missing from local cache for a stage.
|
|
237
306
|
|
|
238
307
|
Returns {hash: [paths]} for hashes not in local cache.
|
|
239
308
|
When allow_missing=True, also includes deps missing locally.
|
|
240
309
|
"""
|
|
241
|
-
output_hashes, dep_hashes = _get_stage_lock_hashes(stage_name)
|
|
310
|
+
output_hashes, dep_hashes = _get_stage_lock_hashes(stage_name, tracked_files)
|
|
242
311
|
hash_to_paths = dict[str, list[str]]()
|
|
243
312
|
|
|
244
313
|
# Check outputs: must be in local cache or (with allow_missing) on remote
|
|
@@ -309,6 +378,7 @@ def _verify_stages(
|
|
|
309
378
|
pipeline_status: list[PipelineStatusInfo],
|
|
310
379
|
cache_dir: pathlib.Path,
|
|
311
380
|
allow_missing: bool,
|
|
381
|
+
tracked_files: TrackedFiles,
|
|
312
382
|
) -> tuple[bool, list[StageVerifyInfo]]:
|
|
313
383
|
"""Verify all stages and return pass/fail status with details.
|
|
314
384
|
|
|
@@ -329,7 +399,7 @@ def _verify_stages(
|
|
|
329
399
|
continue
|
|
330
400
|
try:
|
|
331
401
|
hash_to_paths = _get_stage_missing_hashes(
|
|
332
|
-
stage_info["name"], local_hashes, allow_missing, project_root
|
|
402
|
+
stage_info["name"], local_hashes, allow_missing, project_root, tracked_files
|
|
333
403
|
)
|
|
334
404
|
except exceptions.CorruptListingError as e:
|
|
335
405
|
listing_problems[stage_info["name"]] = f"{e}. {e.REMEDY}"
|
|
@@ -408,7 +478,9 @@ def _verify_stages(
|
|
|
408
478
|
# This catches the case where outputs were deleted from the workspace
|
|
409
479
|
# but the hash is still in the local cache.
|
|
410
480
|
if not missing_files and not allow_missing:
|
|
411
|
-
missing_workspace = _get_stage_missing_workspace_files(
|
|
481
|
+
missing_workspace = _get_stage_missing_workspace_files(
|
|
482
|
+
stage_name, project_root, tracked_files
|
|
483
|
+
)
|
|
412
484
|
if missing_workspace:
|
|
413
485
|
missing_files = missing_workspace
|
|
414
486
|
|
|
@@ -503,10 +575,12 @@ def verify(
|
|
|
503
575
|
if not pipeline_status:
|
|
504
576
|
raise click.ClickException("No stages to verify.")
|
|
505
577
|
|
|
506
|
-
# Verify stages
|
|
507
|
-
|
|
578
|
+
# Verify stages; verify's own checks share one scan of the project's .pvt files. A stage's
|
|
579
|
+
# own .pvt member of its output, in any pipeline, is no tracked file or directory.
|
|
580
|
+
tracked_files = _project_tracked_files(cli_helpers.own_member_skip(cache_dir / "files"))
|
|
581
|
+
passed, results = _verify_stages(pipeline_status, cache_dir, allow_missing, tracked_files)
|
|
508
582
|
if stages_list is None:
|
|
509
|
-
tracked_failures = _tracked_listing_failures(cache_dir / "files")
|
|
583
|
+
tracked_failures = _tracked_listing_failures(cache_dir / "files", tracked_files)
|
|
510
584
|
results.extend(tracked_failures)
|
|
511
585
|
passed = passed and not tracked_failures
|
|
512
586
|
|
|
@@ -14,13 +14,13 @@ import py_compile
|
|
|
14
14
|
import sys
|
|
15
15
|
import types as _stdlib_types
|
|
16
16
|
import zipimport
|
|
17
|
-
from typing import TYPE_CHECKING, Final, cast, override
|
|
17
|
+
from typing import TYPE_CHECKING, Final, NamedTuple, cast, override
|
|
18
18
|
|
|
19
19
|
import cloudpickle
|
|
20
20
|
|
|
21
21
|
from pivot import exceptions, fingerprint, loaded_source, metrics, project, types
|
|
22
22
|
from pivot import state_dir as state_dir_mod
|
|
23
|
-
from pivot.registry import PipelineLike
|
|
23
|
+
from pivot.registry import PipelineLike, RegistryStageInfo
|
|
24
24
|
from pivot.storage import ownership
|
|
25
25
|
from pivot.storage import store as store_mod
|
|
26
26
|
|
|
@@ -1008,9 +1008,16 @@ def _log_skipped_module(strict: bool, message: str) -> None:
|
|
|
1008
1008
|
logger.warning(message)
|
|
1009
1009
|
|
|
1010
1010
|
|
|
1011
|
-
|
|
1011
|
+
class DeclaredOutputs(NamedTuple):
|
|
1012
|
+
"""Every pipeline's outputs (``declared_outputs``) and the stages that declare them."""
|
|
1013
|
+
|
|
1014
|
+
outputs: dict[str, ownership.DeclaredOutput]
|
|
1015
|
+
stages: dict[str, RegistryStageInfo]
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def declared_outputs(project_root: pathlib.Path) -> DeclaredOutputs:
|
|
1012
1019
|
"""Map every output declared by every pipeline in the project to where it resolves and the
|
|
1013
|
-
stage that writes it.
|
|
1020
|
+
stage that writes it, beside every pipeline's stages.
|
|
1014
1021
|
|
|
1015
1022
|
Pipelines are discovered as ``--all`` discovers them, so an output of a pipeline the command
|
|
1016
1023
|
did not load still counts: ``track`` and ``import`` refuse a path overlapping any of them. A
|
|
@@ -1027,7 +1034,7 @@ def declared_outputs(project_root: pathlib.Path) -> dict[str, ownership.Declared
|
|
|
1027
1034
|
except DiscoveryError as exc:
|
|
1028
1035
|
raise exceptions.ValidationError(str(exc)) from exc
|
|
1029
1036
|
if pipeline is None:
|
|
1030
|
-
return {}
|
|
1037
|
+
return DeclaredOutputs({}, {})
|
|
1031
1038
|
|
|
1032
1039
|
workspace = store_mod.WorkspaceStore(
|
|
1033
1040
|
project_root=project_root,
|
|
@@ -1053,7 +1060,8 @@ def declared_outputs(project_root: pathlib.Path) -> dict[str, ownership.Declared
|
|
|
1053
1060
|
raise exceptions.ValidationError(
|
|
1054
1061
|
"Cannot resolve stage outputs (fix these first):\n" + "\n".join(errors)
|
|
1055
1062
|
)
|
|
1056
|
-
|
|
1063
|
+
stages = {stage_name: pipeline.get_stage(stage_name) for stage_name in pipeline.list_stages()}
|
|
1064
|
+
return DeclaredOutputs(resolved, stages)
|
|
1057
1065
|
|
|
1058
1066
|
|
|
1059
1067
|
def load_pipeline_from_path(path: pathlib.Path) -> PipelineLike | None:
|
|
@@ -26,6 +26,7 @@ from pivot import (
|
|
|
26
26
|
config,
|
|
27
27
|
exceptions,
|
|
28
28
|
fingerprint,
|
|
29
|
+
ignore,
|
|
29
30
|
parameters,
|
|
30
31
|
project,
|
|
31
32
|
registry,
|
|
@@ -60,7 +61,7 @@ from pivot.engine.types import (
|
|
|
60
61
|
)
|
|
61
62
|
from pivot.executor import core as executor_core
|
|
62
63
|
from pivot.executor import worker
|
|
63
|
-
from pivot.storage import ownership
|
|
64
|
+
from pivot.storage import member_skip, ownership
|
|
64
65
|
from pivot.storage import state as state_mod
|
|
65
66
|
from pivot.storage import store as store_mod
|
|
66
67
|
from pivot.types import (
|
|
@@ -289,6 +290,7 @@ class Engine:
|
|
|
289
290
|
_graph: nx.DiGraph[str] | None
|
|
290
291
|
_watch_coordinator: engine_watch.WatchCoordinator | None
|
|
291
292
|
_watch_output_stats: dict[pathlib.Path, _WatchOutputSnapshot]
|
|
293
|
+
_ignore_filter: ignore.IgnoreFilter | None
|
|
292
294
|
_scheduler: engine_scheduler.Scheduler
|
|
293
295
|
_cancel_event: anyio.Event
|
|
294
296
|
_stage_indices: dict[str, tuple[int, int]]
|
|
@@ -363,6 +365,7 @@ class Engine:
|
|
|
363
365
|
self._graph = None
|
|
364
366
|
self._watch_coordinator = None
|
|
365
367
|
self._watch_output_stats = dict[pathlib.Path, _WatchOutputSnapshot]()
|
|
368
|
+
self._ignore_filter = None
|
|
366
369
|
self._scheduler = engine_scheduler.Scheduler()
|
|
367
370
|
self._cancel_event = anyio.Event()
|
|
368
371
|
self._stage_indices = dict[str, tuple[int, int]]()
|
|
@@ -1134,12 +1137,19 @@ class Engine:
|
|
|
1134
1137
|
if cache_dir is None:
|
|
1135
1138
|
cache_dir = config.get_cache_dir()
|
|
1136
1139
|
|
|
1137
|
-
# Verify tracked files before building DAG
|
|
1140
|
+
# Verify tracked files before building DAG: a stage's own .pvt member of its output is
|
|
1141
|
+
# none, the loaded pipeline's by the registry and any other's by its lock
|
|
1138
1142
|
project_root = project.get_project_root()
|
|
1139
|
-
|
|
1143
|
+
all_stages = self._get_all_stages()
|
|
1144
|
+
executor_core.verify_tracked_files(
|
|
1145
|
+
project_root,
|
|
1146
|
+
checkout_missing=checkout_missing,
|
|
1147
|
+
skip=member_skip.own_member_skip(
|
|
1148
|
+
project_root, all_stages, config.get_state_dir(), cache_dir / "files"
|
|
1149
|
+
),
|
|
1150
|
+
)
|
|
1140
1151
|
|
|
1141
1152
|
# Build bipartite graph (single source of truth)
|
|
1142
|
-
all_stages = self._get_all_stages()
|
|
1143
1153
|
graph = engine_graph.build_graph(all_stages)
|
|
1144
1154
|
pipeline = self._pipeline
|
|
1145
1155
|
if pipeline is None:
|
|
@@ -1944,7 +1954,7 @@ class Engine:
|
|
|
1944
1954
|
return False
|
|
1945
1955
|
|
|
1946
1956
|
stage_state_dir = registry.get_stage_state_dir(stage_info, state_dir)
|
|
1947
|
-
store = store_mod.store_from_spec(store_spec)
|
|
1957
|
+
store = store_mod.store_from_spec(store_spec, ignore_filter=self._get_ignore_filter())
|
|
1948
1958
|
|
|
1949
1959
|
def _read_lock() -> LockData | None:
|
|
1950
1960
|
stages_dir = lock.get_stages_dir(stage_state_dir)
|
|
@@ -2257,6 +2267,7 @@ class Engine:
|
|
|
2257
2267
|
pipeline_name=pipeline.name,
|
|
2258
2268
|
input_bindings=pipeline.input_bindings,
|
|
2259
2269
|
cache_dir=pathlib.Path(worker_info["store_spec"]["cache_dir"]),
|
|
2270
|
+
ignore_filter=self._get_ignore_filter(),
|
|
2260
2271
|
)
|
|
2261
2272
|
lock_data = lock.StageLock(
|
|
2262
2273
|
stage_name, lock.get_stages_dir(worker_info["state_dir"])
|
|
@@ -2962,13 +2973,23 @@ class Engine:
|
|
|
2962
2973
|
project_root=project.get_project_root(),
|
|
2963
2974
|
pipeline_name=pipeline.name,
|
|
2964
2975
|
input_bindings=pipeline.input_bindings,
|
|
2976
|
+
ignore_filter=self._get_ignore_filter(),
|
|
2965
2977
|
)
|
|
2966
2978
|
self._watch_coordinator = engine_watch.WatchCoordinator(
|
|
2967
2979
|
self._graph,
|
|
2968
2980
|
engine_watch.build_artifact_paths(self._get_all_stages(), store),
|
|
2981
|
+
store,
|
|
2969
2982
|
)
|
|
2970
2983
|
return self._watch_coordinator
|
|
2971
2984
|
|
|
2985
|
+
def _get_ignore_filter(self) -> ignore.IgnoreFilter:
|
|
2986
|
+
"""The project's ``.pivotignore`` filter, one for this engine: its skip checks, its
|
|
2987
|
+
explanations and its watch planning all read the file through it, and it reloads the
|
|
2988
|
+
file when its mtime changes."""
|
|
2989
|
+
if self._ignore_filter is None:
|
|
2990
|
+
self._ignore_filter = ignore.IgnoreFilter(project.get_project_root())
|
|
2991
|
+
return self._ignore_filter
|
|
2992
|
+
|
|
2972
2993
|
def _should_filter_path(self, path: pathlib.Path) -> bool:
|
|
2973
2994
|
"""Check if path should be filtered (output of executing stage)."""
|
|
2974
2995
|
coordinator = self._get_watch_coordinator()
|