pivot 0.6.2__tar.gz → 0.6.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. {pivot-0.6.2 → pivot-0.6.4}/PKG-INFO +1 -2
  2. {pivot-0.6.2 → pivot-0.6.4}/pyproject.toml +1 -2
  3. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/AGENTS.md +1 -1
  4. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/checkout.py +5 -3
  5. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/helpers.py +20 -0
  6. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/status.py +11 -2
  7. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/track.py +48 -10
  8. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/update.py +16 -3
  9. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/verify.py +91 -17
  10. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/discovery.py +14 -6
  11. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/engine.py +26 -5
  12. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/watch.py +53 -2
  13. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/core.py +9 -2
  14. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/worker.py +20 -36
  15. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/explain.py +23 -4
  16. pivot-0.6.4/src/pivot/ignore.py +906 -0
  17. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/import_artifact.py +1 -1
  18. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/sync.py +63 -22
  19. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/state_dir.py +17 -0
  20. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/status.py +27 -10
  21. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/AGENTS.md +17 -2
  22. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/cache.py +61 -12
  23. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/listing.py +35 -17
  24. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/lock.py +215 -36
  25. pivot-0.6.4/src/pivot/storage/member_skip.py +163 -0
  26. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/ownership.py +112 -45
  27. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/store.py +71 -9
  28. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/track.py +34 -13
  29. pivot-0.6.2/src/pivot/ignore.py +0 -358
  30. {pivot-0.6.2 → pivot-0.6.4}/LICENSE +0 -0
  31. {pivot-0.6.2 → pivot-0.6.4}/README.md +0 -0
  32. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/__init__.py +0 -0
  33. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/ast_utils.py +0 -0
  34. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/CLAUDE.md +0 -0
  35. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/__init__.py +0 -0
  36. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/_run_common.py +0 -0
  37. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/check_ignore.py +0 -0
  38. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/commit.py +0 -0
  39. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/completion.py +0 -0
  40. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/config.py +0 -0
  41. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/console.py +0 -0
  42. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/dag.py +0 -0
  43. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/data.py +0 -0
  44. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/decorators.py +0 -0
  45. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/doctor.py +0 -0
  46. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/errors.py +0 -0
  47. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/fingerprint.py +0 -0
  48. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/history.py +0 -0
  49. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/import_cmd.py +0 -0
  50. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/init.py +0 -0
  51. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/list.py +0 -0
  52. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/metrics.py +0 -0
  53. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/params.py +0 -0
  54. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/plots.py +0 -0
  55. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/remote.py +0 -0
  56. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/repro.py +0 -0
  57. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/run.py +0 -0
  58. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/cli/targets.py +0 -0
  59. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/compose.py +0 -0
  60. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/config/__init__.py +0 -0
  61. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/config/io.py +0 -0
  62. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/config/models.py +0 -0
  63. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/config_files.py +0 -0
  64. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/dag/__init__.py +0 -0
  65. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/dag/render.py +0 -0
  66. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/decorators.py +0 -0
  67. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/display.py +0 -0
  68. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/__init__.py +0 -0
  69. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/agent_rpc.py +0 -0
  70. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/graph.py +0 -0
  71. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/scheduler.py +0 -0
  72. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/sinks.py +0 -0
  73. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/sources.py +0 -0
  74. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/types.py +0 -0
  75. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/engine/worker_pool.py +0 -0
  76. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/exceptions.py +0 -0
  77. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/AGENTS.md +0 -0
  78. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/CLAUDE.md +0 -0
  79. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/__init__.py +0 -0
  80. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/executor/commit.py +0 -0
  81. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/fingerprint.py +0 -0
  82. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/git.py +0 -0
  83. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/loaded_source.py +0 -0
  84. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/loaders.py +0 -0
  85. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/metrics.py +0 -0
  86. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/names.py +0 -0
  87. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/parameters.py +0 -0
  88. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/path_policy.py +0 -0
  89. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/project.py +0 -0
  90. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/py.typed +0 -0
  91. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/registry.py +0 -0
  92. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/AGENTS.md +0 -0
  93. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/__init__.py +0 -0
  94. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/batch.py +0 -0
  95. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/config.py +0 -0
  96. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/gcs.py +0 -0
  97. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/git_archive.py +0 -0
  98. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/github.py +0 -0
  99. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/keys.py +0 -0
  100. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/protocol.py +0 -0
  101. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/remote/storage.py +0 -0
  102. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/run_history.py +0 -0
  103. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/__init__.py +0 -0
  104. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/common.py +0 -0
  105. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/data.py +0 -0
  106. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/metrics.py +0 -0
  107. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/params.py +0 -0
  108. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/plots.py +0 -0
  109. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/show/selection.py +0 -0
  110. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/skip.py +0 -0
  111. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/stage_def.py +0 -0
  112. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/stage_io.py +0 -0
  113. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/CLAUDE.md +0 -0
  114. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/__init__.py +0 -0
  115. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/artifact_lock.py +0 -0
  116. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/mounts.py +0 -0
  117. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/presentation.py +0 -0
  118. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/restore.py +0 -0
  119. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/storage/state.py +0 -0
  120. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/types.py +0 -0
  121. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/unix_socket.py +0 -0
  122. {pivot-0.6.2 → pivot-0.6.4}/src/pivot/yaml_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pivot
3
- Version: 0.6.2
3
+ Version: 0.6.4
4
4
  Summary: High-performance Python pipeline tool with automatic code change detection
5
5
  Author: Sami Jawhar
6
6
  Author-email: Sami Jawhar <sami@metr.org>
@@ -21,7 +21,6 @@ Requires-Dist: grandalf>=0.8
21
21
  Requires-Dist: lmdb>=1.4.0
22
22
  Requires-Dist: loky>=3.5.6,<3.8
23
23
  Requires-Dist: networkx>=3.0
24
- Requires-Dist: pathspec>=0.12.0
25
24
  Requires-Dist: pandas>=2.0
26
25
  Requires-Dist: psutil>=7.0
27
26
  Requires-Dist: pydantic>=2.13
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "pivot"
3
- version = "0.6.2"
3
+ version = "0.6.4"
4
4
  description = "High-performance Python pipeline tool with automatic code change detection"
5
5
  requires-python = ">=3.13,<3.14"
6
6
  readme = "README.md"
@@ -25,7 +25,6 @@ dependencies = [
25
25
  "lmdb>=1.4.0",
26
26
  "loky>=3.5.6,<3.8",
27
27
  "networkx>=3.0",
28
- "pathspec>=0.12.0",
29
28
  "pandas>=2.0",
30
29
  "psutil>=7.0",
31
30
  "pydantic>=2.13",
@@ -37,7 +37,7 @@ Set `auto_discover=False` only for commands that don't use the stage registry:
37
37
  | run | True (default) | Need registry to find stages |
38
38
  | list, dag | True (default), allow_all=True | Need registry; --all for multi-pipeline |
39
39
  | checkout | True (default) | Need registry; handles None pipeline gracefully |
40
- | track | False | Discovers every pipeline itself (`discovery.declared_outputs`, as `--all` does) and refuses a new path overlapping any of their outputs |
40
+ | track | False | Discovers every pipeline itself (`discovery.declared_outputs`, as `--all` does) and refuses a new path overlapping any of their outputs, or one whose `.pvt` sidecar the lock of a stage that no longer exists records as that stage's file |
41
41
  | status | True (default), allow_all=True | Handles None pipeline gracefully (skips stages section) |
42
42
  | init, doctor, check-ignore | False | No stage data needed |
43
43
  | history, show | False | Read run history only |
@@ -755,9 +755,6 @@ def checkout(
755
755
  [cache.CheckoutMode(checkout_mode)] if checkout_mode else config.get_checkout_mode_order()
756
756
  )
757
757
 
758
- # Discover tracked files
759
- tracked_files = track.discover_pvt_files(project_root)
760
-
761
758
  # Get stage output info from lock files (cached outputs only)
762
759
  pipeline = cli_decorators.get_pipeline_from_context()
763
760
  stage_outputs = dict[str, Expected]()
@@ -765,6 +762,11 @@ def checkout(
765
762
  if pipeline is not None:
766
763
  stage_outputs, output_state_dirs = _get_stage_output_info(cache_dir)
767
764
 
765
+ # Discover tracked files: a stage's own .pvt member of its output, in any pipeline, is none
766
+ tracked_files = track.discover_pvt_files(
767
+ project_root, skip=cli_helpers.own_member_skip(cache_dir)
768
+ )
769
+
768
770
  state_dir = config.locate_state_dir()
769
771
 
770
772
  # Every target adds its read or restored file hashes to one thread-safe batch. Saving once
@@ -9,10 +9,13 @@ import click
9
9
  from tqdm.asyncio import tqdm as async_tqdm
10
10
 
11
11
  if TYPE_CHECKING:
12
+ import pathlib
13
+
12
14
  from networkx import DiGraph
13
15
 
14
16
  from pivot.cli import CliContext
15
17
  from pivot.registry import PipelineLike, RegistryStageInfo
18
+ from pivot.storage import track
16
19
  from pivot.storage.store import WorkspaceStore
17
20
 
18
21
  from pivot import display, exceptions, names, project
@@ -83,6 +86,23 @@ def require_workspace_store() -> WorkspaceStore:
83
86
  return store
84
87
 
85
88
 
89
+ def own_member_skip(cache_dir: pathlib.Path) -> track.PvtSkip:
90
+ """The skip that leaves a stage's own ``*.pvt`` member out of what this command discovers
91
+ (``member_skip.own_member_skip``): the loaded pipeline's stages (none without a pipeline)
92
+ decide their outputs, and the directory outputs other stages' locks record decide every
93
+ other. ``cache_dir`` is the cache object root."""
94
+ from pivot import config
95
+ from pivot.storage import member_skip
96
+
97
+ pipeline = cli_decorators.get_pipeline_from_context()
98
+ return member_skip.own_member_skip(
99
+ project.get_project_root(),
100
+ get_all_stages() if pipeline is not None else {},
101
+ config.locate_state_dir(),
102
+ cache_dir,
103
+ )
104
+
105
+
86
106
  def _json_default(obj: Any) -> Any: # noqa: ANN401 - json.dumps default requires Any
87
107
  """Handle non-standard JSON types in JSONL output."""
88
108
  if isinstance(obj, enum.Enum):
@@ -91,6 +91,13 @@ def status(
91
91
  show_tracked = show_all or tracked_only
92
92
  show_remote = remote_only or remote
93
93
 
94
+ # A stage's own .pvt member of its output, in any pipeline, is no tracked file or import
95
+ skip = (
96
+ cli_helpers.own_member_skip(resolved_cache_dir / "files")
97
+ if show_tracked or check_imports
98
+ else None
99
+ )
100
+
94
101
  # When --explain is used, fetch detailed explanations instead of basic status
95
102
  pipeline_status = list[PipelineStatusInfo]()
96
103
  pipeline_explanations = list[StageExplanation]()
@@ -141,7 +148,9 @@ def status(
141
148
  pbar.update(1)
142
149
 
143
150
  try:
144
- tracked_status = status_mod.get_tracked_files_status(project_root, on_progress)
151
+ tracked_status = status_mod.get_tracked_files_status(
152
+ project_root, on_progress, skip=skip
153
+ )
145
154
  finally:
146
155
  if pbar is not None:
147
156
  pbar.close()
@@ -161,7 +170,7 @@ def status(
161
170
 
162
171
  import_statuses = list[ImportStatusInfo]()
163
172
  if check_imports:
164
- import_statuses = status_mod.get_import_status(project_root)
173
+ import_statuses = status_mod.get_import_status(project_root, skip=skip)
165
174
 
166
175
  # Compute counts once for suggestions and output
167
176
  if explain and pipeline_explanations:
@@ -2,16 +2,20 @@ from __future__ import annotations
2
2
 
3
3
  import os
4
4
  import pathlib
5
- from typing import TypedDict
5
+ import shlex
6
+ from typing import TYPE_CHECKING, TypedDict
6
7
 
7
8
  import click
8
9
 
9
10
  from pivot import config, discovery, git, project
10
11
  from pivot.cli import decorators as cli_decorators
11
12
  from pivot.cli import helpers as cli_helpers
12
- from pivot.storage import cache, listing, ownership, state
13
+ from pivot.storage import cache, listing, member_skip, ownership, state
13
14
  from pivot.storage import track as track_mod
14
15
 
16
+ if TYPE_CHECKING:
17
+ from collections.abc import Callable
18
+
15
19
 
16
20
  class TrackResult(TypedDict):
17
21
  """Result of tracking a single path."""
@@ -20,6 +24,26 @@ class TrackResult(TypedDict):
20
24
  tracked_path: str
21
25
 
22
26
 
27
+ def _orphan_refusal(path_str: str, orphan: member_skip.OrphanedMember) -> str:
28
+ """Why ``track`` refuses a path whose sidecar a deleted stage's lock records."""
29
+ project_root = project.get_project_root()
30
+ locks = [f"'{ownership.display_path(found, project_root)}'" for found in orphan.lock_files]
31
+ if len(locks) == 1:
32
+ recorded = f"the lock {locks[0]} records"
33
+ remedy = "remove that lock"
34
+ else:
35
+ recorded = f"the locks {' and '.join(locks)} record"
36
+ remedy = "remove those locks"
37
+ sidecar = ownership.display_path(orphan.sidecar, project_root)
38
+ return (
39
+ f"Cannot track '{path_str}': {recorded} '{sidecar}' as a file of stage "
40
+ + f"'{orphan.stage_name}', a stage that no longer exists in any pipeline. Pivot reads "
41
+ + "that stage's files there as its own, not as tracked files. If the stage is gone for "
42
+ + f"good, {remedy}, then run `pivot track --force {shlex.quote(path_str)}`, which tracks "
43
+ + "the file whatever its .pvt then says."
44
+ )
45
+
46
+
23
47
  def _tracking_files(data_path: pathlib.Path) -> tuple[pathlib.Path, pathlib.Path]:
24
48
  """The files tracking ``data_path`` writes, or for a file removes: its .pvt and listing."""
25
49
  pvt_path = track_mod.get_pvt_path(data_path)
@@ -86,7 +110,8 @@ def _foreign_listing(
86
110
  def _track_single_path(
87
111
  path_str: str,
88
112
  cache_dir: pathlib.Path,
89
- stage_outputs: dict[str, ownership.DeclaredOutput],
113
+ outputs: dict[str, ownership.DeclaredOutput],
114
+ orphaned_member: Callable[[pathlib.Path], member_skip.OrphanedMember | None],
90
115
  existing_tracked: dict[str, track_mod.PvtData],
91
116
  force: bool,
92
117
  *,
@@ -152,7 +177,7 @@ def _track_single_path(
152
177
  track_mod.validate_pvt_path_name(path.name)
153
178
 
154
179
  # Check every pipeline output. OSError reaches the command error handler, which names path.
155
- overlap = ownership.find_declared_overlap(path, stage_outputs, project_root)
180
+ overlap = ownership.find_declared_overlap(path, outputs, project_root)
156
181
  if overlap is not None:
157
182
  if not already_tracked:
158
183
  raise click.ClickException(
@@ -166,6 +191,11 @@ def _track_single_path(
166
191
  + "its .pvt sidecar outside every stage output (for example under data/raw/); until "
167
192
  + "then Pivot refuses to replace that output."
168
193
  )
194
+ # A sidecar a stage that no longer exists recorded at this path: every command reads it as
195
+ # that stage's own file, by its lock, so a sidecar written there could be one none reads
196
+ orphan = orphaned_member(path)
197
+ if orphan is not None:
198
+ raise click.ClickException(_orphan_refusal(path_str, orphan))
169
199
  warning = "\n".join(warnings) if warnings else None
170
200
 
171
201
  # Hash and cache
@@ -247,19 +277,27 @@ def track(ctx: click.Context, paths: tuple[str, ...], force: bool) -> None:
247
277
  project_root = project.get_project_root()
248
278
  cache_dir = config.get_cache_dir() / "files"
249
279
 
250
- # Every pipeline's outputs, not only the pipeline this directory selects: a file tracked in
251
- # another pipeline's output would be removed when that pipeline runs.
252
- stage_outputs = discovery.declared_outputs(project_root)
280
+ # Every pipeline's outputs and stages, not only the pipeline this directory selects: a file
281
+ # tracked in another pipeline's output would be removed when that pipeline runs.
282
+ declared = discovery.declared_outputs(project_root)
283
+ state_dir = config.locate_state_dir()
284
+ orphaned_member = member_skip.orphaned_member(
285
+ project_root, state_dir, cache_dir, declared.stages
286
+ )
253
287
 
254
- # Discover existing .pvt files
255
- existing_tracked = track_mod.discover_pvt_files(project_root)
288
+ # Discover existing .pvt files: a stage's own .pvt member of its output tracks nothing
289
+ existing_tracked = track_mod.discover_pvt_files(
290
+ project_root,
291
+ skip=member_skip.own_member_skip(project_root, declared.stages, state_dir, cache_dir),
292
+ )
256
293
 
257
294
  read = list[tuple[pathlib.Path, os.stat_result, str]]()
258
295
  for path_str in paths:
259
296
  result = _track_single_path(
260
297
  path_str,
261
298
  cache_dir,
262
- stage_outputs,
299
+ declared.outputs,
300
+ orphaned_member,
263
301
  existing_tracked,
264
302
  force,
265
303
  warn_left_out_symlinks=not quiet,
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import asyncio
4
+ import functools
4
5
  import logging
5
6
  import pathlib
6
7
  from typing import TYPE_CHECKING
@@ -49,8 +50,8 @@ def update(targets: tuple[str, ...], rev: str | None, dry_run: bool) -> None:
49
50
 
50
51
  If no TARGETS specified, updates all imports found in the project.
51
52
  """
52
- from pivot import project
53
- from pivot.storage import track
53
+ from pivot import config, project
54
+ from pivot.storage import member_skip, track
54
55
 
55
56
  root = project.get_project_root()
56
57
 
@@ -71,7 +72,19 @@ def update(targets: tuple[str, ...], rev: str | None, dry_run: bool) -> None:
71
72
  continue
72
73
  pairs.append((pvt_path, pvt_data))
73
74
  else:
74
- imports = track.discover_import_pvt_files(root)
75
+ # A stage's own .pvt member of its output is no import. The command loads no pipeline,
76
+ # so the directory outputs the stages' locks record decide; it reads no config until an
77
+ # import .pvt is found.
78
+ @functools.cache
79
+ def own_member_skip() -> track.PvtSkip:
80
+ return member_skip.own_member_skip(
81
+ root, {}, config.locate_state_dir(), config.locate_cache_dir() / "files"
82
+ )
83
+
84
+ def skip(path: pathlib.Path, data: track.PvtData) -> bool:
85
+ return own_member_skip()(path, data)
86
+
87
+ imports = track.discover_import_pvt_files(root, skip=skip)
75
88
  pairs = list[tuple[pathlib.Path, track.PvtData]]()
76
89
  for data_path_str, pvt_data in imports.items():
77
90
  pvt_path = track.get_pvt_path(pathlib.Path(data_path_str))
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import asyncio
4
+ import functools
4
5
  import pathlib
5
6
  from typing import TYPE_CHECKING, Literal, TypedDict
6
7
 
@@ -17,8 +18,23 @@ from pivot.storage import listing, lock, track
17
18
  from pivot.types import HashInfo, LockData, PipelineStatus, PipelineStatusInfo, is_dir_hash
18
19
 
19
20
  if TYPE_CHECKING:
21
+ from collections.abc import Callable
22
+
20
23
  from pivot.remote import protocol
21
24
 
25
+ # The project's tracked files by data path (``track.discover_pvt_files``), read when first
26
+ # asked for, and once for all of verify's own checks (the status explanations verify reads
27
+ # scan once more, in ``status``)
28
+ TrackedFiles = Callable[[], dict[str, track.PvtData]]
29
+
30
+
31
+ def _project_tracked_files(skip: track.PvtSkip | None) -> TrackedFiles:
32
+ """The project's tracked files, scanned for ``.pvt`` files once, when first asked for.
33
+ ``skip`` leaves out a stage's own ``*.pvt`` member of its output, which tracks nothing."""
34
+ return functools.cache(
35
+ functools.partial(track.discover_pvt_files, project.get_project_root(), skip=skip)
36
+ )
37
+
22
38
 
23
39
  VerifyStatus = Literal["passed", "failed"]
24
40
 
@@ -57,7 +73,7 @@ def _object_hashes(
57
73
  if not is_dir_hash(hash_info):
58
74
  return result
59
75
  try:
60
- members = listing.read(committed, hash_info, files_dir, strict=True)
76
+ members = listing.read(committed, hash_info, files_dir, on_stale=listing.OnStale.REFUSE)
61
77
  except exceptions.ListingMissingError:
62
78
  return result
63
79
  for entry in members["files"]:
@@ -73,18 +89,22 @@ def _check_tracked_listing(
73
89
  if pvt_data is None or "nfiles" not in pvt_data:
74
90
  return
75
91
  try:
76
- track.tracked_members(data_path, pvt_data, files_dir, strict=True)
92
+ track.tracked_members(data_path, pvt_data, files_dir, on_stale=listing.OnStale.REFUSE)
77
93
  except exceptions.ListingMissingError:
78
94
  return
79
95
 
80
96
 
81
- def _tracked_listing_failures(files_dir: pathlib.Path) -> list[StageVerifyInfo]:
97
+ def _tracked_listing_failures(
98
+ files_dir: pathlib.Path, tracked_files: TrackedFiles
99
+ ) -> list[StageVerifyInfo]:
82
100
  """A failed result, named by its path, for each tracked directory in the project whose
83
101
  committed ``.pvt.listing`` cannot be used (``_check_tracked_listing``), whether or not a
84
- stage reads it: a fresh clone would have no cache to restore it from."""
102
+ stage reads it: a fresh clone would have no cache to restore it from. ``tracked_files``
103
+ leaves out a stage's own ``*.pvt`` member of its output, which tracks nothing and whose
104
+ directory a restore of that output reads from the lock's listing."""
85
105
  project_root = project.get_project_root()
86
106
  failures = list[StageVerifyInfo]()
87
- for path, pvt_data in sorted(track.discover_pvt_files(project_root).items()):
107
+ for path, pvt_data in sorted(tracked_files().items()):
88
108
  try:
89
109
  _check_tracked_listing(pathlib.Path(path), pvt_data, files_dir)
90
110
  except exceptions.CorruptListingError as e:
@@ -99,17 +119,57 @@ def _tracked_listing_failures(files_dir: pathlib.Path) -> list[StageVerifyInfo]:
99
119
  return failures
100
120
 
101
121
 
122
+ def _tracked_input_objects(
123
+ input_path: str, tracked: dict[str, track.PvtData], files_dir: pathlib.Path
124
+ ) -> dict[str, str]:
125
+ """The cache objects that restore the tracked data holding the pipeline input at
126
+ ``input_path`` (project-relative), by workspace path: every tracked file or directory at,
127
+ under or above it, a directory's listing object and each file it lists.
128
+
129
+ These are what pull fetches for the input (``sync.get_target_hashes``) and what checkout
130
+ restores it from. The lock's dependency hash of a directory input leaves out what
131
+ ``.pivotignore`` names, and its listing object is written only to the cache of the machine
132
+ that ran the stage; it is derived from the whole listing wherever it is needed
133
+ (``WorkspaceStore.recorded_input_hash``), so it is never required from a remote. Imports
134
+ are left out, as pull leaves them out. Raises CorruptListingError for a ``.pvt.listing``
135
+ that cannot be used.
136
+ """
137
+ target = pathlib.PurePosixPath(input_path)
138
+ objects = dict[str, str]()
139
+ for data_path, pvt_data in tracked.items():
140
+ if track.is_import(pvt_data):
141
+ continue
142
+ relpath = pathlib.PurePosixPath(project.to_relative_path(data_path))
143
+ if not (relpath.is_relative_to(target) or target.is_relative_to(relpath)):
144
+ continue
145
+ record = track.pvt_to_hash_info(pvt_data)
146
+ objects[str(relpath)] = record["hash"]
147
+ if not is_dir_hash(record):
148
+ continue
149
+ try:
150
+ members = track.tracked_members(
151
+ pathlib.Path(data_path), pvt_data, files_dir, on_stale=listing.OnStale.REFUSE
152
+ )
153
+ except exceptions.ListingMissingError:
154
+ continue
155
+ for entry in members["files"]:
156
+ objects[str(relpath / entry["relpath"])] = entry["hash"]
157
+ return objects
158
+
159
+
102
160
  def _get_stage_lock_hashes(
103
- stage_name: str,
161
+ stage_name: str, tracked_files: TrackedFiles
104
162
  ) -> tuple[dict[str, str], dict[str, str]]:
105
163
  """Get output and dep cache-object hashes from a stage's lock file and listings.
106
164
 
107
165
  Returns (output_hashes, dep_hashes) where each is {path: hash}.
108
166
 
109
- For both outputs and deps, a directory contributes its listing object and its files; a
110
- tracked directory a dep names has its ``.pvt.listing`` checked too. Non-cached outputs
111
- (e.g. Metric with cache=False) are excluded — they are git-tracked, not in cache. Raises
112
- CorruptListingError, naming the file, for a listing that cannot be used.
167
+ For both outputs and deps, a directory contributes its listing object and its files. A
168
+ directory input held by tracked data contributes what restores that data instead
169
+ (``_tracked_input_objects``, over ``tracked_files``); a directory input `pivot import`
170
+ wrote has its ``.pvt.listing`` checked. Non-cached outputs (e.g. Metric with cache=False)
171
+ are excluded — they are git-tracked, not in cache. Raises CorruptListingError, naming the
172
+ file, for a listing that cannot be used.
113
173
  """
114
174
  stage_info = cli_helpers.get_stage(stage_name)
115
175
  default_state_dir = config.locate_state_dir()
@@ -174,6 +234,13 @@ def _get_stage_lock_hashes(
174
234
  committed = lock.committed_listing(producer_lock, committed)
175
235
  else:
176
236
  dep_path = workspace_store.checkout(ref)
237
+ if is_dir_hash(hash_info):
238
+ restore_objects = _tracked_input_objects(
239
+ project.to_relative_path(dep_path), tracked_files(), files_dir
240
+ )
241
+ if restore_objects:
242
+ dep_hashes.update(restore_objects)
243
+ continue
177
244
  committed = lock.committed_listing(
178
245
  lock_data,
179
246
  listing.input_listing_path(
@@ -195,13 +262,14 @@ def _get_stage_lock_hashes(
195
262
  def _get_stage_missing_workspace_files(
196
263
  stage_name: str,
197
264
  project_root: pathlib.Path,
265
+ tracked_files: TrackedFiles,
198
266
  ) -> list[str]:
199
267
  """Get output workspace files that are missing from disk for a non-stale stage.
200
268
 
201
269
  Returns list of relative paths for output files that don't exist on disk.
202
270
  Excludes metric outputs (git-tracked, not workspace files).
203
271
  """
204
- output_hashes, _ = _get_stage_lock_hashes(stage_name)
272
+ output_hashes, _ = _get_stage_lock_hashes(stage_name, tracked_files)
205
273
  missing = list[str]()
206
274
  for rel_path in output_hashes:
207
275
  if not (project_root / rel_path).exists():
@@ -232,13 +300,14 @@ def _get_stage_missing_hashes(
232
300
  local_hashes: set[str],
233
301
  allow_missing: bool,
234
302
  project_root: pathlib.Path,
303
+ tracked_files: TrackedFiles,
235
304
  ) -> dict[str, list[str]]:
236
305
  """Get hashes missing from local cache for a stage.
237
306
 
238
307
  Returns {hash: [paths]} for hashes not in local cache.
239
308
  When allow_missing=True, also includes deps missing locally.
240
309
  """
241
- output_hashes, dep_hashes = _get_stage_lock_hashes(stage_name)
310
+ output_hashes, dep_hashes = _get_stage_lock_hashes(stage_name, tracked_files)
242
311
  hash_to_paths = dict[str, list[str]]()
243
312
 
244
313
  # Check outputs: must be in local cache or (with allow_missing) on remote
@@ -309,6 +378,7 @@ def _verify_stages(
309
378
  pipeline_status: list[PipelineStatusInfo],
310
379
  cache_dir: pathlib.Path,
311
380
  allow_missing: bool,
381
+ tracked_files: TrackedFiles,
312
382
  ) -> tuple[bool, list[StageVerifyInfo]]:
313
383
  """Verify all stages and return pass/fail status with details.
314
384
 
@@ -329,7 +399,7 @@ def _verify_stages(
329
399
  continue
330
400
  try:
331
401
  hash_to_paths = _get_stage_missing_hashes(
332
- stage_info["name"], local_hashes, allow_missing, project_root
402
+ stage_info["name"], local_hashes, allow_missing, project_root, tracked_files
333
403
  )
334
404
  except exceptions.CorruptListingError as e:
335
405
  listing_problems[stage_info["name"]] = f"{e}. {e.REMEDY}"
@@ -408,7 +478,9 @@ def _verify_stages(
408
478
  # This catches the case where outputs were deleted from the workspace
409
479
  # but the hash is still in the local cache.
410
480
  if not missing_files and not allow_missing:
411
- missing_workspace = _get_stage_missing_workspace_files(stage_name, project_root)
481
+ missing_workspace = _get_stage_missing_workspace_files(
482
+ stage_name, project_root, tracked_files
483
+ )
412
484
  if missing_workspace:
413
485
  missing_files = missing_workspace
414
486
 
@@ -503,10 +575,12 @@ def verify(
503
575
  if not pipeline_status:
504
576
  raise click.ClickException("No stages to verify.")
505
577
 
506
- # Verify stages
507
- passed, results = _verify_stages(pipeline_status, cache_dir, allow_missing)
578
+ # Verify stages; verify's own checks share one scan of the project's .pvt files. A stage's
579
+ # own .pvt member of its output, in any pipeline, is no tracked file or directory.
580
+ tracked_files = _project_tracked_files(cli_helpers.own_member_skip(cache_dir / "files"))
581
+ passed, results = _verify_stages(pipeline_status, cache_dir, allow_missing, tracked_files)
508
582
  if stages_list is None:
509
- tracked_failures = _tracked_listing_failures(cache_dir / "files")
583
+ tracked_failures = _tracked_listing_failures(cache_dir / "files", tracked_files)
510
584
  results.extend(tracked_failures)
511
585
  passed = passed and not tracked_failures
512
586
 
@@ -14,13 +14,13 @@ import py_compile
14
14
  import sys
15
15
  import types as _stdlib_types
16
16
  import zipimport
17
- from typing import TYPE_CHECKING, Final, cast, override
17
+ from typing import TYPE_CHECKING, Final, NamedTuple, cast, override
18
18
 
19
19
  import cloudpickle
20
20
 
21
21
  from pivot import exceptions, fingerprint, loaded_source, metrics, project, types
22
22
  from pivot import state_dir as state_dir_mod
23
- from pivot.registry import PipelineLike
23
+ from pivot.registry import PipelineLike, RegistryStageInfo
24
24
  from pivot.storage import ownership
25
25
  from pivot.storage import store as store_mod
26
26
 
@@ -1008,9 +1008,16 @@ def _log_skipped_module(strict: bool, message: str) -> None:
1008
1008
  logger.warning(message)
1009
1009
 
1010
1010
 
1011
- def declared_outputs(project_root: pathlib.Path) -> dict[str, ownership.DeclaredOutput]:
1011
+ class DeclaredOutputs(NamedTuple):
1012
+ """Every pipeline's outputs (``declared_outputs``) and the stages that declare them."""
1013
+
1014
+ outputs: dict[str, ownership.DeclaredOutput]
1015
+ stages: dict[str, RegistryStageInfo]
1016
+
1017
+
1018
+ def declared_outputs(project_root: pathlib.Path) -> DeclaredOutputs:
1012
1019
  """Map every output declared by every pipeline in the project to where it resolves and the
1013
- stage that writes it.
1020
+ stage that writes it, beside every pipeline's stages.
1014
1021
 
1015
1022
  Pipelines are discovered as ``--all`` discovers them, so an output of a pipeline the command
1016
1023
  did not load still counts: ``track`` and ``import`` refuse a path overlapping any of them. A
@@ -1027,7 +1034,7 @@ def declared_outputs(project_root: pathlib.Path) -> dict[str, ownership.Declared
1027
1034
  except DiscoveryError as exc:
1028
1035
  raise exceptions.ValidationError(str(exc)) from exc
1029
1036
  if pipeline is None:
1030
- return {}
1037
+ return DeclaredOutputs({}, {})
1031
1038
 
1032
1039
  workspace = store_mod.WorkspaceStore(
1033
1040
  project_root=project_root,
@@ -1053,7 +1060,8 @@ def declared_outputs(project_root: pathlib.Path) -> dict[str, ownership.Declared
1053
1060
  raise exceptions.ValidationError(
1054
1061
  "Cannot resolve stage outputs (fix these first):\n" + "\n".join(errors)
1055
1062
  )
1056
- return resolved
1063
+ stages = {stage_name: pipeline.get_stage(stage_name) for stage_name in pipeline.list_stages()}
1064
+ return DeclaredOutputs(resolved, stages)
1057
1065
 
1058
1066
 
1059
1067
  def load_pipeline_from_path(path: pathlib.Path) -> PipelineLike | None:
@@ -26,6 +26,7 @@ from pivot import (
26
26
  config,
27
27
  exceptions,
28
28
  fingerprint,
29
+ ignore,
29
30
  parameters,
30
31
  project,
31
32
  registry,
@@ -60,7 +61,7 @@ from pivot.engine.types import (
60
61
  )
61
62
  from pivot.executor import core as executor_core
62
63
  from pivot.executor import worker
63
- from pivot.storage import ownership
64
+ from pivot.storage import member_skip, ownership
64
65
  from pivot.storage import state as state_mod
65
66
  from pivot.storage import store as store_mod
66
67
  from pivot.types import (
@@ -289,6 +290,7 @@ class Engine:
289
290
  _graph: nx.DiGraph[str] | None
290
291
  _watch_coordinator: engine_watch.WatchCoordinator | None
291
292
  _watch_output_stats: dict[pathlib.Path, _WatchOutputSnapshot]
293
+ _ignore_filter: ignore.IgnoreFilter | None
292
294
  _scheduler: engine_scheduler.Scheduler
293
295
  _cancel_event: anyio.Event
294
296
  _stage_indices: dict[str, tuple[int, int]]
@@ -363,6 +365,7 @@ class Engine:
363
365
  self._graph = None
364
366
  self._watch_coordinator = None
365
367
  self._watch_output_stats = dict[pathlib.Path, _WatchOutputSnapshot]()
368
+ self._ignore_filter = None
366
369
  self._scheduler = engine_scheduler.Scheduler()
367
370
  self._cancel_event = anyio.Event()
368
371
  self._stage_indices = dict[str, tuple[int, int]]()
@@ -1134,12 +1137,19 @@ class Engine:
1134
1137
  if cache_dir is None:
1135
1138
  cache_dir = config.get_cache_dir()
1136
1139
 
1137
- # Verify tracked files before building DAG
1140
+ # Verify tracked files before building DAG: a stage's own .pvt member of its output is
1141
+ # none, the loaded pipeline's by the registry and any other's by its lock
1138
1142
  project_root = project.get_project_root()
1139
- executor_core.verify_tracked_files(project_root, checkout_missing=checkout_missing)
1143
+ all_stages = self._get_all_stages()
1144
+ executor_core.verify_tracked_files(
1145
+ project_root,
1146
+ checkout_missing=checkout_missing,
1147
+ skip=member_skip.own_member_skip(
1148
+ project_root, all_stages, config.get_state_dir(), cache_dir / "files"
1149
+ ),
1150
+ )
1140
1151
 
1141
1152
  # Build bipartite graph (single source of truth)
1142
- all_stages = self._get_all_stages()
1143
1153
  graph = engine_graph.build_graph(all_stages)
1144
1154
  pipeline = self._pipeline
1145
1155
  if pipeline is None:
@@ -1944,7 +1954,7 @@ class Engine:
1944
1954
  return False
1945
1955
 
1946
1956
  stage_state_dir = registry.get_stage_state_dir(stage_info, state_dir)
1947
- store = store_mod.store_from_spec(store_spec)
1957
+ store = store_mod.store_from_spec(store_spec, ignore_filter=self._get_ignore_filter())
1948
1958
 
1949
1959
  def _read_lock() -> LockData | None:
1950
1960
  stages_dir = lock.get_stages_dir(stage_state_dir)
@@ -2257,6 +2267,7 @@ class Engine:
2257
2267
  pipeline_name=pipeline.name,
2258
2268
  input_bindings=pipeline.input_bindings,
2259
2269
  cache_dir=pathlib.Path(worker_info["store_spec"]["cache_dir"]),
2270
+ ignore_filter=self._get_ignore_filter(),
2260
2271
  )
2261
2272
  lock_data = lock.StageLock(
2262
2273
  stage_name, lock.get_stages_dir(worker_info["state_dir"])
@@ -2962,13 +2973,23 @@ class Engine:
2962
2973
  project_root=project.get_project_root(),
2963
2974
  pipeline_name=pipeline.name,
2964
2975
  input_bindings=pipeline.input_bindings,
2976
+ ignore_filter=self._get_ignore_filter(),
2965
2977
  )
2966
2978
  self._watch_coordinator = engine_watch.WatchCoordinator(
2967
2979
  self._graph,
2968
2980
  engine_watch.build_artifact_paths(self._get_all_stages(), store),
2981
+ store,
2969
2982
  )
2970
2983
  return self._watch_coordinator
2971
2984
 
2985
+ def _get_ignore_filter(self) -> ignore.IgnoreFilter:
2986
+ """The project's ``.pivotignore`` filter, one for this engine: its skip checks, its
2987
+ explanations and its watch planning all read the file through it, and it reloads the
2988
+ file when its mtime changes."""
2989
+ if self._ignore_filter is None:
2990
+ self._ignore_filter = ignore.IgnoreFilter(project.get_project_root())
2991
+ return self._ignore_filter
2992
+
2972
2993
  def _should_filter_path(self, path: pathlib.Path) -> bool:
2973
2994
  """Check if path should be filtered (output of executing stage)."""
2974
2995
  coordinator = self._get_watch_coordinator()