calkit-python 0.35.4__py3-none-any.whl → 0.35.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/conda.py +78 -28
- calkit/git.py +140 -40
- calkit/overleaf.py +32 -1
- calkit/pipeline.py +27 -8
- calkit/tests/cli/test_overleaf.py +38 -0
- calkit/tests/test_conda.py +120 -2
- calkit/tests/test_git.py +59 -10
- calkit/tests/test_pipeline.py +100 -0
- {calkit_python-0.35.4.dist-info → calkit_python-0.35.6.dist-info}/METADATA +1 -1
- {calkit_python-0.35.4.dist-info → calkit_python-0.35.6.dist-info}/RECORD +28 -28
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/install.json +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/package.json +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/909.651be47ca47390b78a92.js +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/remoteEntry.c091821b3d7f2d287a67.js +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/style.js +0 -0
- {calkit_python-0.35.4.data → calkit_python-0.35.6.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
- {calkit_python-0.35.4.dist-info → calkit_python-0.35.6.dist-info}/WHEEL +0 -0
- {calkit_python-0.35.4.dist-info → calkit_python-0.35.6.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.35.4.dist-info → calkit_python-0.35.6.dist-info}/licenses/LICENSE +0 -0
calkit/conda.py
CHANGED
|
@@ -9,6 +9,7 @@ import shutil
|
|
|
9
9
|
import subprocess
|
|
10
10
|
import warnings
|
|
11
11
|
from pathlib import Path
|
|
12
|
+
from typing import cast
|
|
12
13
|
|
|
13
14
|
import toml
|
|
14
15
|
from packaging.specifiers import SpecifierSet
|
|
@@ -123,7 +124,13 @@ def _editable_package_name_from_dir(dir_path: str) -> str:
|
|
|
123
124
|
elif os.path.isfile(os.path.join(dir_path, "pyproject.toml")):
|
|
124
125
|
# Read pyproject.toml to get the package name
|
|
125
126
|
with open(os.path.join(dir_path, "pyproject.toml")) as f:
|
|
126
|
-
|
|
127
|
+
try:
|
|
128
|
+
pyproject = toml.load(f)
|
|
129
|
+
except Exception as e:
|
|
130
|
+
raise type(e)(
|
|
131
|
+
f"Failed to load pyproject.toml from {dir_path}; "
|
|
132
|
+
"check that it is valid TOML"
|
|
133
|
+
) from e
|
|
127
134
|
if "project" in pyproject:
|
|
128
135
|
if "name" in pyproject["project"]:
|
|
129
136
|
return pyproject["project"]["name"]
|
|
@@ -139,6 +146,7 @@ def _check_single(
|
|
|
139
146
|
"""
|
|
140
147
|
# If this is an editable install it needs to be handled specially
|
|
141
148
|
# It also needs to be relative to the env spec dir
|
|
149
|
+
editable = False
|
|
142
150
|
if req.startswith("-e ") or req.startswith("--editable "):
|
|
143
151
|
req = req.split(" ", 1)[1]
|
|
144
152
|
if "#" in req:
|
|
@@ -147,6 +155,7 @@ def _check_single(
|
|
|
147
155
|
# Create path relative to env spec dir
|
|
148
156
|
req = os.path.join(env_spec_dir, req)
|
|
149
157
|
req = _editable_package_name_from_dir(req)
|
|
158
|
+
editable = True
|
|
150
159
|
# If this is a Git version, we can't check it
|
|
151
160
|
# TODO: Clone Git repos to check?
|
|
152
161
|
if "@git" in req:
|
|
@@ -182,7 +191,7 @@ def _check_single(
|
|
|
182
191
|
# TODO: Check exact version only
|
|
183
192
|
return True
|
|
184
193
|
spec = SpecifierSet(req_spec)
|
|
185
|
-
return spec.contains(version)
|
|
194
|
+
return spec.contains(version, prereleases=editable)
|
|
186
195
|
|
|
187
196
|
|
|
188
197
|
def _check_list(
|
|
@@ -193,6 +202,10 @@ def _check_list(
|
|
|
193
202
|
if "::" in req:
|
|
194
203
|
req = req.split("::", 1)[1]
|
|
195
204
|
for installed in actual:
|
|
205
|
+
if not isinstance(installed, str):
|
|
206
|
+
raise ValueError(
|
|
207
|
+
f"Expected installed package to be a string, got {installed}"
|
|
208
|
+
)
|
|
196
209
|
if _check_single(
|
|
197
210
|
req, installed, env_spec_dir=env_spec_dir, conda=conda
|
|
198
211
|
):
|
|
@@ -200,6 +213,46 @@ def _check_list(
|
|
|
200
213
|
return False
|
|
201
214
|
|
|
202
215
|
|
|
216
|
+
def _split_env_dependencies(
|
|
217
|
+
dependencies: list[str | dict[str, str | list[str]]],
|
|
218
|
+
) -> tuple[list[str], list[str]]:
|
|
219
|
+
"""Split an environment dependency list into conda and pip deps.
|
|
220
|
+
|
|
221
|
+
Conda environment files commonly include both the plain ``"pip"`` package
|
|
222
|
+
marker and a nested ``{"pip": [...]}`` section. This helper normalizes the
|
|
223
|
+
latter so callers do not need to assume it is the final list entry or that
|
|
224
|
+
the pip section is already represented as a list.
|
|
225
|
+
"""
|
|
226
|
+
conda_deps = []
|
|
227
|
+
pip_deps = []
|
|
228
|
+
for dep in dependencies:
|
|
229
|
+
if isinstance(dep, dict):
|
|
230
|
+
dep_pip = dep.get("pip", [])
|
|
231
|
+
if isinstance(dep_pip, str):
|
|
232
|
+
dep_pip = [dep_pip]
|
|
233
|
+
elif dep_pip is None:
|
|
234
|
+
dep_pip = []
|
|
235
|
+
pip_deps.extend(dep_pip)
|
|
236
|
+
else:
|
|
237
|
+
conda_deps.append(dep)
|
|
238
|
+
return conda_deps, pip_deps
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _get_pip_dependency_list(
|
|
242
|
+
dependencies: list[str | dict[str, str | list[str]]],
|
|
243
|
+
) -> list[str]:
|
|
244
|
+
"""Return a mutable pip dependency list from an env dependency list."""
|
|
245
|
+
for dep in dependencies:
|
|
246
|
+
if isinstance(dep, dict) and "pip" in dep:
|
|
247
|
+
dep_pip = dep["pip"]
|
|
248
|
+
if isinstance(dep_pip, str):
|
|
249
|
+
dep["pip"] = [dep_pip]
|
|
250
|
+
elif dep_pip is None:
|
|
251
|
+
dep["pip"] = []
|
|
252
|
+
return cast(list[str], dep["pip"])
|
|
253
|
+
return []
|
|
254
|
+
|
|
255
|
+
|
|
203
256
|
class EnvCheckResult(BaseModel):
|
|
204
257
|
env_exists: bool | None = None
|
|
205
258
|
env_needs_export: bool | None = None
|
|
@@ -393,18 +446,12 @@ def check_env(
|
|
|
393
446
|
ryaml.dump(env_check, f)
|
|
394
447
|
# Determine if the env matches
|
|
395
448
|
env_needs_rebuild = False
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
if isinstance(env_spec["dependencies"][-1], dict):
|
|
403
|
-
required_conda_deps = env_spec["dependencies"][:-1]
|
|
404
|
-
required_pip_deps = env_spec["dependencies"][-1]["pip"]
|
|
405
|
-
else:
|
|
406
|
-
required_conda_deps = env_spec["dependencies"]
|
|
407
|
-
required_pip_deps = []
|
|
449
|
+
existing_conda_deps, existing_pip_deps = _split_env_dependencies(
|
|
450
|
+
env_check["dependencies"]
|
|
451
|
+
)
|
|
452
|
+
required_conda_deps, required_pip_deps = _split_env_dependencies(
|
|
453
|
+
env_spec["dependencies"]
|
|
454
|
+
)
|
|
408
455
|
if relaxed:
|
|
409
456
|
log_func("Running in relaxed mode; combining pip and conda deps")
|
|
410
457
|
for dep in existing_pip_deps:
|
|
@@ -516,20 +563,23 @@ def check_env(
|
|
|
516
563
|
# Note that this needs to be relative to the env lock directory,
|
|
517
564
|
# since that's how pip will interpret it
|
|
518
565
|
editable_pip_deps = {}
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
if
|
|
524
|
-
dir_path =
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
566
|
+
required_pip_deps = _get_pip_dependency_list(env_spec["dependencies"])
|
|
567
|
+
for dep in required_pip_deps:
|
|
568
|
+
if dep.startswith("-e ") or dep.startswith("--editable "):
|
|
569
|
+
dir_path = dep.split(" ", 1)[1]
|
|
570
|
+
if "#" in dir_path:
|
|
571
|
+
dir_path = dir_path.split("#", 1)[0]
|
|
572
|
+
dir_path = dir_path.strip()
|
|
573
|
+
dir_path = os.path.join(env_spec_dir, dir_path)
|
|
574
|
+
pkg_name = _editable_package_name_from_dir(dir_path)
|
|
575
|
+
if verbose:
|
|
576
|
+
log_func(
|
|
577
|
+
f"Found editable pip dependency '{pkg_name}' "
|
|
578
|
+
f"at '{dir_path}'"
|
|
579
|
+
)
|
|
580
|
+
editable_pip_deps[pkg_name] = dir_path
|
|
581
|
+
export_pip_deps = _get_pip_dependency_list(env_export["dependencies"])
|
|
582
|
+
if export_pip_deps:
|
|
533
583
|
for i, dep in enumerate(export_pip_deps):
|
|
534
584
|
dep_name = re.split("[=<>]+", dep, maxsplit=1)[0]
|
|
535
585
|
if dep_name in editable_pip_deps:
|
calkit/git.py
CHANGED
|
@@ -100,6 +100,30 @@ def _resolve_repo_and_ignore_path(
|
|
|
100
100
|
return repo, rel_path
|
|
101
101
|
|
|
102
102
|
|
|
103
|
+
def _get_matching_gitignore_details(
|
|
104
|
+
repo: git.Repo, path: str
|
|
105
|
+
) -> tuple[Path | None, str | None]:
|
|
106
|
+
"""Return the repo-local gitignore file and pattern matching ``path``."""
|
|
107
|
+
try:
|
|
108
|
+
check_ignore = repo.git.check_ignore("-v", "--", path)
|
|
109
|
+
except git.GitCommandError:
|
|
110
|
+
return None, None
|
|
111
|
+
line = check_ignore.splitlines()[0]
|
|
112
|
+
try:
|
|
113
|
+
source_info, _ = line.split("\t", 1)
|
|
114
|
+
source_path, _, pattern = source_info.rsplit(":", 2)
|
|
115
|
+
except ValueError:
|
|
116
|
+
return None, None
|
|
117
|
+
if not source_path.endswith(".gitignore"):
|
|
118
|
+
return None, pattern
|
|
119
|
+
gitignore_path = (Path(repo.working_dir) / source_path).resolve()
|
|
120
|
+
try:
|
|
121
|
+
gitignore_path.relative_to(Path(repo.working_dir).resolve())
|
|
122
|
+
except ValueError:
|
|
123
|
+
return None, pattern
|
|
124
|
+
return gitignore_path, pattern
|
|
125
|
+
|
|
126
|
+
|
|
103
127
|
def ensure_path_is_ignored(
|
|
104
128
|
repo: git.Repo, path: str | PathLike
|
|
105
129
|
) -> None | bool:
|
|
@@ -141,56 +165,132 @@ def ensure_path_is_not_ignored(
|
|
|
141
165
|
# No-op if Git does not ignore this path.
|
|
142
166
|
if not target_repo.ignored(target_path):
|
|
143
167
|
return
|
|
144
|
-
|
|
168
|
+
matching_gitignore_path, matched_pattern = _get_matching_gitignore_details(
|
|
169
|
+
target_repo, target_path
|
|
170
|
+
)
|
|
171
|
+
if matching_gitignore_path is not None:
|
|
172
|
+
gitignore_path = matching_gitignore_path.as_posix()
|
|
173
|
+
path_for_gitignore = (
|
|
174
|
+
(Path(target_repo.working_dir) / target_path)
|
|
175
|
+
.resolve()
|
|
176
|
+
.relative_to(matching_gitignore_path.parent.resolve())
|
|
177
|
+
.as_posix()
|
|
178
|
+
)
|
|
179
|
+
else:
|
|
180
|
+
gitignore_path = os.path.join(target_repo.working_dir, ".gitignore")
|
|
181
|
+
path_for_gitignore = target_path
|
|
145
182
|
if not os.path.isfile(gitignore_path):
|
|
146
183
|
with open(gitignore_path, "w") as f:
|
|
147
|
-
f.write(f"!{
|
|
184
|
+
f.write(f"!{path_for_gitignore}\n")
|
|
148
185
|
return True
|
|
149
186
|
with open(gitignore_path) as f:
|
|
150
187
|
gitignore_txt = f.read()
|
|
151
188
|
lines = gitignore_txt.splitlines()
|
|
152
|
-
|
|
153
|
-
|
|
189
|
+
direct_rule_variants = [path_for_gitignore, f"/{path_for_gitignore}"]
|
|
190
|
+
if matched_pattern is not None and matched_pattern.startswith("/"):
|
|
191
|
+
no_ignore_line = f"!/{path_for_gitignore}"
|
|
192
|
+
else:
|
|
193
|
+
no_ignore_line = f"!{path_for_gitignore}"
|
|
194
|
+
path_parts = Path(path_for_gitignore).parts
|
|
195
|
+
|
|
196
|
+
def ancestor_requires_recursive_unignore() -> bool:
|
|
197
|
+
"""Return True if any ancestor-level ignore rule would block this path.
|
|
198
|
+
|
|
199
|
+
This includes explicit directory ignores (e.g. 'dir/' or '/dir/')
|
|
200
|
+
as well as ancestor-based glob patterns like 'dir/*' or '/dir/*',
|
|
201
|
+
i.e., any rule that would prevent reaching the nested path without
|
|
202
|
+
adding recursive unignore patterns.
|
|
203
|
+
"""
|
|
204
|
+
for i in range(1, len(path_parts)):
|
|
205
|
+
ancestor = "/".join(path_parts[:i])
|
|
206
|
+
if (
|
|
207
|
+
ancestor in lines
|
|
208
|
+
or f"/{ancestor}" in lines
|
|
209
|
+
or f"{ancestor}/" in lines
|
|
210
|
+
or f"/{ancestor}/" in lines
|
|
211
|
+
or f"{ancestor}/*" in lines
|
|
212
|
+
or f"/{ancestor}/*" in lines
|
|
213
|
+
):
|
|
214
|
+
return True
|
|
215
|
+
return False
|
|
216
|
+
|
|
154
217
|
if len(path_parts) == 1:
|
|
155
218
|
# Simple (non-nested) path: remove the direct ignore rule, or add a
|
|
156
|
-
# negation if the ignore comes from a glob or other pattern
|
|
157
|
-
|
|
158
|
-
lines
|
|
159
|
-
|
|
219
|
+
# negation if the ignore comes from a glob or other pattern
|
|
220
|
+
direct_rule = next(
|
|
221
|
+
(rule for rule in direct_rule_variants if rule in lines), None
|
|
222
|
+
)
|
|
223
|
+
if direct_rule is not None:
|
|
224
|
+
lines.remove(direct_rule)
|
|
225
|
+
else:
|
|
226
|
+
# Remove any stale negation and re-append at the end so it takes
|
|
227
|
+
# precedence over any later re-ignore rule
|
|
228
|
+
if no_ignore_line in lines:
|
|
229
|
+
lines.remove(no_ignore_line)
|
|
160
230
|
lines.append(no_ignore_line)
|
|
161
231
|
else:
|
|
162
|
-
# Nested path:
|
|
163
|
-
#
|
|
164
|
-
#
|
|
165
|
-
#
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
if
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
#
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
lines
|
|
193
|
-
|
|
232
|
+
# Nested path: only apply recursive un-ignore rules when an ancestor
|
|
233
|
+
# directory is explicitly ignored
|
|
234
|
+
# Otherwise, remove a direct ignore
|
|
235
|
+
# rule for this path or add a simple negation if needed
|
|
236
|
+
removed_direct_rule = False
|
|
237
|
+
direct_rule = next(
|
|
238
|
+
(rule for rule in direct_rule_variants if rule in lines), None
|
|
239
|
+
)
|
|
240
|
+
if direct_rule is not None:
|
|
241
|
+
lines.remove(direct_rule)
|
|
242
|
+
removed_direct_rule = True
|
|
243
|
+
if ancestor_requires_recursive_unignore():
|
|
244
|
+
# Git will not traverse into a directory excluded by a "dir/"
|
|
245
|
+
# pattern, so a bare "!dir/sub/file" negation has no effect.
|
|
246
|
+
# We need to:
|
|
247
|
+
# 1. Convert any "ancestor/" (or "ancestor") exclude to
|
|
248
|
+
# "ancestor/*" so git traverses the directory while still
|
|
249
|
+
# ignoring direct children by default.
|
|
250
|
+
# 2. Add "!ancestor/" rules for intermediate directories.
|
|
251
|
+
# 3. Add "ancestor/*" re-ignore rules for each intermediate dir.
|
|
252
|
+
# 4. Add "!target_path" for the specific file.
|
|
253
|
+
for i in range(1, len(path_parts)):
|
|
254
|
+
ancestor = "/".join(path_parts[:i])
|
|
255
|
+
reignore_glob = f"{ancestor}/*"
|
|
256
|
+
if f"{ancestor}/" in lines:
|
|
257
|
+
idx = lines.index(f"{ancestor}/")
|
|
258
|
+
lines[idx] = reignore_glob
|
|
259
|
+
elif f"/{ancestor}/" in lines:
|
|
260
|
+
idx = lines.index(f"/{ancestor}/")
|
|
261
|
+
lines[idx] = f"/{ancestor}/*"
|
|
262
|
+
elif ancestor in lines:
|
|
263
|
+
idx = lines.index(ancestor)
|
|
264
|
+
lines[idx] = reignore_glob
|
|
265
|
+
elif f"/{ancestor}" in lines:
|
|
266
|
+
idx = lines.index(f"/{ancestor}")
|
|
267
|
+
lines[idx] = f"/{ancestor}/*"
|
|
268
|
+
no_ignore_dir = f"!{ancestor}/"
|
|
269
|
+
anchored_no_ignore_dir = f"!/{ancestor}/"
|
|
270
|
+
# The first ancestor does not need an explicit un-ignore once
|
|
271
|
+
# converted to "ancestor/*". Deeper ancestors do.
|
|
272
|
+
if i > 1:
|
|
273
|
+
# Remove stale entry and re-append so it takes precedence
|
|
274
|
+
if no_ignore_dir in lines:
|
|
275
|
+
lines.remove(no_ignore_dir)
|
|
276
|
+
elif anchored_no_ignore_dir in lines:
|
|
277
|
+
lines.remove(anchored_no_ignore_dir)
|
|
278
|
+
lines.append(no_ignore_dir)
|
|
279
|
+
if (
|
|
280
|
+
reignore_glob not in lines
|
|
281
|
+
and f"/{ancestor}/*" not in lines
|
|
282
|
+
):
|
|
283
|
+
lines.append(reignore_glob)
|
|
284
|
+
# Remove stale negation and re-append at the end so it takes
|
|
285
|
+
# precedence over any later re-ignore rule
|
|
286
|
+
if no_ignore_line in lines:
|
|
287
|
+
lines.remove(no_ignore_line)
|
|
288
|
+
lines.append(no_ignore_line)
|
|
289
|
+
elif not removed_direct_rule:
|
|
290
|
+
# The path may be ignored by a non-directory pattern (e.g., glob);
|
|
291
|
+
# remove stale negation and append at end so it takes precedence
|
|
292
|
+
if no_ignore_line in lines:
|
|
293
|
+
lines.remove(no_ignore_line)
|
|
194
294
|
lines.append(no_ignore_line)
|
|
195
295
|
with open(gitignore_path, "w") as f:
|
|
196
296
|
f.write(os.linesep.join(lines))
|
calkit/overleaf.py
CHANGED
|
@@ -317,6 +317,32 @@ class OverleafSyncPaths:
|
|
|
317
317
|
- self.files_in_overleaf_last_sync
|
|
318
318
|
)
|
|
319
319
|
|
|
320
|
+
@cached_property
|
|
321
|
+
def dvc_files(self) -> set[str]:
|
|
322
|
+
"""Files tracked by DVC within the Overleaf project folder.
|
|
323
|
+
|
|
324
|
+
These paths are relative to the project directory (i.e., relative to
|
|
325
|
+
the Overleaf repo root). Files tracked by DVC may not exist on disk if
|
|
326
|
+
they haven't been pulled, but should still be kept on Overleaf rather
|
|
327
|
+
than deleted.
|
|
328
|
+
"""
|
|
329
|
+
try:
|
|
330
|
+
import calkit.dvc
|
|
331
|
+
|
|
332
|
+
dvc_paths = calkit.dvc.list_paths(
|
|
333
|
+
wdir=str(self.main_repo.working_dir), recursive=True
|
|
334
|
+
)
|
|
335
|
+
except Exception as e:
|
|
336
|
+
warnings.warn(f"Could not list DVC files: {e}")
|
|
337
|
+
return set()
|
|
338
|
+
prefix = Path(self.path_in_project).as_posix().rstrip("/") + "/"
|
|
339
|
+
result = set()
|
|
340
|
+
for p in dvc_paths:
|
|
341
|
+
p_posix = Path(p).as_posix()
|
|
342
|
+
if p_posix.startswith(prefix):
|
|
343
|
+
result.add(p_posix[len(prefix) :])
|
|
344
|
+
return result
|
|
345
|
+
|
|
320
346
|
@cached_property
|
|
321
347
|
def files_to_keep_on_overleaf(self) -> set[str]:
|
|
322
348
|
"""Files that should be preserved on Overleaf.
|
|
@@ -324,9 +350,14 @@ class OverleafSyncPaths:
|
|
|
324
350
|
This includes:
|
|
325
351
|
1. All files being copied from local
|
|
326
352
|
2. Any files newly added on Overleaf since last sync
|
|
353
|
+
3. Any files tracked by DVC within the project path (these may not
|
|
354
|
+
exist on disk if not pulled, but should not be deleted from
|
|
355
|
+
Overleaf)
|
|
327
356
|
"""
|
|
328
357
|
return (
|
|
329
|
-
set(self.files_to_copy_to_overleaf)
|
|
358
|
+
set(self.files_to_copy_to_overleaf)
|
|
359
|
+
| self.newly_added_on_overleaf
|
|
360
|
+
| self.dvc_files
|
|
330
361
|
)
|
|
331
362
|
|
|
332
363
|
@cached_property
|
calkit/pipeline.py
CHANGED
|
@@ -117,6 +117,20 @@ def to_dvc(
|
|
|
117
117
|
except Exception as e:
|
|
118
118
|
raise ValueError(f"Pipeline is not defined properly: {e}")
|
|
119
119
|
dvc_stages = {}
|
|
120
|
+
# Read existing dvc.yaml now so we can clean up stale .gitignore entries
|
|
121
|
+
# when stage outputs are renamed or removed
|
|
122
|
+
if write:
|
|
123
|
+
dvc_yaml_path = os.path.join(wdir, "dvc.yaml") if wdir else "dvc.yaml"
|
|
124
|
+
if os.path.isfile(dvc_yaml_path):
|
|
125
|
+
with open(dvc_yaml_path) as f:
|
|
126
|
+
existing_dvc_yaml = calkit.ryaml.load(f)
|
|
127
|
+
else:
|
|
128
|
+
existing_dvc_yaml = {}
|
|
129
|
+
if existing_dvc_yaml is None:
|
|
130
|
+
existing_dvc_yaml = {}
|
|
131
|
+
existing_dvc_stages = existing_dvc_yaml.get("stages", {})
|
|
132
|
+
else:
|
|
133
|
+
existing_dvc_stages = {}
|
|
120
134
|
# First, gather up any env lock paths we might need for DVC deps
|
|
121
135
|
used_envs = set(
|
|
122
136
|
[stage.inner_environment for stage in pipeline.stages.values()]
|
|
@@ -227,6 +241,17 @@ def to_dvc(
|
|
|
227
241
|
outputs += stage.notebook_outputs
|
|
228
242
|
elif stage.kind == "sbatch":
|
|
229
243
|
outputs.append(stage.log_output)
|
|
244
|
+
# Build the set of current DVC output paths so we can detect stale
|
|
245
|
+
# .gitignore entries from the previous version of the stage,
|
|
246
|
+
# including synthesized outputs like LaTeX PDFs
|
|
247
|
+
current_out_paths = set(calkit.dvc.out_paths_from_stage(dvc_stage))
|
|
248
|
+
# If this stage already existed, un-ignore any outputs that have
|
|
249
|
+
# been renamed or removed so .gitignore does not accumulate stale
|
|
250
|
+
# entries (e.g., after a capitalization change in the path)
|
|
251
|
+
old_stage = existing_dvc_stages.get(stage_name, {})
|
|
252
|
+
for old_path in calkit.dvc.out_paths_from_stage(old_stage):
|
|
253
|
+
if old_path not in current_out_paths:
|
|
254
|
+
calkit.git.ensure_path_is_not_ignored(repo, path=old_path)
|
|
230
255
|
# Deal with any gitignore changes necessary
|
|
231
256
|
for out in outputs:
|
|
232
257
|
if isinstance(out, PathOutput) and out.storage is None:
|
|
@@ -268,14 +293,8 @@ def to_dvc(
|
|
|
268
293
|
else:
|
|
269
294
|
dvc_stages[stage_name]["deps"].append(out)
|
|
270
295
|
if write:
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
dvc_yaml = calkit.ryaml.load(f)
|
|
274
|
-
else:
|
|
275
|
-
dvc_yaml = {}
|
|
276
|
-
if dvc_yaml is None:
|
|
277
|
-
dvc_yaml = {}
|
|
278
|
-
existing_stages = dvc_yaml.get("stages", {})
|
|
296
|
+
dvc_yaml = existing_dvc_yaml
|
|
297
|
+
existing_stages = existing_dvc_stages
|
|
279
298
|
for stage_name, stage in existing_stages.items():
|
|
280
299
|
# Skip private stages (ones whose names start with an underscore)
|
|
281
300
|
# and stages that are automatically generated
|
|
@@ -181,6 +181,44 @@ def test_overleaf(tmp_dir):
|
|
|
181
181
|
subprocess.run(["calkit", "overleaf", "sync", "--verbose"], check=True)
|
|
182
182
|
print("Overleaf Git show after adding fig2 back:", ol_repo.git.show())
|
|
183
183
|
assert "ol-project/figs/fig2.txt" in ls_files(repo)
|
|
184
|
+
# Test that if a file is deleted from Git but added to DVC, it is not
|
|
185
|
+
# deleted from Overleaf (the file still logically exists in the DVC repo)
|
|
186
|
+
with open(
|
|
187
|
+
os.path.join(repo.working_dir, "ol-project", "figs", "fig3.txt"), "w"
|
|
188
|
+
) as f:
|
|
189
|
+
f.write("Fig3 created in main repo")
|
|
190
|
+
repo.git.add("ol-project/figs/fig3.txt")
|
|
191
|
+
repo.git.commit(["-m", "Add figure 3"])
|
|
192
|
+
assert "ol-project/figs/fig3.txt" in ls_files(repo)
|
|
193
|
+
subprocess.run(["calkit", "overleaf", "sync", "--verbose"], check=True)
|
|
194
|
+
ol_repo_git_show = ol_repo.git.show()
|
|
195
|
+
assert "diff --git a/figs/fig3.txt b/figs/fig3.txt" in ol_repo_git_show
|
|
196
|
+
# Now move from Git to DVC: first remove from Git index (keeping file on
|
|
197
|
+
# disk), then add to DVC so it gets moved to DVC cache
|
|
198
|
+
repo.git.rm(["--cached", "ol-project/figs/fig3.txt"])
|
|
199
|
+
subprocess.run(
|
|
200
|
+
["dvc", "add", "ol-project/figs/fig3.txt"],
|
|
201
|
+
check=True,
|
|
202
|
+
cwd=repo.working_dir,
|
|
203
|
+
)
|
|
204
|
+
# Commit the DVC pointer file (fig3.txt is now tracked by DVC, not Git)
|
|
205
|
+
repo.git.add("ol-project/figs/fig3.txt.dvc", "ol-project/figs/.gitignore")
|
|
206
|
+
repo.git.commit(["-m", "Move figure 3 from git to DVC"])
|
|
207
|
+
assert "ol-project/figs/fig3.txt" not in ls_files(repo)
|
|
208
|
+
# Also remove the local file to simulate the file not being pulled from
|
|
209
|
+
# DVC (i.e., only the DVC pointer exists locally, not the actual file)
|
|
210
|
+
fig3_path = os.path.join(
|
|
211
|
+
repo.working_dir, "ol-project", "figs", "fig3.txt"
|
|
212
|
+
)
|
|
213
|
+
if os.path.exists(fig3_path):
|
|
214
|
+
os.remove(fig3_path)
|
|
215
|
+
assert not os.path.exists(fig3_path)
|
|
216
|
+
subprocess.run(["calkit", "overleaf", "sync", "--verbose"], check=True)
|
|
217
|
+
ol_repo_git_show = ol_repo.git.show()
|
|
218
|
+
print("Git show in OL repo after moving fig3 to DVC:\n", ol_repo_git_show)
|
|
219
|
+
# The file should not have been deleted from Overleaf
|
|
220
|
+
assert "deleted file mode" not in ol_repo_git_show
|
|
221
|
+
assert "--- a/figs/fig3.txt" not in ol_repo_git_show
|
|
184
222
|
|
|
185
223
|
|
|
186
224
|
def test_extract_title_from_tex(tmp_dir):
|
calkit/tests/test_conda.py
CHANGED
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
"""Tests for the ``conda`` module."""
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
+
import shutil
|
|
4
5
|
import subprocess
|
|
5
6
|
|
|
6
7
|
import pytest
|
|
7
8
|
|
|
8
9
|
import calkit
|
|
9
|
-
from calkit.conda import
|
|
10
|
+
from calkit.conda import (
|
|
11
|
+
_check_list,
|
|
12
|
+
_check_single,
|
|
13
|
+
_get_pip_dependency_list,
|
|
14
|
+
_split_env_dependencies,
|
|
15
|
+
check_env,
|
|
16
|
+
)
|
|
10
17
|
|
|
11
18
|
ENV_NAME = "main"
|
|
12
19
|
|
|
@@ -37,6 +44,25 @@ def test_check_list():
|
|
|
37
44
|
assert not _check_list("pandas", installed, env_spec_dir=".", conda=False)
|
|
38
45
|
|
|
39
46
|
|
|
47
|
+
def test_split_env_dependencies():
|
|
48
|
+
dependencies = [
|
|
49
|
+
"python=3.12",
|
|
50
|
+
"pip",
|
|
51
|
+
"numpy=2",
|
|
52
|
+
{"pip": ["sqlalchemy==2.0.39"]},
|
|
53
|
+
]
|
|
54
|
+
conda_deps, pip_deps = _split_env_dependencies(dependencies)
|
|
55
|
+
assert conda_deps == ["python=3.12", "pip", "numpy=2"]
|
|
56
|
+
assert pip_deps == ["sqlalchemy==2.0.39"]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_get_pip_dependency_list():
|
|
60
|
+
dependencies = ["python=3.12", "pip", {"pip": "sqlalchemy==2.0.39"}]
|
|
61
|
+
pip_deps = _get_pip_dependency_list(dependencies)
|
|
62
|
+
assert pip_deps == ["sqlalchemy==2.0.39"]
|
|
63
|
+
assert dependencies[-1]["pip"] == ["sqlalchemy==2.0.39"]
|
|
64
|
+
|
|
65
|
+
|
|
40
66
|
def delete_env(name: str):
|
|
41
67
|
subprocess.check_call(["conda", "env", "remove", "-y", "-n", name])
|
|
42
68
|
|
|
@@ -262,7 +288,7 @@ def test_check_prefix_env(tmp_dir, conda_env_prefix):
|
|
|
262
288
|
)
|
|
263
289
|
|
|
264
290
|
|
|
265
|
-
def
|
|
291
|
+
def test_check_env_editable(tmp_dir, conda_env_name):
|
|
266
292
|
subprocess.check_call(["calkit", "init"])
|
|
267
293
|
# Create a dummy package named 'src' to install in editable mode
|
|
268
294
|
os.makedirs("src", exist_ok=True)
|
|
@@ -306,6 +332,98 @@ setup(
|
|
|
306
332
|
lock = calkit.ryaml.load(f)
|
|
307
333
|
pip_deps = lock["dependencies"][-1]["pip"]
|
|
308
334
|
assert "-e ." in pip_deps
|
|
335
|
+
# Now let's make sure we get proper output if the editable package is
|
|
336
|
+
# has an invalid pyproject.toml
|
|
337
|
+
os.remove("setup.py")
|
|
338
|
+
shutil.rmtree("src.egg-info")
|
|
339
|
+
toml_txt = """[build-system]
|
|
340
|
+
requires = ["setuptools>=61.0.0", "wheel", "setuptools-scm>=8"]
|
|
341
|
+
build-backend = "setuptools.build_meta"
|
|
342
|
+
|
|
343
|
+
[project]
|
|
344
|
+
name = "src-thing"
|
|
345
|
+
dynamic = ["version"]
|
|
346
|
+
authors = [
|
|
347
|
+
{name = "Someone"}
|
|
348
|
+
]
|
|
349
|
+
description = "Test"
|
|
350
|
+
|
|
351
|
+
dependencies = [
|
|
352
|
+
"numpy>=1.21",
|
|
353
|
+
"scipy>=1.7",
|
|
354
|
+
"pandas>=1.5",
|
|
355
|
+
"matplotlib>=3.5",
|
|
356
|
+
"h5netcdf>=0.12",
|
|
357
|
+
"h5py>=3.0",
|
|
358
|
+
"xarray>=2023.0",
|
|
359
|
+
"streamlit>=1.0"
|
|
360
|
+
]
|
|
361
|
+
|
|
362
|
+
[tool.setuptools]
|
|
363
|
+
package-dir = {"" = "src"}
|
|
364
|
+
packages = ["src-thing"]
|
|
365
|
+
|
|
366
|
+
[tool.setuptools.packages.find]
|
|
367
|
+
where = ["src"]
|
|
368
|
+
|
|
369
|
+
[tool.setuptools.package-data]
|
|
370
|
+
"src-thing" = [] # Explicitly state no package data
|
|
371
|
+
|
|
372
|
+
[tool.setuptools_scm]
|
|
373
|
+
local_scheme = "no-local-version"
|
|
374
|
+
fallback_version = "0+unknown"
|
|
375
|
+
"""
|
|
376
|
+
with open("pyproject.toml", "w") as f:
|
|
377
|
+
f.write(toml_txt)
|
|
378
|
+
with pytest.raises(Exception, match="Failed to load pyproject.toml"):
|
|
379
|
+
res = check_env()
|
|
380
|
+
# Fix it and make sure it runs with relaxed mode
|
|
381
|
+
toml_txt = """[build-system]
|
|
382
|
+
requires = ["setuptools>=61.0.0", "wheel", "setuptools-scm>=8"]
|
|
383
|
+
build-backend = "setuptools.build_meta"
|
|
384
|
+
|
|
385
|
+
[project]
|
|
386
|
+
name = "src-thing"
|
|
387
|
+
dynamic = ["version"]
|
|
388
|
+
authors = [
|
|
389
|
+
{name = "Someone"}
|
|
390
|
+
]
|
|
391
|
+
description = "Test"
|
|
392
|
+
|
|
393
|
+
dependencies = []
|
|
394
|
+
|
|
395
|
+
[tool.setuptools]
|
|
396
|
+
packages = ["src"]
|
|
397
|
+
|
|
398
|
+
[tool.setuptools_scm]
|
|
399
|
+
local_scheme = "no-local-version"
|
|
400
|
+
fallback_version = "0+unknown"
|
|
401
|
+
"""
|
|
402
|
+
with open("pyproject.toml", "w") as f:
|
|
403
|
+
f.write(toml_txt)
|
|
404
|
+
res = check_env(relaxed=True)
|
|
405
|
+
assert res.env_exists
|
|
406
|
+
assert res.env_needs_rebuild
|
|
407
|
+
assert res.env_needs_export
|
|
408
|
+
# Make sure we can import the editable package
|
|
409
|
+
os.makedirs("subdir")
|
|
410
|
+
subprocess.check_call(
|
|
411
|
+
[
|
|
412
|
+
"conda",
|
|
413
|
+
"run",
|
|
414
|
+
"-n",
|
|
415
|
+
conda_env_name,
|
|
416
|
+
"python",
|
|
417
|
+
"-c",
|
|
418
|
+
"import src; print('src file:', src.__file__);",
|
|
419
|
+
],
|
|
420
|
+
cwd="subdir",
|
|
421
|
+
)
|
|
422
|
+
# Check again and make sure we don't need a rebuild since the editable
|
|
423
|
+
# package is still valid
|
|
424
|
+
res = check_env(relaxed=True)
|
|
425
|
+
assert res.env_exists
|
|
426
|
+
assert not res.env_needs_rebuild
|
|
309
427
|
|
|
310
428
|
|
|
311
429
|
def test_find_conda_exe():
|
calkit/tests/test_git.py
CHANGED
|
@@ -105,7 +105,7 @@ def test_ensure_path_is_not_ignored_nested(tmp_dir):
|
|
|
105
105
|
|
|
106
106
|
When a parent directory is excluded with a trailing slash (e.g.,
|
|
107
107
|
``results/``), git will not traverse into it, so a simple negation like
|
|
108
|
-
``!results/StageName/end.json`` has no effect.
|
|
108
|
+
``!results/StageName/end.json`` has no effect. The fix converts the
|
|
109
109
|
directory exclude to a glob pattern and adds intermediate un-ignore rules.
|
|
110
110
|
"""
|
|
111
111
|
repo = git.Repo.init()
|
|
@@ -124,15 +124,13 @@ def test_ensure_path_is_not_ignored_nested(tmp_dir):
|
|
|
124
124
|
assert result is True
|
|
125
125
|
with open(".gitignore") as f:
|
|
126
126
|
lines = f.read().splitlines()
|
|
127
|
-
#
|
|
128
|
-
assert
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
# The specific file must be explicitly un-ignored
|
|
135
|
-
assert "!results/StageName/end.json" in lines
|
|
127
|
+
# Keep the rules minimal while preserving the required behavior
|
|
128
|
+
assert lines == [
|
|
129
|
+
"results/*",
|
|
130
|
+
"!results/StageName/",
|
|
131
|
+
"results/StageName/*",
|
|
132
|
+
"!results/StageName/end.json",
|
|
133
|
+
]
|
|
136
134
|
# Verify git no longer considers the target file as ignored
|
|
137
135
|
assert not repo.ignored("results/StageName/end.json")
|
|
138
136
|
# Other files in results/ must still be ignored
|
|
@@ -148,3 +146,54 @@ def test_ensure_path_is_not_ignored_nested(tmp_dir):
|
|
|
148
146
|
repo, path="results/StageName/end.json"
|
|
149
147
|
)
|
|
150
148
|
assert result2 is None
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def test_ensure_path_is_not_ignored_nested_direct_path_rule(tmp_dir):
|
|
152
|
+
"""Unignoring a directly ignored nested path should only remove that
|
|
153
|
+
rule.
|
|
154
|
+
"""
|
|
155
|
+
repo = git.Repo.init()
|
|
156
|
+
target = "pubs/applied-ocean-research-model/references.bib"
|
|
157
|
+
sibling = "pubs/applied-ocean-research-model/paper.pdf"
|
|
158
|
+
with open(".gitignore", "w") as f:
|
|
159
|
+
f.write(f"{target}\n")
|
|
160
|
+
os.makedirs("pubs/applied-ocean-research-model", exist_ok=True)
|
|
161
|
+
with open(target, "w") as f:
|
|
162
|
+
f.write("@article{test}\n")
|
|
163
|
+
with open(sibling, "w") as f:
|
|
164
|
+
f.write("pdf\n")
|
|
165
|
+
# Only the direct target path should be ignored initially
|
|
166
|
+
assert repo.ignored(target)
|
|
167
|
+
assert not repo.ignored(sibling)
|
|
168
|
+
result = calkit.git.ensure_path_is_not_ignored(repo, path=target)
|
|
169
|
+
assert result is True
|
|
170
|
+
with open(".gitignore") as f:
|
|
171
|
+
lines = f.read().splitlines()
|
|
172
|
+
# Remove only the direct rule, with no recursive ancestor entries
|
|
173
|
+
assert target not in lines
|
|
174
|
+
assert f"!{target}" not in lines
|
|
175
|
+
assert "!pubs/" not in lines
|
|
176
|
+
assert "pubs/*" not in lines
|
|
177
|
+
assert "!pubs/applied-ocean-research-model/" not in lines
|
|
178
|
+
assert "pubs/applied-ocean-research-model/*" not in lines
|
|
179
|
+
assert not repo.ignored(target)
|
|
180
|
+
assert not repo.ignored(sibling)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def test_ensure_path_is_not_ignored_nested_gitignore_direct_path_rule(tmp_dir):
|
|
184
|
+
repo = git.Repo.init()
|
|
185
|
+
os.makedirs("paper", exist_ok=True)
|
|
186
|
+
target = "paper/main.pdf"
|
|
187
|
+
with open("paper/.gitignore", "w") as f:
|
|
188
|
+
f.write("/main.pdf\n")
|
|
189
|
+
with open(target, "w") as f:
|
|
190
|
+
f.write("pdf\n")
|
|
191
|
+
assert repo.ignored(target)
|
|
192
|
+
result = calkit.git.ensure_path_is_not_ignored(repo, path=target)
|
|
193
|
+
assert result is True
|
|
194
|
+
with open("paper/.gitignore") as f:
|
|
195
|
+
lines = f.read().splitlines()
|
|
196
|
+
assert "/main.pdf" not in lines
|
|
197
|
+
assert "!/main.pdf" not in lines
|
|
198
|
+
assert not os.path.exists(".gitignore")
|
|
199
|
+
assert not repo.ignored(target)
|
calkit/tests/test_pipeline.py
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
"""Tests for ``calkit.pipeline``."""
|
|
2
2
|
|
|
3
|
+
import os
|
|
3
4
|
import subprocess
|
|
4
5
|
|
|
6
|
+
import git
|
|
5
7
|
import pytest
|
|
6
8
|
|
|
9
|
+
import calkit
|
|
7
10
|
import calkit.pipeline
|
|
8
11
|
from calkit.environments import get_env_lock_fpath
|
|
9
12
|
from calkit.pipeline import stages_are_similar
|
|
@@ -648,3 +651,100 @@ def test_shell_script_stage_allows_non_composite_slurm_env():
|
|
|
648
651
|
"calkit slurm batch --name run --environment mycluster "
|
|
649
652
|
"-- scripts/run.sh a b"
|
|
650
653
|
)
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def test_gitignore_updated_when_stage_output_renamed(tmp_dir):
|
|
657
|
+
"""When a stage output path is renamed, stale .gitignore entries are
|
|
658
|
+
replaced.
|
|
659
|
+
|
|
660
|
+
Use the .gitignore contents for verification because case-only renames can
|
|
661
|
+
still appear ignored on case-insensitive filesystems like the default macOS
|
|
662
|
+
setup.
|
|
663
|
+
"""
|
|
664
|
+
subprocess.check_call(["calkit", "init"])
|
|
665
|
+
# Stage 1: initial calkit.yaml with output 'b_sparsity_plot.pdf' stored in DVC
|
|
666
|
+
ck_info = {
|
|
667
|
+
"pipeline": {
|
|
668
|
+
"stages": {
|
|
669
|
+
"plot": {
|
|
670
|
+
"kind": "command",
|
|
671
|
+
"environment": "_system",
|
|
672
|
+
"command": "touch b_sparsity_plot.pdf",
|
|
673
|
+
"outputs": [
|
|
674
|
+
{
|
|
675
|
+
"path": "b_sparsity_plot.pdf",
|
|
676
|
+
"storage": "dvc",
|
|
677
|
+
}
|
|
678
|
+
],
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
with open("calkit.yaml", "w") as f:
|
|
684
|
+
calkit.ryaml.dump(ck_info, f)
|
|
685
|
+
subprocess.check_call(["calkit", "run"])
|
|
686
|
+
# Verify DVC has added the old output path to .gitignore
|
|
687
|
+
repo = git.Repo(".")
|
|
688
|
+
assert repo.ignored("b_sparsity_plot.pdf")
|
|
689
|
+
with open(".gitignore") as f:
|
|
690
|
+
lines = f.read().splitlines()
|
|
691
|
+
assert "/b_sparsity_plot.pdf" in lines
|
|
692
|
+
# Stage 2: rename output (capitalization change) to 'B_sparsity_plot.pdf'
|
|
693
|
+
ck_info["pipeline"]["stages"]["plot"]["command"] = (
|
|
694
|
+
"touch B_sparsity_plot.pdf"
|
|
695
|
+
)
|
|
696
|
+
ck_info["pipeline"]["stages"]["plot"]["outputs"] = [
|
|
697
|
+
{"path": "B_sparsity_plot.pdf", "storage": "dvc"}
|
|
698
|
+
]
|
|
699
|
+
with open("calkit.yaml", "w") as f:
|
|
700
|
+
calkit.ryaml.dump(ck_info, f)
|
|
701
|
+
subprocess.check_call(["calkit", "run"])
|
|
702
|
+
with open(".gitignore") as f:
|
|
703
|
+
lines = f.read().splitlines()
|
|
704
|
+
assert "/b_sparsity_plot.pdf" not in lines
|
|
705
|
+
assert "/B_sparsity_plot.pdf" in lines
|
|
706
|
+
assert repo.ignored("B_sparsity_plot.pdf")
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def test_gitignore_not_unignored_latex_pdf_output(tmp_dir):
|
|
710
|
+
repo = git.Repo.init()
|
|
711
|
+
subprocess.check_call(["calkit", "init"])
|
|
712
|
+
os.makedirs("paper", exist_ok=True)
|
|
713
|
+
with open("paper/.gitignore", "w") as f:
|
|
714
|
+
f.write("/main.pdf\n")
|
|
715
|
+
ck_info = {
|
|
716
|
+
"environments": {
|
|
717
|
+
"tex": {
|
|
718
|
+
"kind": "conda",
|
|
719
|
+
"path": "environment.yaml",
|
|
720
|
+
}
|
|
721
|
+
},
|
|
722
|
+
"pipeline": {
|
|
723
|
+
"stages": {
|
|
724
|
+
"build-paper": {
|
|
725
|
+
"kind": "latex",
|
|
726
|
+
"environment": "tex",
|
|
727
|
+
"target_path": "paper/main.tex",
|
|
728
|
+
"force": True,
|
|
729
|
+
"inputs": [
|
|
730
|
+
"paper/references.bib",
|
|
731
|
+
"paper/aasjournal.bst",
|
|
732
|
+
"paper/aastex631.cls",
|
|
733
|
+
"paper/results.tex",
|
|
734
|
+
"paper/diagrams",
|
|
735
|
+
"paper/figures",
|
|
736
|
+
],
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
},
|
|
740
|
+
}
|
|
741
|
+
with open("calkit.yaml", "w") as f:
|
|
742
|
+
calkit.ryaml.dump(ck_info, f)
|
|
743
|
+
calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
|
|
744
|
+
assert not os.path.exists(".gitignore")
|
|
745
|
+
assert repo.ignored("paper/main.pdf")
|
|
746
|
+
calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
|
|
747
|
+
assert not os.path.exists(".gitignore")
|
|
748
|
+
with open("paper/.gitignore") as f:
|
|
749
|
+
assert f.read().splitlines() == ["/main.pdf"]
|
|
750
|
+
assert repo.ignored("paper/main.pdf")
|
|
@@ -3,7 +3,7 @@ calkit/__main__.py,sha256=whqS7I7q_c9004emvKptZ9B9OVn-jgr5kaLaCFKmwa0,112
|
|
|
3
3
|
calkit/calc.py,sha256=ucTWBssZhDDc585q2R6l5Zb-924vxwGID4KX7K-eUPA,7563
|
|
4
4
|
calkit/check.py,sha256=wL7xMoyX8yT8lGuy0KBMObq_fjaiCdqC5dhJcX6JjL4,9075
|
|
5
5
|
calkit/cloud.py,sha256=Gc9l8GlW3g6DKKUuvFtFxrkbVq3QPukYXXwWziiw2sU,3277
|
|
6
|
-
calkit/conda.py,sha256=
|
|
6
|
+
calkit/conda.py,sha256=ZSF9bDiC73WMeZr5uQot4uFIUXlF79vYmyDIkZpqWkk,22498
|
|
7
7
|
calkit/config.py,sha256=_LebIjIRHlWbmFBa5vibS2VZtqzf0bmDA0jPbG3Aqm8,7546
|
|
8
8
|
calkit/core.py,sha256=a9bG3fDSiGH2GX4y97WAUagvDSIKCNIOLQYm9hwnPSc,21320
|
|
9
9
|
calkit/datasets.py,sha256=9lOlOreey2aJGMw6MiV9MDzSHp-fYkCf1k1AZbWTdQE,2235
|
|
@@ -12,7 +12,7 @@ calkit/docker.py,sha256=EEcgbXjs-1ttixAZ-G-hmDlvnqSxYbIZmbzeAza37TI,15706
|
|
|
12
12
|
calkit/dvc.py,sha256=aLQKB14Fg0L-Lxr1Rz0WE8hQ72MF8pALNlBP6-4wg3E,16999
|
|
13
13
|
calkit/environments.py,sha256=Y5GufgcPsEAz_NfZ-DAC5mxRUiJ1G6h48vG956Gd7Q0,63270
|
|
14
14
|
calkit/fs.py,sha256=T7AwoNGfZpGnUsksUXU31_98BDfqwboFvF2ho7N3ATI,39741
|
|
15
|
-
calkit/git.py,sha256=
|
|
15
|
+
calkit/git.py,sha256=MDm0ARxg8eqVzGcYzEHv7l0pvBnhgTgJj9c3cGEX-tM,11327
|
|
16
16
|
calkit/github.py,sha256=8_yZ0ej8TKM4qk8iPeGq47R1I4MHdSoJ_Zpr1gRwNEM,1701
|
|
17
17
|
calkit/gui.py,sha256=2UCrMyosc5v4CLlDcHO7oDwW9eLn5_WbNaLaG5bjMTc,23
|
|
18
18
|
calkit/invenio.py,sha256=dU1Mw6G9R0LSNp4vKnJU3P7RcPXrteYV7J5ThBMvJ20,3468
|
|
@@ -24,8 +24,8 @@ calkit/matlab.py,sha256=dthWLDhtpE10qHkJy80h0QzKlbUR1FxFwPTth4-Vs0s,26060
|
|
|
24
24
|
calkit/notebooks.py,sha256=7OISqRi1sOtPO5xFoi41zKxOu0kNIvp8U5WDI2PZcYw,8800
|
|
25
25
|
calkit/office.py,sha256=mZqCxkIsmWthBAeJZ5SiGe21fn5zD2zXsy1E6nQIpWE,1291
|
|
26
26
|
calkit/ops.py,sha256=lLlZAFeplU8uLeMavr99d__Iof-YYwKijus74WfLipk,854
|
|
27
|
-
calkit/overleaf.py,sha256=
|
|
28
|
-
calkit/pipeline.py,sha256=
|
|
27
|
+
calkit/overleaf.py,sha256=0LcVvMmYu8sfwEGG1GTPedNuk7hOIQTr6Hwv4u9S5Xs,23540
|
|
28
|
+
calkit/pipeline.py,sha256=HxfdVM4j9rwpYtnU4X5sVR_ZrD6V1rAVvz6l-teHjMI,13000
|
|
29
29
|
calkit/releases.py,sha256=gIjSVhrFpNdNXUAGqDt4C-85cc06wAESJUbCGdau0uo,5598
|
|
30
30
|
calkit/server.py,sha256=sKJpSzfBnSqipQ9MJW70AUZe5Hosnv7E9UwWDUTs2Vc,21700
|
|
31
31
|
calkit/cli/__init__.py,sha256=WE2C1rXrwby5mEs8NGbGLwt4K-4uO7PHUoYS4GkxBKo,92
|
|
@@ -66,21 +66,21 @@ calkit/templates/latex/jfm/upmath.sty,sha256=9nzNixqQd18zVrv0vLBjGjfnc8L19tP_m0b
|
|
|
66
66
|
calkit/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
67
67
|
calkit/tests/test_calc.py,sha256=h-wOUIRqfyi06LTzhY_rS-a1xChZTkmwdEfBybMUM7E,2033
|
|
68
68
|
calkit/tests/test_check.py,sha256=ngRZq1_dSRQoa0oSjcx_QpR94omG9Bpas_h0wqjwrK0,1475
|
|
69
|
-
calkit/tests/test_conda.py,sha256=
|
|
69
|
+
calkit/tests/test_conda.py,sha256=z-v3GGrFkog7dpvpKwic7MBUHyVoXz1YChKtJ3LeGCE,11440
|
|
70
70
|
calkit/tests/test_core.py,sha256=kcwbDibBR9b-aBKG_ALvqnpZGuJXF9_T7cj9IQ39Xsw,3873
|
|
71
71
|
calkit/tests/test_detect.py,sha256=GT08cpfFIeI133iiSz-i4qAQy4VA9GKNXjegDuCvQu8,32899
|
|
72
72
|
calkit/tests/test_docker.py,sha256=27d0K1GOWiCNv_5TkZSob6EaC0EtQ9DSBIDfkEAEzGk,2077
|
|
73
73
|
calkit/tests/test_dvc.py,sha256=so6db2K_amSmA1n3fVOsr0gzxEBoExC2HedJgLPvfi4,2570
|
|
74
74
|
calkit/tests/test_environments.py,sha256=MWoWu9zCusXvhQdVdlsBcRMeSqBEcq5-Hy_H49ygtLA,25029
|
|
75
75
|
calkit/tests/test_fs.py,sha256=i6VfBime4ay6e9jpbbF2CekfDdPufdlZUZWIb4DdP4w,4928
|
|
76
|
-
calkit/tests/test_git.py,sha256=
|
|
76
|
+
calkit/tests/test_git.py,sha256=oUg02LpMZF6sTr864iGXH8AVes2NCCrV_H9m7RtQJaQ,7053
|
|
77
77
|
calkit/tests/test_invenio.py,sha256=RpiwgcYu0yIpl2gOMTkAQNsSStYPZGa2ReFnA4KCt94,296
|
|
78
78
|
calkit/tests/test_julia.py,sha256=V_q-6wVuIJ-Qj1xkstdZliTqKh0qTeIFjYx4sYETeqA,1234
|
|
79
79
|
calkit/tests/test_jupyter.py,sha256=YTL6zI740UM2KUjskSipzvSKxsyQ8rVPFQIX5cb2tMQ,114
|
|
80
80
|
calkit/tests/test_magics.py,sha256=1O_hYPLMBVQJj8rLCFV3sU2DejlLLnk6Tv4_Tr-FZHQ,2101
|
|
81
81
|
calkit/tests/test_matlab.py,sha256=7yFKOYDCtxYIkRyFOwTIxlvzx03Pac0FKGIUT4Zx7zA,7589
|
|
82
82
|
calkit/tests/test_notebooks.py,sha256=6xUpHjCbCv08oJkArj-NkTUMlz9zzm8pf1qam3s27M0,2692
|
|
83
|
-
calkit/tests/test_pipeline.py,sha256=
|
|
83
|
+
calkit/tests/test_pipeline.py,sha256=P_q0cJoaUnZZK2pfamuHqsMVEeEGufk-GS4tjcYGSDE,25540
|
|
84
84
|
calkit/tests/test_releases.py,sha256=zsy3JMg69uZP5aNPO_TtaP-iPmFu0mDQ7bdruDd1jKU,1478
|
|
85
85
|
calkit/tests/test_templates.py,sha256=RN4RIgmuFlmFYQFgfJUYS-KaLmie8fnWzmfg5Ay1egk,430
|
|
86
86
|
calkit/tests/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -91,7 +91,7 @@ calkit/tests/cli/test_latex.py,sha256=FOiLlG7PMzQk8gGc9BeqdSJ7F0HoPbqZZ0F5sqvSI7
|
|
|
91
91
|
calkit/tests/cli/test_list.py,sha256=h7p5jt8B-7x-GYWjggB6YyrOHXXlywnnebgvEcLKmvs,1024
|
|
92
92
|
calkit/tests/cli/test_new.py,sha256=OqeSuayDNrUkNJ7TJYrE3-O47TCrPesU-j0aSmGIOM4,18044
|
|
93
93
|
calkit/tests/cli/test_notebooks.py,sha256=fVE4J6qTxJvALQeyaBTRxX_aEmMdszp2YUczrAdan3M,6776
|
|
94
|
-
calkit/tests/cli/test_overleaf.py,sha256=
|
|
94
|
+
calkit/tests/cli/test_overleaf.py,sha256=FPzz4nlEJpMt0NEwr-3cTYr3uhOzWT0IIwjwyrqROt8,10562
|
|
95
95
|
calkit/tests/cli/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
96
96
|
calkit/tests/cli/main/test_core.py,sha256=JzwfYZAY3LhpE_b6x04zrd3eGoys9bCrcL4GZQDHtSQ,25102
|
|
97
97
|
calkit/tests/cli/main/test_xr.py,sha256=UhtzJEjRiHgE_p_tdjGyYS7I9yy_Ubs2W2lSmLPpBLU,22386
|
|
@@ -100,23 +100,23 @@ calkit/tests/jupyterlab/test_routes.py,sha256=TEEyoHiKnzzKOkQsJcMPKdqpKFQ25yBCvV
|
|
|
100
100
|
calkit/tests/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
101
101
|
calkit/tests/models/test_iteration.py,sha256=vjuDKwyYQAV4w1qF9VJWQEHksq3KRd6gtT39_a-RhnI,644
|
|
102
102
|
calkit/tests/models/test_pipeline.py,sha256=yefZIG0oHO3xBSwEc3BLx6JOuqKr8GwZvlx1eMcr6SI,9429
|
|
103
|
-
calkit_python-0.35.
|
|
104
|
-
calkit_python-0.35.
|
|
105
|
-
calkit_python-0.35.
|
|
106
|
-
calkit_python-0.35.
|
|
107
|
-
calkit_python-0.35.
|
|
108
|
-
calkit_python-0.35.
|
|
109
|
-
calkit_python-0.35.
|
|
110
|
-
calkit_python-0.35.
|
|
111
|
-
calkit_python-0.35.
|
|
112
|
-
calkit_python-0.35.
|
|
113
|
-
calkit_python-0.35.
|
|
114
|
-
calkit_python-0.35.
|
|
115
|
-
calkit_python-0.35.
|
|
116
|
-
calkit_python-0.35.
|
|
117
|
-
calkit_python-0.35.
|
|
118
|
-
calkit_python-0.35.
|
|
119
|
-
calkit_python-0.35.
|
|
120
|
-
calkit_python-0.35.
|
|
121
|
-
calkit_python-0.35.
|
|
122
|
-
calkit_python-0.35.
|
|
103
|
+
calkit_python-0.35.6.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
|
|
104
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/package.json,sha256=XuEPczy1Q_u1I6pu8lF8k01SydcO3-qc8p9W-XcXtiI,6343
|
|
105
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=v3acRho7E8H-L_0D7qH1KwaXdYLhqsDBSOlZ0NtBkNY,6201
|
|
106
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
|
|
107
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
|
|
108
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
|
|
109
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
|
|
110
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/909.651be47ca47390b78a92.js,sha256=ZRvkfKRzkLeKkhSUg3qX-df9A80XTd0F0CPcXMfmb-Q,114572
|
|
111
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
|
|
112
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
|
|
113
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
|
|
114
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.c091821b3d7f2d287a67.js,sha256=wJGCGz1_LSh6Z9lt5VFlfXs92iKQNlLUJshrAQl4ziw,8737
|
|
115
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
|
|
116
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
|
|
117
|
+
calkit_python-0.35.6.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
|
|
118
|
+
calkit_python-0.35.6.dist-info/METADATA,sha256=JZp7hloviRQ4KKKqaP5gmU8KYx9MXdD-bwLbnmXX_zM,8777
|
|
119
|
+
calkit_python-0.35.6.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
120
|
+
calkit_python-0.35.6.dist-info/entry_points.txt,sha256=59JWYjwz2N3yBOk_9vYUzsMsSs-It1LW8gYE8NsTUJ4,113
|
|
121
|
+
calkit_python-0.35.6.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
|
|
122
|
+
calkit_python-0.35.6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|