inmanta-module-git-ops 0.7.2__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/CHANGELOG.md +10 -0
  2. {inmanta_module_git_ops-0.7.2/inmanta_module_git_ops.egg-info → inmanta_module_git_ops-0.8.0}/PKG-INFO +3 -3
  3. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/README.md +2 -2
  4. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/tests/test_unroll.py +22 -3
  5. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_git_ops/cli.py +36 -8
  6. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_git_ops/const.py +6 -0
  7. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0/inmanta_module_git_ops.egg-info}/PKG-INFO +3 -3
  8. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/config.py +1 -1
  9. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/setup.cfg +1 -1
  10. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/slice.py +4 -2
  11. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/store.py +90 -20
  12. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/pytest_inmanta_git_ops/project.py +8 -1
  13. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/setup.cfg +1 -1
  14. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/tests/test_cli.py +60 -0
  15. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/tests/test_migrations.py +60 -0
  16. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/tests/test_slice_schema.py +10 -0
  17. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/MANIFEST.in +0 -0
  18. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/README.md +0 -0
  19. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/.editorconfig +0 -0
  20. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/.gitignore +0 -0
  21. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/MANIFEST.in +0 -0
  22. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/README.md +0 -0
  23. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/files/.gitkeep +0 -0
  24. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/files/slices/fs.json +0 -0
  25. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/inmanta_plugins/example/__init__.py +0 -0
  26. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/inmanta_plugins/example/slices/__init__.py +0 -0
  27. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/inmanta_plugins/example/slices/fs.py +0 -0
  28. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/inmanta_plugins/example/slices/recursive.py +0 -0
  29. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/inmanta_plugins/example/slices/simple.py +0 -0
  30. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/model/_init.cf +0 -0
  31. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/model/slices/_init.cf +0 -0
  32. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/model/slices/fs/_init.cf +0 -0
  33. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/model/slices/fs/unroll.cf +0 -0
  34. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/model/slices/recursive/_init.cf +0 -0
  35. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/model/slices/simple/_init.cf +0 -0
  36. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/pyproject.toml +0 -0
  37. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/requirements.dev.txt +0 -0
  38. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/requirements.txt +0 -0
  39. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/setup.cfg +0 -0
  40. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/templates/.gitkeep +0 -0
  41. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/docs/example/tests/test_basics.py +0 -0
  42. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_git_ops/__init__.py +0 -0
  43. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_git_ops/generator.py +0 -0
  44. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_git_ops/py.typed +0 -0
  45. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_module_git_ops.egg-info/SOURCES.txt +0 -0
  46. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_module_git_ops.egg-info/dependency_links.txt +0 -0
  47. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_module_git_ops.egg-info/entry_points.txt +0 -0
  48. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_module_git_ops.egg-info/not-zip-safe +0 -0
  49. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_module_git_ops.egg-info/requires.txt +0 -0
  50. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_module_git_ops.egg-info/top_level.txt +0 -0
  51. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/__init__.py +0 -0
  52. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/files/.gitkeep +0 -0
  53. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/model/_init.cf +0 -0
  54. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/model/processors.cf +0 -0
  55. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/model/slice/_init.cf +0 -0
  56. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/processors.py +0 -0
  57. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/py.typed +0 -0
  58. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/inmanta_plugins/git_ops/templates/.gitkeep +0 -0
  59. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/pyproject.toml +0 -0
  60. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/pytest_inmanta_git_ops/__init__.py +0 -0
  61. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/pytest_inmanta_git_ops/plugin.py +0 -0
  62. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/pytest_inmanta_git_ops/py.typed +0 -0
  63. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/pytest_inmanta_git_ops/slice.py +0 -0
  64. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/tests/test_basics.py +0 -0
  65. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/tests/test_example.py +0 -0
  66. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/tests/test_generator.py +0 -0
  67. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/tests/test_processors.py +0 -0
  68. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/tests/test_slice_reload.py +0 -0
  69. {inmanta_module_git_ops-0.7.2 → inmanta_module_git_ops-0.8.0}/tests/test_unroll.py +0 -0
@@ -1,5 +1,15 @@
1
1
  # Changelog
2
2
 
3
+ ## v0.8.0 - 2026-11-21
4
+
5
+ - Report compile failures of the git-ops project commands without the python traceback of the cli.
6
+ - Accept slice patterns (`<name>` or `<store>/<name>`, with glob support) as arguments of `git-ops project prune` and as the `slices` parameter of `git_ops_project.prune()`, to only prune the matching slices.
7
+ - Fail with an explicit error, instead of a `KeyError`, when a source slice is re-created while its deletion is not pruned yet.
8
+ - Leave all slice files untouched when the result of a migration can not be written for one of the slices.
9
+ - Fix the `schema_path` description in the json schema of the git-ops config, which was a list instead of a string.
10
+ - Fix model-only attributes missing from the slices after a failed slice write, for the rest of the python process.
11
+ - Fix the migration state file of the first compiled project being reused when compiling another project in the same python process.
12
+
3
13
  ## v0.7.2 - 2026-09-28
4
14
 
5
15
  - Support discriminated unions with a single member (`typing.Annotated[A, pydantic.Field(discriminator="type")]`), to declare a polymorphic relation before its second member exists.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: inmanta-module-git-ops
3
- Version: 0.7.2
3
+ Version: 0.8.0
4
4
  Summary: Declarative parametrization, extension and validation of Inmanta DSL models.
5
5
  Author: Guillaume Everarts de Velp
6
6
  Author-email: edvgui@gmail.com
@@ -94,10 +94,10 @@ def test_fs(git_ops_project: GitOpsProject) -> None:
94
94
 
95
95
  # Write the slice to disk and trigger an update + sync compile
96
96
  assert git_ops_project.write_slice(slice1).version == 1
97
- assert len(git_ops_project.get_instance(slice1).directories) == 0
97
+ assert len(git_ops_project.get_instance(slice1).content) == 0
98
98
 
99
99
  # Mutate the slice, write again, and check the new version
100
- slice1.slice.directories = [fs.Folder(name="a"), fs.Folder(name="b")]
100
+ slice1.slice.content = [fs.Folder(name="a"), fs.Folder(name="b")]
101
101
  assert git_ops_project.write_slice(slice1).version == 2
102
102
 
103
103
  # Remove the slice
@@ -75,10 +75,10 @@ def test_fs(git_ops_project: GitOpsProject) -> None:
75
75
 
76
76
  # Write the slice to disk and trigger an update + sync compile
77
77
  assert git_ops_project.write_slice(slice1).version == 1
78
- assert len(git_ops_project.get_instance(slice1).directories) == 0
78
+ assert len(git_ops_project.get_instance(slice1).content) == 0
79
79
 
80
80
  # Mutate the slice, write again, and check the new version
81
- slice1.slice.directories = [fs.Folder(name="a"), fs.Folder(name="b")]
81
+ slice1.slice.content = [fs.Folder(name="a"), fs.Folder(name="b")]
82
82
  assert git_ops_project.write_slice(slice1).version == 2
83
83
 
84
84
  # Remove the slice
@@ -19,11 +19,12 @@ Contact: edvgui@gmail.com
19
19
  import pytest
20
20
  from inmanta_plugins.example.slices import fs, recursive, simple
21
21
 
22
+ from inmanta.ast import ExternalException
22
23
  from inmanta_plugins.git_ops import const
23
24
  from pytest_inmanta_git_ops.project import GitOpsProject
24
25
 
25
26
 
26
- def test_fs(git_ops_project: GitOpsProject) -> None:
27
+ def test_fs(git_ops_project: GitOpsProject, caplog: pytest.LogCaptureFixture) -> None:
27
28
  git_ops_project.load_stores("import example::slices::fs::unroll")
28
29
 
29
30
  assert git_ops_project.stores is not None
@@ -60,9 +61,18 @@ def test_fs(git_ops_project: GitOpsProject) -> None:
60
61
  == "content[type=folder][name=a]"
61
62
  )
62
63
 
63
- # Prune version after update
64
- git_ops_project.prune()
64
+ # Prune other slices, the versions of this slice are kept, and the
65
+ # patterns which match no slice are reported
66
+ git_ops_project.prune(slices=["other/*", "missing"])
67
+ assert len(slice1.get_versions()) == 2
68
+ assert "No slice matches the prune pattern 'other/*'" in caplog.text
69
+ assert "No slice matches the prune pattern 'missing'" in caplog.text
70
+
71
+ # Prune version after update, the pattern matched a slice
72
+ caplog.clear()
73
+ git_ops_project.prune(slices=[f"{fs.STORE.name}/*"])
65
74
  assert len(slice1.get_versions()) == 1
75
+ assert "No slice matches" not in caplog.text
66
76
 
67
77
  # Remove a folder
68
78
  slice1.slice.content = [
@@ -92,6 +102,12 @@ def test_fs(git_ops_project: GitOpsProject) -> None:
92
102
  assert git_ops_project.remove_slice(slice1).emit_slice(fs.STORE.name).deleted
93
103
  assert len(git_ops_project.get_instance(slice1).content) == 2
94
104
 
105
+ # The slice can not be re-created until its deletion is pruned
106
+ with pytest.raises(ExternalException) as exc:
107
+ git_ops_project.write_slice(slice1)
108
+ assert "is still being deleted" in str(exc.value.__cause__)
109
+ slice1.get_source().delete()
110
+
95
111
  # Prune after delete
96
112
  git_ops_project.prune()
97
113
  assert len(slice1.get_versions()) == 0
@@ -99,6 +115,9 @@ def test_fs(git_ops_project: GitOpsProject) -> None:
99
115
  with pytest.raises(LookupError):
100
116
  git_ops_project.get_instance(slice1)
101
117
 
118
+ # Once pruned, the slice can be re-created
119
+ assert git_ops_project.write_slice(slice1).version == 1
120
+
102
121
 
103
122
  def test_recursive(git_ops_project: GitOpsProject) -> None:
104
123
  git_ops_project.load_stores("import example::slices::recursive")
@@ -22,6 +22,7 @@ import logging
22
22
  import os
23
23
  import pathlib
24
24
  import re
25
+ import shlex
25
26
  import subprocess
26
27
  import sys
27
28
  import tempfile
@@ -33,7 +34,7 @@ import texttable
33
34
  from inmanta.const import ENVIRON_FORCE_TTY
34
35
  from inmanta.module import ModuleV2, Project
35
36
  from inmanta_git_ops import const
36
- from inmanta_plugins.git_ops.store import SLICE_STORE_REGISTRY
37
+ from inmanta_plugins.git_ops.store import SLICE_STORE_REGISTRY, parse_slice_pattern
37
38
 
38
39
  LOGGER = logging.getLogger(__name__)
39
40
 
@@ -226,6 +227,7 @@ def run_compile(
226
227
  inmanta_compile_arg: Sequence[str],
227
228
  *,
228
229
  compile_mode: str,
230
+ env: Mapping[str, str] | None = None,
229
231
  ) -> None:
230
232
  """
231
233
  Run a compile on the current project, in a subprocess, with the given
@@ -234,12 +236,15 @@ def run_compile(
234
236
  :param inmanta_compile_arg: Additional arguments to pass to the inmanta
235
237
  compile command.
236
238
  :param compile_mode: The compile mode the compile should run in.
239
+ :param env: Additional environment variables to pass to the compile.
237
240
  """
238
- subprocess.run(
241
+ process = subprocess.run(
239
242
  inmanta_compile_command(inmanta_compile_arg),
240
- check=True,
241
- env={**os.environ, const.COMPILE_MODE_ENV_VAR: compile_mode},
243
+ env={**os.environ, const.COMPILE_MODE_ENV_VAR: compile_mode, **(env or {})},
242
244
  )
245
+ if process.returncode != 0:
246
+ # The compiler already reported the failure, don't add a traceback
247
+ raise click.ClickException("The compile failed (see logs above)")
243
248
 
244
249
 
245
250
  # Environment variables, defined by inmanta-core, with which the user can
@@ -413,20 +418,43 @@ def sync(inmanta_compile_arg: list[str]) -> None:
413
418
  run_compile(inmanta_compile_arg, compile_mode=const.COMPILE_SYNC)
414
419
 
415
420
 
421
+ def validate_slice_patterns(
422
+ ctx: click.Context, param: click.Parameter, value: tuple[str, ...]
423
+ ) -> tuple[str, ...]:
424
+ """
425
+ Click callback, making sure all the given slice patterns are valid.
426
+ """
427
+ for pattern in value:
428
+ try:
429
+ parse_slice_pattern(pattern)
430
+ except ValueError as e:
431
+ raise click.BadParameter(str(e))
432
+ return value
433
+
434
+
416
435
  @project.command("prune")
436
+ @click.argument("slices", nargs=-1, callback=validate_slice_patterns)
417
437
  @click.option(
418
438
  "--inmanta-compile-arg",
419
439
  multiple=True,
420
440
  help="Additional arguments to pass to the inmanta compile command.",
421
441
  )
422
- def prune(inmanta_compile_arg: list[str]) -> None:
423
- """
442
+ def prune(slices: tuple[str, ...], inmanta_compile_arg: list[str]) -> None:
443
+ r"""
424
444
  Prune the slice store.
425
445
 
426
446
  Remove from the slice store all active slices which have a more recent version
427
- or which are deleted.
447
+ or which are deleted. When SLICES are given, only prune the slices matching
448
+ them, each one being a NAME (in any store) or STORE/NAME, where both the store
449
+ and the name can be glob patterns. Use a backslash to escape a /, a \ or a
450
+ glob character in the store or the name. Quote the patterns so that the
451
+ shell doesn't expand them.
428
452
  """
429
- run_compile(inmanta_compile_arg, compile_mode=const.COMPILE_PRUNE)
453
+ run_compile(
454
+ inmanta_compile_arg,
455
+ compile_mode=const.COMPILE_PRUNE,
456
+ env={const.PRUNE_SLICES_ENV_VAR: shlex.join(slices)},
457
+ )
430
458
 
431
459
 
432
460
  @project.group("slice")
@@ -17,6 +17,7 @@ Contact: edvgui@gmail.com
17
17
  """
18
18
 
19
19
  import os
20
+ import shlex
20
21
  import typing
21
22
 
22
23
  import pydantic
@@ -67,6 +68,11 @@ SLICE_NAME: str | None = os.getenv(SLICE_NAME_ENV_VAR)
67
68
  SLICE_EXTENSION_ENV_VAR = "INMANTA_GIT_OPS_SLICE_EXTENSION"
68
69
  SLICE_EXTENSION: str = os.getenv(SLICE_EXTENSION_ENV_VAR, "json")
69
70
 
71
+ # Slices the prune compile is restricted to, as a space-separated (shell
72
+ # quoted) list of slice patterns. All the slices are pruned when empty.
73
+ PRUNE_SLICES_ENV_VAR = "INMANTA_GIT_OPS_PRUNE_SLICES"
74
+ PRUNE_SLICES: list[str] = shlex.split(os.getenv(PRUNE_SLICES_ENV_VAR, ""))
75
+
70
76
  # File in which the result of a slice command should be written by the
71
77
  # finalizer executing it, so that the cli process can read it back.
72
78
  OUTPUT_FILE_ENV_VAR = "INMANTA_GIT_OPS_OUTPUT_FILE"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: inmanta-module-git-ops
3
- Version: 0.7.2
3
+ Version: 0.8.0
4
4
  Summary: Declarative parametrization, extension and validation of Inmanta DSL models.
5
5
  Author: Guillaume Everarts de Velp
6
6
  Author-email: edvgui@gmail.com
@@ -94,10 +94,10 @@ def test_fs(git_ops_project: GitOpsProject) -> None:
94
94
 
95
95
  # Write the slice to disk and trigger an update + sync compile
96
96
  assert git_ops_project.write_slice(slice1).version == 1
97
- assert len(git_ops_project.get_instance(slice1).directories) == 0
97
+ assert len(git_ops_project.get_instance(slice1).content) == 0
98
98
 
99
99
  # Mutate the slice, write again, and check the new version
100
- slice1.slice.directories = [fs.Folder(name="a"), fs.Folder(name="b")]
100
+ slice1.slice.content = [fs.Folder(name="a"), fs.Folder(name="b")]
101
101
  assert git_ops_project.write_slice(slice1).version == 2
102
102
 
103
103
  # Remove the slice
@@ -34,7 +34,7 @@ class SliceStoreConfig(pydantic.BaseModel):
34
34
  default=None,
35
35
  description=(
36
36
  "The path to the schema file for this slice. "
37
- "When this is set, the schema will be updated every time the slices of this type are updated.",
37
+ "When this is set, the schema will be updated every time the slices of this type are updated."
38
38
  ),
39
39
  )
40
40
 
@@ -1,6 +1,6 @@
1
1
  [metadata]
2
2
  name = inmanta-module-git-ops
3
- version = 0.7.2
3
+ version = 0.8.0
4
4
  description = Declarative parametrization, extension and validation of Inmanta DSL models.
5
5
  long_description = file: README.md
6
6
  long_description_content_type = text/markdown
@@ -70,8 +70,10 @@ def exclude_model_values() -> Generator[None, None, None]:
70
70
  not saved to the slice files (as they have no additional values).
71
71
  """
72
72
  token = SLICE_UPDATE.set(True)
73
- yield
74
- SLICE_UPDATE.reset(token)
73
+ try:
74
+ yield
75
+ finally:
76
+ SLICE_UPDATE.reset(token)
75
77
 
76
78
 
77
79
  def get_optional_type(python_type: type[object]) -> type[object]:
@@ -17,6 +17,7 @@ Contact: edvgui@gmail.com
17
17
  """
18
18
 
19
19
  import collections
20
+ import fnmatch
20
21
  import itertools
21
22
  import json
22
23
  import logging
@@ -110,16 +111,14 @@ class SliceFile[S: slice.SliceObjectABC]:
110
111
  .model_dump(mode="json")
111
112
  )
112
113
 
113
- def write_raw(self, attributes: dict) -> None:
114
+ def dump_raw(self, attributes: dict) -> str:
114
115
  """
115
- Write the given raw attributes to the slice file, preserving the
116
- file's extension, without applying any schema validation. This
117
- is useful for migrations, where the attributes may not yet
118
- conform to the current schema.
116
+ Serialize the given raw attributes in the format matching the file's
117
+ extension, without applying any schema validation.
119
118
  """
120
119
  if self.extension == "json":
121
120
  try:
122
- return self.path.write_text(json.dumps(attributes, indent=2))
121
+ return json.dumps(attributes, indent=2)
123
122
  except TypeError as e:
124
123
  raise ValueError(
125
124
  f"Attributes can not be serialized: {attributes}"
@@ -127,7 +126,7 @@ class SliceFile[S: slice.SliceObjectABC]:
127
126
 
128
127
  if self.extension in ["yaml", "yml"]:
129
128
  try:
130
- return self.path.write_text(yaml.safe_dump(attributes, sort_keys=False))
129
+ return yaml.safe_dump(attributes, sort_keys=False)
131
130
  except yaml.error.YAMLError as e:
132
131
  raise ValueError(
133
132
  f"Attributes can not be serialized: {attributes}"
@@ -135,6 +134,15 @@ class SliceFile[S: slice.SliceObjectABC]:
135
134
 
136
135
  raise ValueError(f"Unsupported slice file extension: {self.extension}")
137
136
 
137
+ def write_raw(self, attributes: dict) -> None:
138
+ """
139
+ Write the given raw attributes to the slice file, preserving the
140
+ file's extension, without applying any schema validation. This
141
+ is useful for migrations, where the attributes may not yet
142
+ conform to the current schema.
143
+ """
144
+ self.path.write_text(self.dump_raw(attributes))
145
+
138
146
  def write(self, attributes: dict) -> None:
139
147
  """
140
148
  Write the given attributes to the slice file. The parent folder
@@ -524,6 +532,13 @@ class SliceStore[S: slice.SliceObjectABC]:
524
532
 
525
533
  # Load the source slice and compare it to the latest slice
526
534
  attributes = slice_file.read()
535
+ if latest.deleted and attributes != {}:
536
+ raise ValueError(
537
+ f"Slice {name} in store {self.name} is still being deleted "
538
+ f"(v{latest.version}), it can not be re-created until its deletion "
539
+ f"has been exported and pruned (git-ops project prune "
540
+ f"'{self.name}/{name}'). Source file: {slice_file.path}"
541
+ )
527
542
  if attributes == latest.attributes:
528
543
  # Same attributes, same version, same slice
529
544
  self.source_slices[name] = latest
@@ -747,11 +762,6 @@ class SliceStore[S: slice.SliceObjectABC]:
747
762
 
748
763
  :param name: The name of the migration to apply.
749
764
  """
750
- if name not in self.migrations:
751
- raise ValueError(
752
- "No migration with name %s registered for store %s", name, self.name
753
- )
754
-
755
765
  LOGGER.debug(f"Applying migration {name} for store {self.name}")
756
766
  migration_func = self.migrations[name]
757
767
 
@@ -761,7 +771,7 @@ class SliceStore[S: slice.SliceObjectABC]:
761
771
  self.load_source_slice_files().values(),
762
772
  *self.load_active_slice_files().values(),
763
773
  )
764
- updated_slices: list[tuple[SliceFile, dict]] = []
774
+ updated_slices: list[tuple[SliceFile, str]] = []
765
775
  for slice_file in slice_files:
766
776
  raw_attributes = slice_file.read_raw()
767
777
  if raw_attributes == {}:
@@ -776,18 +786,20 @@ class SliceStore[S: slice.SliceObjectABC]:
776
786
  raw_attributes,
777
787
  )
778
788
  try:
779
- migrated_attributes = migration_func(raw_attributes)
789
+ # Serialize the result too, so that no file is written if the
790
+ # result of the migration can not be saved for any slice
791
+ migrated_content = slice_file.dump_raw(migration_func(raw_attributes))
780
792
  except Exception as e:
781
793
  raise RuntimeError(
782
794
  f"Error occurred while applying migration {name} to slice {slice_file.path}: {e}"
783
795
  ) from e
784
796
 
785
- updated_slices.append((slice_file, migrated_attributes))
797
+ updated_slices.append((slice_file, migrated_content))
786
798
 
787
799
  # Then save the migrated attributes for all the slices, now that we know the migration works on all of them
788
- for slice_file, migrated_attributes in updated_slices:
800
+ for slice_file, migrated_content in updated_slices:
789
801
  try:
790
- slice_file.write_raw(migrated_attributes)
802
+ slice_file.path.write_text(migrated_content)
791
803
  except Exception as e:
792
804
  raise RuntimeError(
793
805
  f"Error occurred while writing migrated attributes for slice {slice_file.path} during migration {name}: {e}"
@@ -859,16 +871,30 @@ class SliceStore[S: slice.SliceObjectABC]:
859
871
  )
860
872
  slice_file.write(slice.attributes)
861
873
 
862
- def prune(self) -> None:
874
+ def prune(self) -> set[str]:
863
875
  """
864
876
  Remove all the slices from the store that are not in use.
865
877
  These are the slices which have a more recent version, or which are deleted.
878
+ If const.PRUNE_SLICES is not empty, only the slices it designates are pruned.
879
+ Return the patterns of const.PRUNE_SLICES which matched a slice of this store.
866
880
  """
867
881
  if const.COMPILE_MODE != const.COMPILE_PRUNE:
868
882
  raise RuntimeError("Slices can only be pruned during a prune compile")
869
883
 
884
+ patterns = {p: parse_slice_pattern(p) for p in const.PRUNE_SLICES}
885
+ matched: set[str] = set()
870
886
  active_slice_files = self.load_active_slice_files()
871
- for slice_files in active_slice_files.values():
887
+ for name, slice_files in active_slice_files.items():
888
+ matching = {
889
+ pattern
890
+ for pattern, (store_pattern, name_pattern) in patterns.items()
891
+ if fnmatch.fnmatchcase(self.name, store_pattern)
892
+ and fnmatch.fnmatchcase(name, name_pattern)
893
+ }
894
+ if patterns and not matching:
895
+ continue
896
+ matched |= matching
897
+
872
898
  latest_slice_file = max(slice_files, key=lambda s: s.version or 0)
873
899
  for slice_file in slice_files:
874
900
  if slice_file != latest_slice_file:
@@ -879,6 +905,8 @@ class SliceStore[S: slice.SliceObjectABC]:
879
905
  # The latest slice file is deleted, it should be pruned too
880
906
  latest_slice_file.delete()
881
907
 
908
+ return matched
909
+
882
910
  def update(self) -> None:
883
911
  """
884
912
  Save all the source slices in the store back to file. If a schema
@@ -1098,6 +1126,41 @@ def get_store(store_name: str) -> SliceStore[slice.SliceObjectABC]:
1098
1126
  return SLICE_STORE_REGISTRY[store_name]
1099
1127
 
1100
1128
 
1129
+ def parse_slice_pattern(pattern: str) -> tuple[str, str]:
1130
+ r"""
1131
+ Parse a slice pattern, either <name> or <store>/<name>, into a glob
1132
+ pattern for the store name ("*" when not specified) and one for the
1133
+ slice name. A backslash escapes the next character, to allow a /, a \
1134
+ or a literal glob character in the store or the name.
1135
+
1136
+ :param pattern: The slice pattern to parse.
1137
+ """
1138
+ parts = [""]
1139
+ escaped = False
1140
+ for char in pattern:
1141
+ if escaped:
1142
+ # Wrap the glob characters in brackets to make fnmatch match them literally
1143
+ parts[-1] += f"[{char}]" if char in "*?[" else char
1144
+ escaped = False
1145
+ elif char == "\\":
1146
+ escaped = True
1147
+ elif char == "/":
1148
+ parts.append("")
1149
+ else:
1150
+ parts[-1] += char
1151
+
1152
+ match parts:
1153
+ case [name] if not escaped:
1154
+ return "*", name
1155
+ case [store, name] if not escaped:
1156
+ return store, name
1157
+ case _:
1158
+ raise ValueError(
1159
+ f"Invalid slice pattern {pattern!r}, expected <name> or "
1160
+ "<store>/<name> (use \\/ for a / in the store or the name)"
1161
+ )
1162
+
1163
+
1101
1164
  def write_command_output(result: object) -> None:
1102
1165
  """
1103
1166
  Write the result of a slice command to the output file designated by
@@ -1160,17 +1223,23 @@ def persist_store() -> None:
1160
1223
  At the end of the compile, write all slices back to file and clear
1161
1224
  the in-memory cache.
1162
1225
  """
1226
+ pruned_patterns: set[str] = set()
1163
1227
  for store in SLICE_STORE_REGISTRY.values():
1164
1228
  if const.COMPILE_MODE == const.COMPILE_UPDATE:
1165
1229
  store.update()
1166
1230
  elif const.COMPILE_MODE == const.COMPILE_SYNC:
1167
1231
  store.sync()
1168
1232
  elif const.COMPILE_MODE == const.COMPILE_PRUNE:
1169
- store.prune()
1233
+ pruned_patterns |= store.prune()
1170
1234
  else:
1171
1235
  pass
1172
1236
  store.clear()
1173
1237
 
1238
+ if const.COMPILE_MODE == const.COMPILE_PRUNE:
1239
+ for pattern in const.PRUNE_SLICES:
1240
+ if pattern not in pruned_patterns:
1241
+ LOGGER.warning("No slice matches the prune pattern %r", pattern)
1242
+
1174
1243
 
1175
1244
  @finalizer
1176
1245
  def clear_project_paths() -> None:
@@ -1181,6 +1250,7 @@ def clear_project_paths() -> None:
1181
1250
  for store in SLICE_STORE_REGISTRY.values():
1182
1251
  store._source_path = None
1183
1252
  store._active_path = None
1253
+ store._migration_state_path = None
1184
1254
 
1185
1255
 
1186
1256
  def resolve_operation(
@@ -18,6 +18,7 @@ Contact: edvgui@gmail.com
18
18
 
19
19
  import typing
20
20
  import uuid
21
+ from collections.abc import Sequence
21
22
 
22
23
  import pytest
23
24
  from pytest_inmanta.plugin import Project
@@ -260,12 +261,18 @@ class GitOpsProject:
260
261
  ctx.setattr(const, "COMPILE_MODE", const.COMPILE_EXPORT)
261
262
  self.project.compile(self._model(model))
262
263
 
263
- def prune(self, model: str | None = None) -> None:
264
+ def prune(self, model: str | None = None, slices: Sequence[str] = ()) -> None:
264
265
  """
265
266
  Run a pruning compile on the current project. Don't unroll any
266
267
  service, but cleanup any old version of slices present in the
267
268
  project.
269
+
270
+ :param model: The model to compile, defaults to the model of the stores.
271
+ :param slices: Only prune the slices matching these patterns, like the
272
+ arguments of the `git-ops project prune` command. All the slices
273
+ are pruned when empty.
268
274
  """
269
275
  with self.monkeypatch.context() as ctx:
270
276
  ctx.setattr(const, "COMPILE_MODE", const.COMPILE_PRUNE)
277
+ ctx.setattr(const, "PRUNE_SLICES", list(slices))
271
278
  self.project.compile(self._model(model))
@@ -1,6 +1,6 @@
1
1
  [metadata]
2
2
  name = inmanta-module-git-ops
3
- version = 0.7.2
3
+ version = 0.8.0
4
4
  description = Declarative parametrization, extension and validation of Inmanta DSL models.
5
5
  long_description = file: README.md
6
6
  long_description_content_type = text/markdown
@@ -21,8 +21,33 @@ import os
21
21
  import pathlib
22
22
  import subprocess
23
23
 
24
+ import pytest
24
25
  import yaml
25
26
 
27
+ from inmanta_plugins.git_ops.store import parse_slice_pattern
28
+
29
+
30
+ @pytest.mark.parametrize(
31
+ ("pattern", "expected"),
32
+ [
33
+ ("name", ("*", "name")),
34
+ ("store/name", ("store", "name")),
35
+ ("st*/na?e", ("st*", "na?e")),
36
+ ("na\\/me", ("*", "na/me")),
37
+ ("st\\/ore/na\\\\me", ("st/ore", "na\\me")),
38
+ ("na\\*me\\?\\[", ("*", "na[*]me[?][[]")),
39
+ ("", ("*", "")),
40
+ ],
41
+ )
42
+ def test_parse_slice_pattern(pattern: str, expected: tuple[str, str]) -> None:
43
+ assert parse_slice_pattern(pattern) == expected
44
+
45
+
46
+ @pytest.mark.parametrize("pattern", ["a/b/c", "name\\"])
47
+ def test_parse_invalid_slice_pattern(pattern: str) -> None:
48
+ with pytest.raises(ValueError):
49
+ parse_slice_pattern(pattern)
50
+
26
51
 
27
52
  def test_basics() -> None:
28
53
  example_path = pathlib.Path(__file__).parent.parent / "docs/example"
@@ -77,6 +102,7 @@ def test_project_slice_commands(tmp_path: pathlib.Path) -> None:
77
102
  )
78
103
  if expect_failure:
79
104
  assert result.returncode != 0, result.stdout + result.stderr
105
+ assert "Traceback" not in result.stderr, result.stderr
80
106
  else:
81
107
  assert result.returncode == 0, result.stdout + result.stderr
82
108
  if args[0] == "slice":
@@ -207,3 +233,37 @@ def test_project_slice_commands(tmp_path: pathlib.Path) -> None:
207
233
  json.loads(git_ops("slice", "list", "--store", "simple", "--format", "json"))
208
234
  == []
209
235
  )
236
+
237
+ # Sync the deletion: the active store now holds the deleted version of
238
+ # the slice next to its previous version
239
+ empty_source.unlink()
240
+ git_ops("sync")
241
+ fs_active = project_path / "git_ops" / "active" / "fs"
242
+ assert sorted(p.name for p in fs_active.glob("*@v*")) == [
243
+ "test-folder@v1.json",
244
+ "test-folder@v2.json",
245
+ ]
246
+
247
+ # An invalid slice pattern is rejected
248
+ git_ops("prune", "fs/test-folder/extra", expect_failure=True)
249
+ assert len(list(fs_active.glob("*@v*"))) == 2
250
+
251
+ # Pruning other slices leaves this one untouched, escaped characters
252
+ # are part of the slice name
253
+ git_ops(
254
+ "prune",
255
+ "missing slice",
256
+ "simple/test-folder",
257
+ "fs\\/test-folder",
258
+ "fs/test-\\*",
259
+ )
260
+ assert len(list(fs_active.glob("*@v*"))) == 2
261
+
262
+ # Pruning the slice, with glob patterns on the store and the name,
263
+ # removes all its versions
264
+ git_ops("prune", "fs/missing", "f?/test-*")
265
+ assert list(fs_active.glob("*@v*")) == []
266
+
267
+ # A failing compile is reported without the traceback of the cli
268
+ (project_path / "main.cf").write_text("x = y")
269
+ git_ops("update", expect_failure=True)
@@ -244,3 +244,63 @@ def test_migrations_initial_compile(project: Project, tmp_path: pathlib.Path) ->
244
244
  store.migrations.keys()
245
245
  )
246
246
  assert store.pending_migrations() == []
247
+
248
+
249
+ def test_migration_serialization_failure(
250
+ project: Project, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
251
+ ) -> None:
252
+ """
253
+ A migration whose result can not be written for one of the slice files
254
+ must not modify any slice file, and must stay pending.
255
+ """
256
+ store = SliceStore(
257
+ name="test_migration_serialization_failure",
258
+ folder="file://" + str(tmp_path / "test"),
259
+ schema=Slice,
260
+ )
261
+
262
+ model = """
263
+ import git_ops
264
+ git_ops::unroll_slices("test_migration_serialization_failure")
265
+ """
266
+ project.compile(model)
267
+
268
+ # Activate one slice, then change its source so that the source and
269
+ # the active versions differ
270
+ s1 = store.source_path / "s1.json"
271
+ s1.parent.mkdir(parents=True, exist_ok=True)
272
+ s1.write_text(
273
+ json.dumps(
274
+ Slice(name="a", embedded_required=EmbeddedSlice(name="aa")).model_dump(
275
+ mode="json"
276
+ )
277
+ )
278
+ )
279
+ with monkeypatch.context() as ctx:
280
+ ctx.setattr(const, "COMPILE_MODE", const.COMPILE_UPDATE)
281
+ project.compile(model, no_dedent=False)
282
+ ctx.setattr(const, "COMPILE_MODE", const.COMPILE_SYNC)
283
+ project.compile(model, no_dedent=False)
284
+
285
+ s1.write_text(json.dumps({**json.loads(s1.read_text()), "name": "b"}))
286
+ s1_v1 = store.active_path / "s1@v1.json"
287
+ source_content = s1.read_text()
288
+ active_content = s1_v1.read_text()
289
+
290
+ # The source version (processed first) can be migrated, the active
291
+ # version can not be serialized
292
+ @store.migration("001_broken")
293
+ def broken(attrs: dict) -> dict:
294
+ if attrs["name"] == "a":
295
+ return {**attrs, "description": {"not", "json"}}
296
+ return {**attrs, "description": "migrated"}
297
+
298
+ with pytest.raises(ExternalException) as exc:
299
+ with monkeypatch.context() as ctx:
300
+ ctx.setattr(const, "COMPILE_MODE", const.COMPILE_UPDATE)
301
+ project.compile(model, no_dedent=False)
302
+
303
+ assert "001_broken" in str(exc.value.__cause__)
304
+ assert s1.read_text() == source_content
305
+ assert s1_v1.read_text() == active_content
306
+ assert store.pending_migrations() == ["001_broken"]
@@ -20,6 +20,7 @@ import typing
20
20
  from collections.abc import Sequence
21
21
 
22
22
  import pydantic
23
+ import pytest
23
24
  from inmanta_plugins.example.slices.fs import File, Folder, RootFolder
24
25
  from inmanta_plugins.example.slices.recursive import EmbeddedSlice, Slice
25
26
  from inmanta_plugins.example.slices.simple import Slice as SimpleSlice
@@ -264,3 +265,12 @@ def test_single_member_discriminated_union() -> None:
264
265
  items = next(r for r in Root.entity_schema().all_relations() if r.name == "items")
265
266
  assert items.entity.discriminator == "type"
266
267
  assert [sub.name for sub in items.entity.sub_entities] == ["A"]
268
+
269
+
270
+ def test_exclude_model_values_reset_on_error() -> None:
271
+ # An error raised inside the context must not leave the model-only
272
+ # attributes excluded for the rest of the process
273
+ with pytest.raises(RuntimeError):
274
+ with slice.exclude_model_values():
275
+ raise RuntimeError()
276
+ assert not slice.slice_update(None)