FourCIPP 1.2.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fourcipp-1.2.0/src/FourCIPP.egg-info → fourcipp-1.4.0}/PKG-INFO +2 -2
- {fourcipp-1.2.0 → fourcipp-1.4.0}/README.md +1 -1
- {fourcipp-1.2.0 → fourcipp-1.4.0/src/FourCIPP.egg-info}/PKG-INFO +2 -2
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/fourc_input.py +79 -5
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/cli.py +41 -1
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/configuration.py +4 -3
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/dict_utils.py +32 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/yaml_io.py +76 -11
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/version.py +3 -3
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/test_fourc_input.py +49 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/utils/test_dict_utils.py +29 -0
- fourcipp-1.4.0/tests/fourcipp/utils/test_yaml_io.py +126 -0
- fourcipp-1.2.0/tests/fourcipp/utils/test_yaml_io.py +0 -42
- {fourcipp-1.2.0 → fourcipp-1.4.0}/.coveragerc +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/.github/workflows/check_code.yaml +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/.github/workflows/publish_pypi.yaml +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/.github/workflows/run_testsuite.yaml +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/.github/workflows/tag_version.yaml +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/.github/workflows/update_4C_metadata_schema_file.yaml +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/.gitignore +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/.pre-commit-config.yaml +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/LICENSE +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/docs/assets/fourcipp_logo_black.svg +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/docs/assets/fourcipp_logo_white.svg +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/pyproject.toml +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/requirements.in +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/requirements.txt +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/setup.cfg +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/FourCIPP.egg-info/SOURCES.txt +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/FourCIPP.egg-info/dependency_links.txt +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/FourCIPP.egg-info/entry_points.txt +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/FourCIPP.egg-info/requires.txt +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/FourCIPP.egg-info/top_level.txt +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/__init__.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/config/4C_metadata.yaml +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/config/4C_schema.json +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/config/config.yaml +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/legacy_io/__init__.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/legacy_io/element.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/legacy_io/inline_dat.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/legacy_io/node.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/legacy_io/node_topology.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/legacy_io/particle.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/__init__.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/converter.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/metadata.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/not_set.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/typing.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/src/fourcipp/utils/validation.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/__init__.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/conftest.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/__init__.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/legacy_io/__init__.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/legacy_io/test_element.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/legacy_io/test_inline_dat.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/legacy_io/test_legacy_io.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/legacy_io/test_node.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/legacy_io/test_node_topology.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/legacy_io/test_particle.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/legacy_io/utils.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/test_readme_quickstart_example.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/utils/__init__.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/utils/test_configuration.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/utils/test_converter.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/utils/test_not_set.py +0 -0
- {fourcipp-1.2.0 → fourcipp-1.4.0}/tests/fourcipp/utils/test_validation.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: FourCIPP
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: A streamlined Python Parser for 4C input files
|
|
5
5
|
Author: FourCIPP Authors
|
|
6
6
|
License: The MIT License (MIT)
|
|
@@ -168,7 +168,7 @@ input_4C["PROBLEM SIZE"]["NODES"] = 10_000_000
|
|
|
168
168
|
removed_section = input_4C.pop("PROBLEM SIZE")
|
|
169
169
|
|
|
170
170
|
# Dump to file
|
|
171
|
-
input_4C.dump(input_file_path,
|
|
171
|
+
input_4C.dump(input_file_path, validate=True)
|
|
172
172
|
```
|
|
173
173
|
<!--example, do not remove this comment-->
|
|
174
174
|
|
|
@@ -124,7 +124,7 @@ input_4C["PROBLEM SIZE"]["NODES"] = 10_000_000
|
|
|
124
124
|
removed_section = input_4C.pop("PROBLEM SIZE")
|
|
125
125
|
|
|
126
126
|
# Dump to file
|
|
127
|
-
input_4C.dump(input_file_path,
|
|
127
|
+
input_4C.dump(input_file_path, validate=True)
|
|
128
128
|
```
|
|
129
129
|
<!--example, do not remove this comment-->
|
|
130
130
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: FourCIPP
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: A streamlined Python Parser for 4C input files
|
|
5
5
|
Author: FourCIPP Authors
|
|
6
6
|
License: The MIT License (MIT)
|
|
@@ -168,7 +168,7 @@ input_4C["PROBLEM SIZE"]["NODES"] = 10_000_000
|
|
|
168
168
|
removed_section = input_4C.pop("PROBLEM SIZE")
|
|
169
169
|
|
|
170
170
|
# Dump to file
|
|
171
|
-
input_4C.dump(input_file_path,
|
|
171
|
+
input_4C.dump(input_file_path, validate=True)
|
|
172
172
|
```
|
|
173
173
|
<!--example, do not remove this comment-->
|
|
174
174
|
|
|
@@ -27,7 +27,7 @@ import copy
|
|
|
27
27
|
import difflib
|
|
28
28
|
import pathlib
|
|
29
29
|
from collections.abc import Sequence
|
|
30
|
-
from typing import Any
|
|
30
|
+
from typing import Any, Callable
|
|
31
31
|
|
|
32
32
|
from loguru import logger
|
|
33
33
|
|
|
@@ -37,7 +37,10 @@ from fourcipp.legacy_io import (
|
|
|
37
37
|
interpret_legacy_section,
|
|
38
38
|
)
|
|
39
39
|
from fourcipp.utils.converter import Converter
|
|
40
|
-
from fourcipp.utils.dict_utils import
|
|
40
|
+
from fourcipp.utils.dict_utils import (
|
|
41
|
+
compare_nested_dicts_or_lists,
|
|
42
|
+
sort_by_key_order,
|
|
43
|
+
)
|
|
41
44
|
from fourcipp.utils.not_set import NotSet, check_if_set
|
|
42
45
|
from fourcipp.utils.typing import Path
|
|
43
46
|
from fourcipp.utils.validation import ValidationError, validate_using_json_schema
|
|
@@ -66,6 +69,75 @@ def is_section_known(section_name: str, known_section_names: list[str]) -> bool:
|
|
|
66
69
|
return section_name in known_section_names or section_name.startswith("FUNCT")
|
|
67
70
|
|
|
68
71
|
|
|
72
|
+
def _sort_by_section_names(data: dict) -> dict:
|
|
73
|
+
"""Sort a dictionary by its 4C sections.
|
|
74
|
+
|
|
75
|
+
This sorts the dictionary in the following style:
|
|
76
|
+
|
|
77
|
+
1. "TITLE" section
|
|
78
|
+
2. Alphabetically sorted sections
|
|
79
|
+
3. Alphabetically sorted legacy sections
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
data: Dictionary to sort.
|
|
83
|
+
section_names: List of all section names in the 4C style order.
|
|
84
|
+
|
|
85
|
+
Returns:
|
|
86
|
+
Dict sorted in 4C fashion
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
required_sections = CONFIG.fourc_json_schema["required"]
|
|
90
|
+
n_sections_splitter = len(CONFIG.sections.all_sections) * 1000
|
|
91
|
+
|
|
92
|
+
# collect typed sections + numeric FUNCT sections
|
|
93
|
+
typed_and_funct = sorted(
|
|
94
|
+
sorted(CONFIG.sections.typed_sections, key=str.lower)
|
|
95
|
+
+ [s for s in data.keys() if s.startswith("FUNCT") and s[5:].isdigit()],
|
|
96
|
+
key=lambda s: (
|
|
97
|
+
s.lower() if not s.startswith("FUNCT") else f"funct{s[5:].zfill(10)}"
|
|
98
|
+
),
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
def ordering_score(section: str) -> int:
|
|
102
|
+
"""Get ordering score, small score comes first, larger comes later.
|
|
103
|
+
|
|
104
|
+
We offset the score by the number of sections multiplied by 1000. This way a score is guaranteed to never appear twice.
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
section: Section name to score
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
ordering score
|
|
111
|
+
"""
|
|
112
|
+
# Title sections
|
|
113
|
+
if section == CONFIG.fourc_metadata["metadata"]["description_section_name"]:
|
|
114
|
+
return 0
|
|
115
|
+
# Required sections
|
|
116
|
+
elif section in required_sections:
|
|
117
|
+
return 1 * n_sections_splitter + required_sections.index(section)
|
|
118
|
+
# Typed + FUNCT sections (alphabetical + case insensitive)
|
|
119
|
+
elif section in typed_and_funct:
|
|
120
|
+
return 2 * n_sections_splitter + typed_and_funct.index(section)
|
|
121
|
+
# Legacy sections
|
|
122
|
+
elif section in CONFIG.sections.legacy_sections:
|
|
123
|
+
return 3 * n_sections_splitter + CONFIG.sections.legacy_sections.index(
|
|
124
|
+
section
|
|
125
|
+
)
|
|
126
|
+
# Unknown section
|
|
127
|
+
else:
|
|
128
|
+
raise KeyError(f"Unknown section {section}")
|
|
129
|
+
|
|
130
|
+
unknown_sections = set(data.keys()) - set(CONFIG.sections.all_sections)
|
|
131
|
+
|
|
132
|
+
# Remove functions, these are a special case
|
|
133
|
+
if [section for section in unknown_sections if not section.startswith("FUNCT")]:
|
|
134
|
+
raise ValueError(
|
|
135
|
+
f"Sections {list(unknown_sections)} are not known in 'section_names'"
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
return sort_by_key_order(data, sorted(data.keys(), key=ordering_score))
|
|
139
|
+
|
|
140
|
+
|
|
69
141
|
class FourCInput:
|
|
70
142
|
"""4C inout file object."""
|
|
71
143
|
|
|
@@ -420,20 +492,22 @@ class FourCInput:
|
|
|
420
492
|
def dump(
|
|
421
493
|
self,
|
|
422
494
|
input_file_path: Path,
|
|
423
|
-
sort_sections: bool = False,
|
|
424
495
|
validate: bool = False,
|
|
425
496
|
validate_sections_only: bool = False,
|
|
426
497
|
convert_to_native_types: bool = True,
|
|
498
|
+
sort_function: Callable[[dict], dict] | None = _sort_by_section_names,
|
|
499
|
+
use_fourcipp_yaml_style: bool = True,
|
|
427
500
|
) -> None:
|
|
428
501
|
"""Dump object to yaml.
|
|
429
502
|
|
|
430
503
|
Args:
|
|
431
504
|
input_file_path: Path to dump the data to
|
|
432
|
-
sort_sections: Sort the sections alphabetically
|
|
433
505
|
validate: Validate input data before dumping
|
|
434
506
|
validate_sections_only: Validate each section independently.
|
|
435
507
|
Requiredness of the sections themselves is ignored.
|
|
436
508
|
convert_to_native_types: Convert all sections to native Python types
|
|
509
|
+
sort_function: Function to sort the sections.
|
|
510
|
+
use_fourcipp_yaml_style: If FourCIPP yaml style is to be used
|
|
437
511
|
"""
|
|
438
512
|
|
|
439
513
|
if validate or validate_sections_only:
|
|
@@ -448,7 +522,7 @@ class FourCInput:
|
|
|
448
522
|
if convert_to_native_types:
|
|
449
523
|
self.convert_to_native_types()
|
|
450
524
|
|
|
451
|
-
dump_yaml(self.inlined, input_file_path,
|
|
525
|
+
dump_yaml(self.inlined, input_file_path, sort_function, use_fourcipp_yaml_style)
|
|
452
526
|
|
|
453
527
|
def validate(
|
|
454
528
|
self,
|
|
@@ -34,9 +34,12 @@ from fourcipp.utils.configuration import (
|
|
|
34
34
|
show_config,
|
|
35
35
|
)
|
|
36
36
|
from fourcipp.utils.typing import Path
|
|
37
|
+
from fourcipp.utils.yaml_io import dump_yaml, load_yaml
|
|
37
38
|
|
|
38
39
|
|
|
39
|
-
def modify_input_with_defaults(
|
|
40
|
+
def modify_input_with_defaults(
|
|
41
|
+
input_path: Path, overwrite: bool
|
|
42
|
+
) -> None: # pragma: no cover
|
|
40
43
|
"""Apply user defaults to an input file located at input_path.
|
|
41
44
|
|
|
42
45
|
Args:
|
|
@@ -62,6 +65,24 @@ def modify_input_with_defaults(input_path: Path, overwrite: bool) -> None:
|
|
|
62
65
|
logger.info(f"Input file incl. user defaults is now '{output_filename}'.")
|
|
63
66
|
|
|
64
67
|
|
|
68
|
+
def format_file(
|
|
69
|
+
input_file: str, sort_sections: bool = False
|
|
70
|
+
) -> None: # pragma: no cover
|
|
71
|
+
"""Formatting file.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
input_file: File to format
|
|
75
|
+
sort_sections: Sort sections
|
|
76
|
+
"""
|
|
77
|
+
if sort_sections:
|
|
78
|
+
# Requires reading the config
|
|
79
|
+
fourc_input = FourCInput.from_4C_yaml(input_file)
|
|
80
|
+
fourc_input.dump(input_file, use_fourcipp_yaml_style=True)
|
|
81
|
+
else:
|
|
82
|
+
# No config required, is purely a style question
|
|
83
|
+
dump_yaml(load_yaml(input_file), input_file, use_fourcipp_yaml_style=True)
|
|
84
|
+
|
|
85
|
+
|
|
65
86
|
def main() -> None:
|
|
66
87
|
"""Main CLI interface."""
|
|
67
88
|
# Set up the logger
|
|
@@ -110,6 +131,23 @@ def main() -> None:
|
|
|
110
131
|
type=str,
|
|
111
132
|
)
|
|
112
133
|
|
|
134
|
+
# Format parser
|
|
135
|
+
format_parser = subparsers.add_parser(
|
|
136
|
+
"format",
|
|
137
|
+
help="Format the file in fourcipp style. This sorts the sections and uses the flow styles from FourCIPP",
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
format_parser.add_argument(
|
|
141
|
+
"input-file",
|
|
142
|
+
help=f"4C input file.",
|
|
143
|
+
type=str,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
format_parser.add_argument(
|
|
147
|
+
"--sort-sections",
|
|
148
|
+
action="store_true",
|
|
149
|
+
help=f"Overwrite existing input file.",
|
|
150
|
+
)
|
|
113
151
|
# Replace "-" with "_" for variable names
|
|
114
152
|
kwargs: dict = {}
|
|
115
153
|
for key, value in vars(main_parser.parse_args(sys.argv[1:])).items():
|
|
@@ -126,3 +164,5 @@ def main() -> None:
|
|
|
126
164
|
input_path = pathlib.Path(kwargs.pop("input_file"))
|
|
127
165
|
overwrite = kwargs.pop("overwrite")
|
|
128
166
|
modify_input_with_defaults(input_path, overwrite)
|
|
167
|
+
case "format":
|
|
168
|
+
format_file(**kwargs)
|
|
@@ -45,7 +45,7 @@ class Sections:
|
|
|
45
45
|
"""
|
|
46
46
|
self.legacy_sections: list[str] = legacy_sections
|
|
47
47
|
self.typed_sections: list[str] = typed_sections
|
|
48
|
-
self.all_sections: list[str] =
|
|
48
|
+
self.all_sections: list[str] = typed_sections + legacy_sections
|
|
49
49
|
|
|
50
50
|
@classmethod
|
|
51
51
|
def from_metadata(cls, fourc_metadata: dict) -> Sections:
|
|
@@ -58,9 +58,10 @@ class Sections:
|
|
|
58
58
|
Sections: sections object
|
|
59
59
|
"""
|
|
60
60
|
description_section = fourc_metadata["metadata"]["description_section_name"]
|
|
61
|
-
sections = [
|
|
61
|
+
sections = [description_section] + [
|
|
62
62
|
section["name"] for section in fourc_metadata["sections"]["specs"]
|
|
63
|
-
]
|
|
63
|
+
]
|
|
64
|
+
|
|
64
65
|
legacy_sections = list(fourc_metadata["legacy_string_sections"])
|
|
65
66
|
|
|
66
67
|
return cls(legacy_sections, sections)
|
|
@@ -373,3 +373,35 @@ def rename_parameter(
|
|
|
373
373
|
|
|
374
374
|
for entry, last_key in _split_off_last_key(nested_dict, keys):
|
|
375
375
|
entry[new_name] = entry.pop(last_key)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def sort_by_key_order(data: dict, key_order: list[str]) -> dict:
|
|
379
|
+
"""Sort a dictionary by a specific key order.
|
|
380
|
+
|
|
381
|
+
Args:
|
|
382
|
+
data: Dictionary to sort.
|
|
383
|
+
key_order: List of keys in the desired order.
|
|
384
|
+
|
|
385
|
+
Returns:
|
|
386
|
+
Sorted dictionary.
|
|
387
|
+
"""
|
|
388
|
+
if set(key_order) != set(data.keys()):
|
|
389
|
+
raise ValueError("'key_order' must include all keys in the dictionary!")
|
|
390
|
+
|
|
391
|
+
return {key: data[key] for key in key_order if key in data}
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def sort_alphabetically(
|
|
395
|
+
data: dict,
|
|
396
|
+
) -> dict:
|
|
397
|
+
"""Sort a dictionary alphabetically.
|
|
398
|
+
|
|
399
|
+
Args:
|
|
400
|
+
data: Dictionary to sort.
|
|
401
|
+
|
|
402
|
+
Returns:
|
|
403
|
+
Sorted dictionary.
|
|
404
|
+
"""
|
|
405
|
+
return sort_by_key_order(
|
|
406
|
+
data, sorted(data.keys(), key=lambda s: (s.lower(), 0 if s.islower() else 1))
|
|
407
|
+
)
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
import json
|
|
25
25
|
import pathlib
|
|
26
26
|
import re
|
|
27
|
+
from typing import Callable
|
|
27
28
|
|
|
28
29
|
import ryml
|
|
29
30
|
|
|
@@ -61,26 +62,83 @@ def load_yaml(path_to_yaml_file: Path) -> dict:
|
|
|
61
62
|
return data
|
|
62
63
|
|
|
63
64
|
|
|
64
|
-
def dict_to_yaml_string(
|
|
65
|
+
def dict_to_yaml_string(
|
|
66
|
+
data: dict,
|
|
67
|
+
sort_function: Callable[[dict], dict] | None = None,
|
|
68
|
+
use_fourcipp_yaml_style: bool = True,
|
|
69
|
+
) -> str:
|
|
65
70
|
"""Dump dict as yaml.
|
|
66
71
|
|
|
72
|
+
The FourCIPP yaml style sets flow
|
|
67
73
|
Args:
|
|
68
74
|
data: Data to dump.
|
|
69
|
-
|
|
75
|
+
sort_function: Function to sort the data.
|
|
76
|
+
use_fourcipp_yaml_style: If FourCIPP yaml style is to be used
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
YAML string representation of the data
|
|
70
80
|
"""
|
|
71
81
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
data = {key: data[key] for key in sorted(data.keys())}
|
|
82
|
+
if sort_function is not None:
|
|
83
|
+
data = sort_function(data)
|
|
75
84
|
|
|
76
85
|
# Convert dictionary into a ryml tree
|
|
77
86
|
tree = ryml.parse_in_arena(bytearray(json.dumps(data).encode("utf8")))
|
|
78
87
|
|
|
79
|
-
|
|
88
|
+
def check_is_vector(tree: ryml.Tree, node_id: int) -> bool:
|
|
89
|
+
"""Check if sequence is of ints, floats or sequence there of.
|
|
90
|
+
|
|
91
|
+
In 4C metadata terms, list of strings, bools, etc. could also be vectors.
|
|
92
|
+
For the sake of simplicity these are omitted.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
tree (ryml.Tree): Tree to check
|
|
96
|
+
node_id (int): Node id
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
returns if entry is a vector
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
for sub_node, _ in ryml.walk(tree, node_id):
|
|
103
|
+
# Ignore the root node
|
|
104
|
+
if sub_node == node_id:
|
|
105
|
+
continue
|
|
106
|
+
|
|
107
|
+
# If sequence contains a dict
|
|
108
|
+
if tree.is_map(sub_node):
|
|
109
|
+
return False
|
|
110
|
+
|
|
111
|
+
# If sequence contains a sequence
|
|
112
|
+
elif tree.is_seq(sub_node):
|
|
113
|
+
if not check_is_vector(tree, sub_node):
|
|
114
|
+
return False
|
|
115
|
+
|
|
116
|
+
# Else it's a value
|
|
117
|
+
else:
|
|
118
|
+
val = tree.val(sub_node).tobytes().decode("ascii")
|
|
119
|
+
is_not_numeric = (
|
|
120
|
+
tree.is_val_quoted(sub_node) # string
|
|
121
|
+
or tree.val_is_null(sub_node) # null
|
|
122
|
+
or val == "true" # bool
|
|
123
|
+
or val == "false" # bool
|
|
124
|
+
)
|
|
125
|
+
if is_not_numeric:
|
|
126
|
+
return False
|
|
127
|
+
|
|
128
|
+
return True
|
|
129
|
+
|
|
130
|
+
# Change style bits to avoid JSON output and format vectors correctly
|
|
80
131
|
# see https://github.com/biojppm/rapidyaml/issues/520
|
|
81
|
-
for node_id,
|
|
82
|
-
if tree.is_map(node_id)
|
|
132
|
+
for node_id, depth in ryml.walk(tree):
|
|
133
|
+
if tree.is_map(node_id):
|
|
83
134
|
tree.set_container_style(node_id, ryml.NOTYPE)
|
|
135
|
+
elif tree.is_seq(node_id):
|
|
136
|
+
if (
|
|
137
|
+
not use_fourcipp_yaml_style # do not do special formatting
|
|
138
|
+
or depth == 1 # is a section
|
|
139
|
+
or not check_is_vector(tree, node_id) # is not a vector
|
|
140
|
+
):
|
|
141
|
+
tree.set_container_style(node_id, ryml.NOTYPE)
|
|
84
142
|
|
|
85
143
|
if tree.has_key(node_id):
|
|
86
144
|
tree.set_key_style(node_id, ryml.NOTYPE)
|
|
@@ -88,14 +146,21 @@ def dict_to_yaml_string(data: dict, sort_keys: bool = False) -> str:
|
|
|
88
146
|
return ryml.emit_yaml(tree)
|
|
89
147
|
|
|
90
148
|
|
|
91
|
-
def dump_yaml(
|
|
149
|
+
def dump_yaml(
|
|
150
|
+
data: dict,
|
|
151
|
+
path_to_yaml_file: Path,
|
|
152
|
+
sort_function: Callable[[dict], dict] | None = None,
|
|
153
|
+
use_fourcipp_yaml_style: bool = True,
|
|
154
|
+
) -> None:
|
|
92
155
|
"""Dump yaml to file.
|
|
93
156
|
|
|
94
157
|
Args:
|
|
95
158
|
data: Data to dump.
|
|
96
159
|
path_to_yaml_file: Yaml file path
|
|
97
|
-
|
|
160
|
+
sort_function: Function to sort the data
|
|
161
|
+
use_fourcipp_yaml_style: If FourCIPP yaml style is to be used
|
|
98
162
|
"""
|
|
99
163
|
pathlib.Path(path_to_yaml_file).write_text(
|
|
100
|
-
dict_to_yaml_string(data,
|
|
164
|
+
dict_to_yaml_string(data, sort_function, use_fourcipp_yaml_style),
|
|
165
|
+
encoding="utf-8",
|
|
101
166
|
)
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '1.
|
|
32
|
-
__version_tuple__ = version_tuple = (1,
|
|
31
|
+
__version__ = version = '1.4.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (1, 4, 0)
|
|
33
33
|
|
|
34
|
-
__commit_id__ = commit_id = '
|
|
34
|
+
__commit_id__ = commit_id = 'g861e0a123'
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
import contextlib
|
|
25
25
|
import pathlib
|
|
26
|
+
import random
|
|
26
27
|
import subprocess
|
|
27
28
|
import time
|
|
28
29
|
from collections.abc import Callable
|
|
@@ -33,6 +34,7 @@ from fourcipp import CONFIG
|
|
|
33
34
|
from fourcipp.fourc_input import (
|
|
34
35
|
FourCInput,
|
|
35
36
|
UnknownSectionException,
|
|
37
|
+
_sort_by_section_names,
|
|
36
38
|
)
|
|
37
39
|
from fourcipp.utils.cli import modify_input_with_defaults
|
|
38
40
|
from fourcipp.utils.validation import ValidationError
|
|
@@ -552,6 +554,53 @@ def test_validation(fourc_input, error_context, sections_only):
|
|
|
552
554
|
fourc_input.validate(sections_only=sections_only)
|
|
553
555
|
|
|
554
556
|
|
|
557
|
+
def test_sort_by_section_names():
|
|
558
|
+
"""Test sorting by section names."""
|
|
559
|
+
|
|
560
|
+
# create list of typed sections without title and required sections
|
|
561
|
+
typed_sections = [
|
|
562
|
+
sec
|
|
563
|
+
for sec in CONFIG.sections.typed_sections
|
|
564
|
+
if sec != CONFIG.fourc_metadata["metadata"]["description_section_name"]
|
|
565
|
+
and sec not in set(CONFIG.fourc_json_schema["required"])
|
|
566
|
+
]
|
|
567
|
+
|
|
568
|
+
# also use end subset to also add some lowercase sections
|
|
569
|
+
typed_and_functions = (
|
|
570
|
+
typed_sections[:15]
|
|
571
|
+
+ typed_sections[-15:]
|
|
572
|
+
+ [f"FUNCT{i}" for i in [1, 2, 9, 10, 33]]
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
# sort with proper key: alphabetically for typed, numerically for FUNCT
|
|
576
|
+
typed_and_functions = sorted(
|
|
577
|
+
typed_and_functions,
|
|
578
|
+
key=lambda s: (
|
|
579
|
+
s.lower() if not s.startswith("FUNCT") else f"funct{s[5:].zfill(10)}"
|
|
580
|
+
),
|
|
581
|
+
)
|
|
582
|
+
|
|
583
|
+
correct_section_order = (
|
|
584
|
+
[CONFIG.fourc_metadata["metadata"]["description_section_name"]]
|
|
585
|
+
+ CONFIG.fourc_json_schema["required"]
|
|
586
|
+
+ typed_and_functions
|
|
587
|
+
+ CONFIG.sections.legacy_sections
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
shuffled_section_order = correct_section_order.copy()
|
|
591
|
+
random.seed(42)
|
|
592
|
+
random.shuffle(shuffled_section_order)
|
|
593
|
+
|
|
594
|
+
shuffled_data = {k: 1 for k in shuffled_section_order}
|
|
595
|
+
|
|
596
|
+
sorted_data = _sort_by_section_names(shuffled_data)
|
|
597
|
+
|
|
598
|
+
assert list(sorted_data.keys()) == correct_section_order
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
## performance tests
|
|
602
|
+
|
|
603
|
+
|
|
555
604
|
def create_dummy_elements() -> dict:
|
|
556
605
|
"""Create dummy elements for the performance test.
|
|
557
606
|
|
|
@@ -34,6 +34,8 @@ from fourcipp.utils.dict_utils import (
|
|
|
34
34
|
remove,
|
|
35
35
|
rename_parameter,
|
|
36
36
|
replace_value,
|
|
37
|
+
sort_alphabetically,
|
|
38
|
+
sort_by_key_order,
|
|
37
39
|
)
|
|
38
40
|
|
|
39
41
|
|
|
@@ -664,3 +666,30 @@ def test_get_dict_optional(nested_input_dict):
|
|
|
664
666
|
result = entry
|
|
665
667
|
# assert if nothing changed
|
|
666
668
|
assert result == "some value"
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def test_sort_by_key_order_basic():
|
|
672
|
+
"""Test sorting by key order."""
|
|
673
|
+
|
|
674
|
+
data = {"b": 2, "a": 1, "c": 3}
|
|
675
|
+
|
|
676
|
+
assert sort_by_key_order(data, ["a", "b", "c"]) == {"a": 1, "b": 2, "c": 3}
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def test_sort_by_key_order_error_mismatched_keys():
|
|
680
|
+
"""Test sorting by key order with mismatched keys."""
|
|
681
|
+
|
|
682
|
+
data = {"b": 2, "a": 1, "c": 3}
|
|
683
|
+
|
|
684
|
+
with pytest.raises(
|
|
685
|
+
ValueError, match="'key_order' must include all keys in the dictionary!"
|
|
686
|
+
):
|
|
687
|
+
sort_by_key_order(data, ["a", "b"])
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def test_sort_alphabetically():
|
|
691
|
+
"""Test alphabetical sorting."""
|
|
692
|
+
|
|
693
|
+
data = {"A": 1, "b": 2, "a": 0, "B": 3, "c": 4, "C": 5}
|
|
694
|
+
|
|
695
|
+
assert sort_alphabetically(data) == {"a": 0, "A": 1, "b": 2, "B": 3, "c": 4, "C": 5}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# The MIT License (MIT)
|
|
2
|
+
#
|
|
3
|
+
# Copyright (c) 2025 FourCIPP Authors
|
|
4
|
+
#
|
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
# furnished to do so, subject to the following conditions:
|
|
11
|
+
#
|
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
|
13
|
+
# all copies or substantial portions of the Software.
|
|
14
|
+
#
|
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
# THE SOFTWARE.
|
|
22
|
+
"""Test yaml io utils."""
|
|
23
|
+
|
|
24
|
+
import pytest
|
|
25
|
+
|
|
26
|
+
from fourcipp.utils.dict_utils import sort_alphabetically
|
|
27
|
+
from fourcipp.utils.yaml_io import dict_to_yaml_string, dump_yaml, load_yaml
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_dump_not_sorted(tmp_path):
|
|
31
|
+
"""Test if key order is preserved."""
|
|
32
|
+
data = {"c": 1, "b": 2, "a": 3}
|
|
33
|
+
sorted_file_path = tmp_path / "sorted.yaml"
|
|
34
|
+
dump_yaml(data, path_to_yaml_file=sorted_file_path)
|
|
35
|
+
reloaded_data = load_yaml(sorted_file_path)
|
|
36
|
+
assert reloaded_data == data
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_dump_sorted_alphabetically(tmp_path):
|
|
40
|
+
"""Test if key order is sorted."""
|
|
41
|
+
data = {"c": 1, "b": 2, "a": 3}
|
|
42
|
+
sorted_file_path = tmp_path / "sorted.yaml"
|
|
43
|
+
dump_yaml(
|
|
44
|
+
data, path_to_yaml_file=sorted_file_path, sort_function=sort_alphabetically
|
|
45
|
+
)
|
|
46
|
+
reloaded_data = load_yaml(sorted_file_path)
|
|
47
|
+
assert list(reloaded_data.keys()) == sorted(data.keys())
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@pytest.mark.parametrize(
|
|
51
|
+
"use_fourcipp_yaml_style, expected",
|
|
52
|
+
[
|
|
53
|
+
(
|
|
54
|
+
False,
|
|
55
|
+
"""SECTION:
|
|
56
|
+
- vector:
|
|
57
|
+
- 1.23
|
|
58
|
+
- 2
|
|
59
|
+
- 3
|
|
60
|
+
nested_vector:
|
|
61
|
+
- - 1
|
|
62
|
+
- 2.0
|
|
63
|
+
- 3
|
|
64
|
+
- - 4.5
|
|
65
|
+
- 5
|
|
66
|
+
- 2.333
|
|
67
|
+
- list_with_bool:
|
|
68
|
+
- 1
|
|
69
|
+
- true
|
|
70
|
+
- 3
|
|
71
|
+
list_with_null:
|
|
72
|
+
- 1
|
|
73
|
+
- null
|
|
74
|
+
- 4
|
|
75
|
+
- list_with_string:
|
|
76
|
+
- 1
|
|
77
|
+
- "abc"
|
|
78
|
+
- 5
|
|
79
|
+
nested_list_with_string:
|
|
80
|
+
- - 1
|
|
81
|
+
- 2.1
|
|
82
|
+
- - 2
|
|
83
|
+
- "def"
|
|
84
|
+
""",
|
|
85
|
+
),
|
|
86
|
+
(
|
|
87
|
+
True,
|
|
88
|
+
"""SECTION:
|
|
89
|
+
- vector: [1.23,2,3]
|
|
90
|
+
nested_vector: [[1,2.0,3],[4.5,5,2.333]]
|
|
91
|
+
- list_with_bool:
|
|
92
|
+
- 1
|
|
93
|
+
- true
|
|
94
|
+
- 3
|
|
95
|
+
list_with_null:
|
|
96
|
+
- 1
|
|
97
|
+
- null
|
|
98
|
+
- 4
|
|
99
|
+
- list_with_string:
|
|
100
|
+
- 1
|
|
101
|
+
- "abc"
|
|
102
|
+
- 5
|
|
103
|
+
nested_list_with_string:
|
|
104
|
+
- [1,2.1]
|
|
105
|
+
- - 2
|
|
106
|
+
- "def"
|
|
107
|
+
""",
|
|
108
|
+
),
|
|
109
|
+
],
|
|
110
|
+
)
|
|
111
|
+
def test_yaml_style(use_fourcipp_yaml_style, expected):
|
|
112
|
+
"""Test yaml output style."""
|
|
113
|
+
data = {
|
|
114
|
+
"SECTION": [
|
|
115
|
+
{"vector": [1.23, 2, 3], "nested_vector": [[1, 2.0, 3], [4.5, 5, 2.333]]},
|
|
116
|
+
{"list_with_bool": [1, True, 3], "list_with_null": [1, None, 4]},
|
|
117
|
+
{
|
|
118
|
+
"list_with_string": [1, "abc", 5],
|
|
119
|
+
"nested_list_with_string": [[1, 2.1], [2, "def"]],
|
|
120
|
+
},
|
|
121
|
+
]
|
|
122
|
+
}
|
|
123
|
+
assert (
|
|
124
|
+
dict_to_yaml_string(data, use_fourcipp_yaml_style=use_fourcipp_yaml_style)
|
|
125
|
+
== expected
|
|
126
|
+
)
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
# The MIT License (MIT)
|
|
2
|
-
#
|
|
3
|
-
# Copyright (c) 2025 FourCIPP Authors
|
|
4
|
-
#
|
|
5
|
-
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
# of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
# in the Software without restriction, including without limitation the rights
|
|
8
|
-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
# copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
# furnished to do so, subject to the following conditions:
|
|
11
|
-
#
|
|
12
|
-
# The above copyright notice and this permission notice shall be included in
|
|
13
|
-
# all copies or substantial portions of the Software.
|
|
14
|
-
#
|
|
15
|
-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
-
# THE SOFTWARE.
|
|
22
|
-
"""Test yaml io utils."""
|
|
23
|
-
|
|
24
|
-
from fourcipp.utils.yaml_io import dump_yaml, load_yaml
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def test_dump_not_sorted(tmp_path):
|
|
28
|
-
"""Test if key order is preserved."""
|
|
29
|
-
data = {"c": 1, "b": 2, "a": 3}
|
|
30
|
-
sorted_file_path = tmp_path / "sorted.yaml"
|
|
31
|
-
dump_yaml(data, path_to_yaml_file=sorted_file_path)
|
|
32
|
-
reloaded_data = load_yaml(sorted_file_path)
|
|
33
|
-
assert reloaded_data == data
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def test_dump_sorted(tmp_path):
|
|
37
|
-
"""Test if key order is sorted."""
|
|
38
|
-
data = {"c": 1, "b": 2, "a": 3}
|
|
39
|
-
sorted_file_path = tmp_path / "sorted.yaml"
|
|
40
|
-
dump_yaml(data, path_to_yaml_file=sorted_file_path, sort_keys=True)
|
|
41
|
-
reloaded_data = load_yaml(sorted_file_path)
|
|
42
|
-
assert list(reloaded_data.keys()) == sorted(data.keys())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|