yaml-reference 2.6.2__tar.gz → 2.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/.github/copilot-instructions.md +30 -21
- yaml_reference-2.8.0/.vscode/settings.json +13 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/PKG-INFO +73 -11
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/README.md +72 -10
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/tests/unit/test_flatten.py +30 -0
- yaml_reference-2.8.0/tests/unit/test_ignore.py +184 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/tests/unit/test_merge.py +35 -0
- yaml_reference-2.8.0/tests/unit/test_multidocument.py +34 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/tests/unit/test_reference.py +125 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/yaml_reference/__init__.py +267 -28
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/yaml_reference/cli.py +7 -0
- yaml_reference-2.6.2/.vscode/settings.json +0 -8
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/.github/workflows/pytests-pr.yml +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/.github/workflows/release.yml +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/.github/workflows/spectests-pr.yml +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/.gitignore +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/.pre-commit-config.yaml +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/.python-version +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/.zed/settings.json +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/GitVersion.yml +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/LICENSE +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/Makefile +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/pyproject.toml +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/scripts/spec-test.sh +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/scripts/update-readme-badge.sh +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/tests/unit/conftest.py +0 -0
- {yaml_reference-2.6.2 → yaml_reference-2.8.0}/uv.lock +0 -0
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
## Project Overview
|
|
4
4
|
|
|
5
|
-
**yaml-reference** is a Python library that extends `ruamel.yaml` with cross-file YAML composition using custom tags (`!reference`, `!reference-all`, `!flatten`, `!merge`). It's built to be a reference implementation of the [yaml-reference-specs](https://github.com/dsillman2000/yaml-reference-specs) specification.
|
|
5
|
+
**yaml-reference** is a Python library that extends `ruamel.yaml` with cross-file YAML composition using custom tags (`!reference`, `!reference-all`, `!flatten`, `!merge`, `!ignore`). It's built to be a reference implementation of the [yaml-reference-specs](https://github.com/dsillman2000/yaml-reference-specs) specification.
|
|
6
6
|
|
|
7
7
|
## Build, Test, and Lint
|
|
8
8
|
|
|
@@ -62,19 +62,21 @@ uv build
|
|
|
62
62
|
The library is structured in two key parts:
|
|
63
63
|
|
|
64
64
|
### Core Module (`yaml_reference/__init__.py`)
|
|
65
|
-
- **Reference & ReferenceAll classes**: Represent the `!reference` and `!reference-all` YAML tags as Python objects
|
|
66
|
-
- **
|
|
67
|
-
- **
|
|
68
|
-
- **
|
|
69
|
-
- **
|
|
65
|
+
- **Reference & ReferenceAll classes**: Represent the `!reference` and `!reference-all` YAML tags as Python objects, supporting both mapping form and scalar shorthand (`!reference path/to/file.yml`, `!reference-all glob/*.yml`)
|
|
66
|
+
- **Ignore, Flatten, and Merge classes**: Represent `!ignore`, `!flatten`, and `!merge` tag logic
|
|
67
|
+
- **parse_yaml_with_references()**: Parses YAML and preserves composition tags as Python objects without resolving cross-file references
|
|
68
|
+
- **load_yaml_with_references()**: Fully resolves references, then prunes ignored content, flattens sequences, and merges mappings to produce the final Python data structure
|
|
69
|
+
- **Helper transforms**: `prune_ignores()`, `flatten_sequences()`, and `merge_mappings()` implement the post-resolution evaluation pipeline
|
|
70
|
+
- **YAML loader setup**: Registers custom constructors with `ruamel.yaml.YAML` for each supported tag
|
|
70
71
|
|
|
71
72
|
### CLI Module (`yaml_reference/cli.py`)
|
|
72
|
-
- Simple entry point that calls the core loading functions
|
|
73
|
+
- Simple entry point that calls the core loading functions for YAML containing any supported composition tags
|
|
73
74
|
- Outputs JSON to stdout (compatible with spec tests)
|
|
74
75
|
- Takes optional `--allow` flag for path restrictions
|
|
75
76
|
|
|
76
77
|
### Test Structure (`tests/unit/`)
|
|
77
78
|
- `test_reference.py`: Tests for `!reference` and `!reference-all` tag resolution
|
|
79
|
+
- `test_ignore.py`: Tests for `!ignore` parsing and pruning behavior
|
|
78
80
|
- `test_flatten.py`: Tests for `!flatten` tag behavior
|
|
79
81
|
- `test_merge.py`: Tests for `!merge` tag behavior
|
|
80
82
|
- `conftest.py`: Pytest fixtures and test utilities
|
|
@@ -83,33 +85,40 @@ The library is structured in two key parts:
|
|
|
83
85
|
|
|
84
86
|
### Security-First Path Handling
|
|
85
87
|
1. **Relative paths only**: All references must use relative paths (e.g., `path: "config/db.yaml"`). Absolute paths raise `ValueError`.
|
|
86
|
-
2. **Path restriction by default**:
|
|
88
|
+
2. **Path restriction by default**: The referencing file's parent directory is always allowed. Use `allow_paths` to explicitly allow additional directory trees.
|
|
87
89
|
3. **Security invariant**: Disallowed files are **never opened or read into memory**. Path filtering happens before file I/O.
|
|
88
|
-
4. **Silent omission (for `!reference-all`)**: When a glob pattern matches files outside allowed paths, those files are silently dropped from results
|
|
90
|
+
4. **Silent omission (for `!reference-all`)**: When a glob pattern matches files outside allowed paths, those files are silently dropped from results. Empty or fully filtered globs resolve to `[]` rather than an error.
|
|
89
91
|
|
|
90
92
|
### YAML Tag Implementation Pattern
|
|
91
93
|
Each custom tag follows this pattern:
|
|
92
94
|
1. Define a class with `yaml_tag` attribute
|
|
93
|
-
2. Implement `@classmethod from_yaml(cls, constructor, node)` to parse from YAML
|
|
95
|
+
2. Implement `@classmethod from_yaml(cls, constructor, node)` to parse from YAML, handling scalar, mapping, or sequence nodes as needed
|
|
94
96
|
3. Register constructor with the YAML loader in `__init__.py`
|
|
95
97
|
4. The class instance persists through `parse_yaml_with_references()`, allowing layer-by-layer resolution
|
|
96
98
|
|
|
99
|
+
### Reference Tag Forms
|
|
100
|
+
1. **Scalar shorthand is supported**: `!reference path/to/file.yml` and `!reference-all glob/*.yml` are valid when only `path` or `glob` is needed.
|
|
101
|
+
2. **Mapping form is still required for optional fields**: Use mappings such as `{ path: "file.yml", anchor: "section" }` when specifying `anchor`.
|
|
102
|
+
|
|
97
103
|
### Reference Resolution Order
|
|
98
|
-
1. **Circular reference detection** occurs during recursive resolution by tracking
|
|
104
|
+
1. **Circular reference detection** occurs during recursive resolution by tracking visited file paths
|
|
99
105
|
2. **Anchors** (optional parameter): If specified, extract only the anchored section from the referenced file
|
|
100
|
-
3. **Recursive expansion**: `load_yaml_with_references()` recursively
|
|
106
|
+
3. **Recursive expansion**: `load_yaml_with_references()` recursively resolves `!reference` and `!reference-all` first
|
|
107
|
+
4. **Ignore pruning**: `!ignore` content is removed after full reference resolution so ignored values from referenced files can remove their parent keys or list items
|
|
108
|
+
5. **Post-processing**: `!flatten` is evaluated after ignore pruning, and `!merge` is evaluated last
|
|
101
109
|
|
|
102
110
|
### Error Handling
|
|
103
|
-
- **ValueError** for spec violations: absolute paths, circular references, invalid anchors
|
|
111
|
+
- **ValueError** for spec violations: absolute paths, circular references, invalid anchors, malformed merge contents
|
|
104
112
|
- **FileNotFoundError** for missing referenced files
|
|
105
|
-
- **
|
|
113
|
+
- **PermissionError** for disallowed `!reference` targets
|
|
114
|
+
- **Glob behavior**: `!reference-all` returns `[]` when a glob matches no files or when all matches are filtered out by path restrictions
|
|
106
115
|
|
|
107
116
|
### Spec Compliance Testing
|
|
108
117
|
The project tests against `yaml-reference-specs`, a Go-based reference implementation. The spec tests verify:
|
|
109
|
-
- Correct expansion of all
|
|
118
|
+
- Correct expansion of all supported tags
|
|
110
119
|
- Proper error detection (bad paths, missing files, circular refs)
|
|
111
120
|
- Path restriction enforcement
|
|
112
|
-
- Edge cases like empty globs and nested composition
|
|
121
|
+
- Edge cases like empty globs, ignored content, shorthand reference syntax, and nested composition
|
|
113
122
|
|
|
114
123
|
Run with: `make spec-test` or `scripts/spec-test.sh`
|
|
115
124
|
|
|
@@ -127,15 +136,15 @@ Install hooks with: `pre-commit install`
|
|
|
127
136
|
### Adding a new tag type
|
|
128
137
|
1. Create a class in `yaml_reference/__init__.py` with `yaml_tag` attribute and `from_yaml()` classmethod
|
|
129
138
|
2. Register the constructor after the class definition
|
|
130
|
-
3. Add resolution logic (
|
|
139
|
+
3. Add resolution or post-processing logic in the appropriate stage (`_recursively_resolve_references()`, `prune_ignores()`, `flatten_sequences()`, or `merge_mappings()`)
|
|
131
140
|
4. Write tests in `tests/unit/test_*.py` following existing patterns
|
|
132
141
|
5. Update README.md with usage example
|
|
133
142
|
|
|
134
143
|
### Debugging a reference resolution issue
|
|
135
|
-
1. Use `parse_yaml_with_references()` to
|
|
136
|
-
2.
|
|
137
|
-
3. Check the
|
|
138
|
-
4. Run
|
|
144
|
+
1. Use `parse_yaml_with_references()` to inspect raw `Reference`, `ReferenceAll`, `Ignore`, `Flatten`, and `Merge` objects before evaluation
|
|
145
|
+
2. Trace `_recursively_resolve_references()` to debug cross-file expansion and circular reference handling
|
|
146
|
+
3. Check the post-processing stages in order: `prune_ignores()`, then `flatten_sequences()`, then `merge_mappings()`
|
|
147
|
+
4. Run the most specific unit test with `-v` flag to see detailed assertion output
|
|
139
148
|
|
|
140
149
|
### Updating error messages
|
|
141
150
|
Ensure error messages follow this pattern: include the problematic value, the path of the file where the error occurred, and the specific constraint violated. This helps spec tests verify proper error handling.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: yaml-reference
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.8.0
|
|
4
4
|
Summary: Extension package built on top of `ruamel.yaml` to support cross-file references in YAML files using tags `!reference` and `!reference-all`.
|
|
5
5
|
Project-URL: Repository, https://github.com/dsillman2000/yaml-reference.git
|
|
6
6
|
Author-email: David Sillman <dsillman2000@gmail.com>
|
|
@@ -26,7 +26,7 @@ Description-Content-Type: text/markdown
|
|
|
26
26
|
|
|
27
27
|
# yaml-reference
|
|
28
28
|
|
|
29
|
-
Using `ruamel.yaml`,
|
|
29
|
+
Using `ruamel.yaml`, yaml-reference supports cross-file references and YAML composition in YAML files using tags `!reference`, `!reference-all`, `!flatten`, `!merge`, and `!ignore`.
|
|
30
30
|
|
|
31
31
|
Install the package from PyPI with:
|
|
32
32
|
|
|
@@ -40,9 +40,9 @@ uv add yaml-reference
|
|
|
40
40
|
```
|
|
41
41
|
|
|
42
42
|
## Spec
|
|
43
|
-

|
|
44
44
|
|
|
45
|
-
This Python library implements the YAML specification for cross-file references and YAML composition in YAML files using tags `!reference`, `!reference-all`, `!flatten`, and `!
|
|
45
|
+
This Python library implements the YAML specification for cross-file references and YAML composition in YAML files using tags `!reference`, `!reference-all`, `!flatten`, `!merge`, and `!ignore` as defined in the [yaml-reference-specs project](https://github.com/dsillman2000/yaml-reference-specs).
|
|
46
46
|
|
|
47
47
|
## Example
|
|
48
48
|
|
|
@@ -50,15 +50,13 @@ This Python library implements the YAML specification for cross-file references
|
|
|
50
50
|
# root.yaml
|
|
51
51
|
version: "3.1"
|
|
52
52
|
services:
|
|
53
|
-
- !reference
|
|
54
|
-
path: "services/website.yaml"
|
|
53
|
+
- !reference "services/website.yaml"
|
|
55
54
|
|
|
56
55
|
- !reference
|
|
57
56
|
path: "services/database.yaml"
|
|
58
57
|
|
|
59
58
|
networkConfigs:
|
|
60
|
-
!reference-all
|
|
61
|
-
glob: "networks/*.yaml"
|
|
59
|
+
!reference-all "networks/*.yaml"
|
|
62
60
|
|
|
63
61
|
tags: !flatten
|
|
64
62
|
- !reference { path: "common/tags.yaml" }
|
|
@@ -69,6 +67,14 @@ config: !merge
|
|
|
69
67
|
- !reference { path: "config/defaults.yaml" }
|
|
70
68
|
- !reference { path: "config/overrides.yaml" }
|
|
71
69
|
|
|
70
|
+
.anchors: !ignore
|
|
71
|
+
commonTags: &commonTags
|
|
72
|
+
- common:http
|
|
73
|
+
- common:security
|
|
74
|
+
dbDefaults: &dbDefaults
|
|
75
|
+
host: localhost
|
|
76
|
+
port: 5432
|
|
77
|
+
|
|
72
78
|
```
|
|
73
79
|
|
|
74
80
|
Supposing there are `services/website.yaml` and `services/database.yaml` files in the same directory as `root.yaml`, and a `networks` directory with YAML files, the above will be expanded to account for the referenced files with the following Python code:
|
|
@@ -99,6 +105,57 @@ print(data["networkConfigs"])
|
|
|
99
105
|
data = parse_yaml_with_references("root.yaml", allow_paths=["/allowed/path"])
|
|
100
106
|
```
|
|
101
107
|
|
|
108
|
+
For `!reference` and `!reference-all`, both mapping and scalar shorthand forms are supported. These are equivalent:
|
|
109
|
+
|
|
110
|
+
```yaml
|
|
111
|
+
# Scalar shorthand
|
|
112
|
+
service: !reference "services/api.yaml"
|
|
113
|
+
|
|
114
|
+
# Mapping form
|
|
115
|
+
service: !reference { path: "services/api.yaml" }
|
|
116
|
+
|
|
117
|
+
# Scalar shorthand
|
|
118
|
+
networks: !reference-all "networks/*.yaml"
|
|
119
|
+
|
|
120
|
+
# Mapping form
|
|
121
|
+
networks: !reference-all { glob: "networks/*.yaml" }
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Use the mapping form when you need optional arguments such as `anchor`; use the scalar shorthand when you only need `path` or `glob`.
|
|
125
|
+
|
|
126
|
+
### Multi-Document YAML
|
|
127
|
+
|
|
128
|
+
yaml-reference distinguishes between a single YAML document whose root value is a sequence and a YAML file that contains multiple documents separated by `---`.
|
|
129
|
+
|
|
130
|
+
- `!reference` requires the target file to contain exactly one YAML document. If the referenced file contains multiple documents, loading fails with a `ValueError`.
|
|
131
|
+
- `!reference-all` expands matched files document-by-document. A single-document file contributes one list element, while a multi-document file contributes one element per document in document order.
|
|
132
|
+
- When `anchor` is used with `!reference-all`, the anchored value is extracted from every document in each matched file, preserving file order and then document order.
|
|
133
|
+
- If the root input file contains multiple documents, `load_yaml_with_references()` returns a Python list with one resolved output element per document. Root documents tagged with `!ignore` are omitted entirely.
|
|
134
|
+
|
|
135
|
+
### The `!ignore` Tag
|
|
136
|
+
|
|
137
|
+
The `!ignore` tag marks YAML content that should be parsed but omitted from the final resolved output. The most common use case is a hidden section of reusable anchors that should remain available for aliases elsewhere in the document without being emitted in the resolved result.
|
|
138
|
+
|
|
139
|
+
```yaml
|
|
140
|
+
.anchors: !ignore
|
|
141
|
+
commonLabels: &commonLabels
|
|
142
|
+
app: payments
|
|
143
|
+
team: platform
|
|
144
|
+
defaultResources: &defaultResources
|
|
145
|
+
requests:
|
|
146
|
+
cpu: "100m"
|
|
147
|
+
memory: "128Mi"
|
|
148
|
+
|
|
149
|
+
service:
|
|
150
|
+
metadata:
|
|
151
|
+
labels: *commonLabels
|
|
152
|
+
resources: *defaultResources
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
When loaded with `load_yaml_with_references`, the `.anchors` key is removed entirely, but the anchors it defined remain usable by aliases elsewhere in the document.
|
|
156
|
+
|
|
157
|
+
Ignored items are also pruned before `!flatten` and `!merge` are evaluated, so an ignored sequence entry inside either tag is simply omitted from the flattened or merged result.
|
|
158
|
+
|
|
102
159
|
### The `!merge` Tag
|
|
103
160
|
|
|
104
161
|
The `!merge` tag combines multiple YAML mappings (dictionaries) into a single mapping. This is useful for composing configuration from multiple sources or applying overrides. When you use `!merge`, you provide a sequence of mappings that will be merged together, with later mappings overriding keys from earlier ones.
|
|
@@ -177,20 +234,25 @@ Note that the `app_name` and `cache_settings` fields from `config.yaml` are not
|
|
|
177
234
|
|
|
178
235
|
### VSCode squigglies
|
|
179
236
|
|
|
180
|
-
To get rid of red squigglies in VSCode when using the `!reference`, `!reference-all`, `!flatten`, and `!
|
|
237
|
+
To get rid of red squigglies in VSCode when using the `!reference`, `!reference-all`, `!flatten`, `!merge`, and `!ignore` tags, you can add the following to your `settings.json` file:
|
|
181
238
|
|
|
182
239
|
```json
|
|
183
240
|
"yaml.customTags": [
|
|
241
|
+
"!reference scalar",
|
|
184
242
|
"!reference mapping",
|
|
243
|
+
"!reference-all scalar",
|
|
185
244
|
"!reference-all mapping",
|
|
186
245
|
"!flatten sequence",
|
|
187
|
-
"!merge sequence"
|
|
246
|
+
"!merge sequence",
|
|
247
|
+
"!ignore scalar",
|
|
248
|
+
"!ignore sequence",
|
|
249
|
+
"!ignore mapping"
|
|
188
250
|
]
|
|
189
251
|
```
|
|
190
252
|
|
|
191
253
|
## CLI interface
|
|
192
254
|
|
|
193
|
-
There is a CLI interface for this package which can be used to read a YAML file which contains `!reference
|
|
255
|
+
There is a CLI interface for this package which can be used to read a YAML file which contains composition tags such as `!reference`, `!reference-all`, `!flatten`, `!merge`, and `!ignore`, and dump its contents as pretty-printed JSON with references expanded and ignored content removed. This is useful for generating a single file for deployment or other purposes. Note that the keys of mappings will be sorted alphabetically. This CLI interface is used to test the contract of this package against the `yaml-reference-specs` project.
|
|
194
256
|
|
|
195
257
|
```bash
|
|
196
258
|
$ yaml-reference-cli -h
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# yaml-reference
|
|
2
2
|
|
|
3
|
-
Using `ruamel.yaml`,
|
|
3
|
+
Using `ruamel.yaml`, yaml-reference supports cross-file references and YAML composition in YAML files using tags `!reference`, `!reference-all`, `!flatten`, `!merge`, and `!ignore`.
|
|
4
4
|
|
|
5
5
|
Install the package from PyPI with:
|
|
6
6
|
|
|
@@ -14,9 +14,9 @@ uv add yaml-reference
|
|
|
14
14
|
```
|
|
15
15
|
|
|
16
16
|
## Spec
|
|
17
|
-

|
|
18
18
|
|
|
19
|
-
This Python library implements the YAML specification for cross-file references and YAML composition in YAML files using tags `!reference`, `!reference-all`, `!flatten`, and `!
|
|
19
|
+
This Python library implements the YAML specification for cross-file references and YAML composition in YAML files using tags `!reference`, `!reference-all`, `!flatten`, `!merge`, and `!ignore` as defined in the [yaml-reference-specs project](https://github.com/dsillman2000/yaml-reference-specs).
|
|
20
20
|
|
|
21
21
|
## Example
|
|
22
22
|
|
|
@@ -24,15 +24,13 @@ This Python library implements the YAML specification for cross-file references
|
|
|
24
24
|
# root.yaml
|
|
25
25
|
version: "3.1"
|
|
26
26
|
services:
|
|
27
|
-
- !reference
|
|
28
|
-
path: "services/website.yaml"
|
|
27
|
+
- !reference "services/website.yaml"
|
|
29
28
|
|
|
30
29
|
- !reference
|
|
31
30
|
path: "services/database.yaml"
|
|
32
31
|
|
|
33
32
|
networkConfigs:
|
|
34
|
-
!reference-all
|
|
35
|
-
glob: "networks/*.yaml"
|
|
33
|
+
!reference-all "networks/*.yaml"
|
|
36
34
|
|
|
37
35
|
tags: !flatten
|
|
38
36
|
- !reference { path: "common/tags.yaml" }
|
|
@@ -43,6 +41,14 @@ config: !merge
|
|
|
43
41
|
- !reference { path: "config/defaults.yaml" }
|
|
44
42
|
- !reference { path: "config/overrides.yaml" }
|
|
45
43
|
|
|
44
|
+
.anchors: !ignore
|
|
45
|
+
commonTags: &commonTags
|
|
46
|
+
- common:http
|
|
47
|
+
- common:security
|
|
48
|
+
dbDefaults: &dbDefaults
|
|
49
|
+
host: localhost
|
|
50
|
+
port: 5432
|
|
51
|
+
|
|
46
52
|
```
|
|
47
53
|
|
|
48
54
|
Supposing there are `services/website.yaml` and `services/database.yaml` files in the same directory as `root.yaml`, and a `networks` directory with YAML files, the above will be expanded to account for the referenced files with the following Python code:
|
|
@@ -73,6 +79,57 @@ print(data["networkConfigs"])
|
|
|
73
79
|
data = parse_yaml_with_references("root.yaml", allow_paths=["/allowed/path"])
|
|
74
80
|
```
|
|
75
81
|
|
|
82
|
+
For `!reference` and `!reference-all`, both mapping and scalar shorthand forms are supported. These are equivalent:
|
|
83
|
+
|
|
84
|
+
```yaml
|
|
85
|
+
# Scalar shorthand
|
|
86
|
+
service: !reference "services/api.yaml"
|
|
87
|
+
|
|
88
|
+
# Mapping form
|
|
89
|
+
service: !reference { path: "services/api.yaml" }
|
|
90
|
+
|
|
91
|
+
# Scalar shorthand
|
|
92
|
+
networks: !reference-all "networks/*.yaml"
|
|
93
|
+
|
|
94
|
+
# Mapping form
|
|
95
|
+
networks: !reference-all { glob: "networks/*.yaml" }
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Use the mapping form when you need optional arguments such as `anchor`; use the scalar shorthand when you only need `path` or `glob`.
|
|
99
|
+
|
|
100
|
+
### Multi-Document YAML
|
|
101
|
+
|
|
102
|
+
yaml-reference distinguishes between a single YAML document whose root value is a sequence and a YAML file that contains multiple documents separated by `---`.
|
|
103
|
+
|
|
104
|
+
- `!reference` requires the target file to contain exactly one YAML document. If the referenced file contains multiple documents, loading fails with a `ValueError`.
|
|
105
|
+
- `!reference-all` expands matched files document-by-document. A single-document file contributes one list element, while a multi-document file contributes one element per document in document order.
|
|
106
|
+
- When `anchor` is used with `!reference-all`, the anchored value is extracted from every document in each matched file, preserving file order and then document order.
|
|
107
|
+
- If the root input file contains multiple documents, `load_yaml_with_references()` returns a Python list with one resolved output element per document. Root documents tagged with `!ignore` are omitted entirely.
|
|
108
|
+
|
|
109
|
+
### The `!ignore` Tag
|
|
110
|
+
|
|
111
|
+
The `!ignore` tag marks YAML content that should be parsed but omitted from the final resolved output. The most common use case is a hidden section of reusable anchors that should remain available for aliases elsewhere in the document without being emitted in the resolved result.
|
|
112
|
+
|
|
113
|
+
```yaml
|
|
114
|
+
.anchors: !ignore
|
|
115
|
+
commonLabels: &commonLabels
|
|
116
|
+
app: payments
|
|
117
|
+
team: platform
|
|
118
|
+
defaultResources: &defaultResources
|
|
119
|
+
requests:
|
|
120
|
+
cpu: "100m"
|
|
121
|
+
memory: "128Mi"
|
|
122
|
+
|
|
123
|
+
service:
|
|
124
|
+
metadata:
|
|
125
|
+
labels: *commonLabels
|
|
126
|
+
resources: *defaultResources
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
When loaded with `load_yaml_with_references`, the `.anchors` key is removed entirely, but the anchors it defined remain usable by aliases elsewhere in the document.
|
|
130
|
+
|
|
131
|
+
Ignored items are also pruned before `!flatten` and `!merge` are evaluated, so an ignored sequence entry inside either tag is simply omitted from the flattened or merged result.
|
|
132
|
+
|
|
76
133
|
### The `!merge` Tag
|
|
77
134
|
|
|
78
135
|
The `!merge` tag combines multiple YAML mappings (dictionaries) into a single mapping. This is useful for composing configuration from multiple sources or applying overrides. When you use `!merge`, you provide a sequence of mappings that will be merged together, with later mappings overriding keys from earlier ones.
|
|
@@ -151,20 +208,25 @@ Note that the `app_name` and `cache_settings` fields from `config.yaml` are not
|
|
|
151
208
|
|
|
152
209
|
### VSCode squigglies
|
|
153
210
|
|
|
154
|
-
To get rid of red squigglies in VSCode when using the `!reference`, `!reference-all`, `!flatten`, and `!
|
|
211
|
+
To get rid of red squigglies in VSCode when using the `!reference`, `!reference-all`, `!flatten`, `!merge`, and `!ignore` tags, you can add the following to your `settings.json` file:
|
|
155
212
|
|
|
156
213
|
```json
|
|
157
214
|
"yaml.customTags": [
|
|
215
|
+
"!reference scalar",
|
|
158
216
|
"!reference mapping",
|
|
217
|
+
"!reference-all scalar",
|
|
159
218
|
"!reference-all mapping",
|
|
160
219
|
"!flatten sequence",
|
|
161
|
-
"!merge sequence"
|
|
220
|
+
"!merge sequence",
|
|
221
|
+
"!ignore scalar",
|
|
222
|
+
"!ignore sequence",
|
|
223
|
+
"!ignore mapping"
|
|
162
224
|
]
|
|
163
225
|
```
|
|
164
226
|
|
|
165
227
|
## CLI interface
|
|
166
228
|
|
|
167
|
-
There is a CLI interface for this package which can be used to read a YAML file which contains `!reference
|
|
229
|
+
There is a CLI interface for this package which can be used to read a YAML file which contains composition tags such as `!reference`, `!reference-all`, `!flatten`, `!merge`, and `!ignore`, and dump its contents as pretty-printed JSON with references expanded and ignored content removed. This is useful for generating a single file for deployment or other purposes. Note that the keys of mappings will be sorted alphabetically. This CLI interface is used to test the contract of this package against the `yaml-reference-specs` project.
|
|
168
230
|
|
|
169
231
|
```bash
|
|
170
232
|
$ yaml-reference-cli -h
|
|
@@ -130,6 +130,21 @@ data: !flatten
|
|
|
130
130
|
assert data["data"] == [1, 2, 3, 4, 5, 6]
|
|
131
131
|
|
|
132
132
|
|
|
133
|
+
def test_flatten_combined_with_multi_document_reference_all(stage_files):
|
|
134
|
+
files = {
|
|
135
|
+
"main.yml": """
|
|
136
|
+
data: !flatten
|
|
137
|
+
- !reference-all { glob: ./entries.yml }
|
|
138
|
+
""",
|
|
139
|
+
"entries.yml": "---\n- [1, 2]\n---\n- [3, 4]\n",
|
|
140
|
+
}
|
|
141
|
+
stg = stage_files(files)
|
|
142
|
+
|
|
143
|
+
data = load_yaml_with_references(stg / "main.yml")
|
|
144
|
+
|
|
145
|
+
assert data["data"] == [1, 2, 3, 4]
|
|
146
|
+
|
|
147
|
+
|
|
133
148
|
def test_parse_flatten_tag(stage_files):
|
|
134
149
|
"""Test that !flatten tags are parsed correctly without resolution."""
|
|
135
150
|
files = {
|
|
@@ -197,6 +212,21 @@ def test_flatten_with_scalars(stage_files):
|
|
|
197
212
|
assert data["data"] == [1, 2, 3, 4, 5, 6, 7]
|
|
198
213
|
|
|
199
214
|
|
|
215
|
+
def test_flatten_ignores_ignored_sequence_items(stage_files):
|
|
216
|
+
"""Test that !ignore items inside a !flatten sequence are omitted from the flattened result."""
|
|
217
|
+
files = {
|
|
218
|
+
"test.yml": """
|
|
219
|
+
data: !flatten
|
|
220
|
+
- [1, 2]
|
|
221
|
+
- !ignore [99, 100]
|
|
222
|
+
- [3, 4]
|
|
223
|
+
""",
|
|
224
|
+
}
|
|
225
|
+
stg = stage_files(files)
|
|
226
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
227
|
+
assert data["data"] == [1, 2, 3, 4]
|
|
228
|
+
|
|
229
|
+
|
|
200
230
|
def test_flatten_mixed_objects_references(stage_files):
|
|
201
231
|
"""Test flattening a sequence of objects, references, and reference-all tags."""
|
|
202
232
|
files = {
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
from yaml_reference import (
|
|
2
|
+
Ignore,
|
|
3
|
+
load_yaml_with_references,
|
|
4
|
+
parse_yaml_with_references,
|
|
5
|
+
prune_ignores,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_ignore_parse_produces_ignore_object(stage_files):
|
|
10
|
+
"""Test that !ignore tags are parsed into Ignore objects by parse_yaml_with_references."""
|
|
11
|
+
files = {
|
|
12
|
+
"test.yml": "key: !ignore some_value",
|
|
13
|
+
}
|
|
14
|
+
stg = stage_files(files)
|
|
15
|
+
data = parse_yaml_with_references(stg / "test.yml")
|
|
16
|
+
assert isinstance(data["key"], Ignore)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_ignore_dict_value_removed(stage_files):
|
|
20
|
+
"""Test that a dict value tagged with !ignore is removed from the output."""
|
|
21
|
+
files = {
|
|
22
|
+
"test.yml": """\
|
|
23
|
+
keep: hello
|
|
24
|
+
drop: !ignore world
|
|
25
|
+
""",
|
|
26
|
+
}
|
|
27
|
+
stg = stage_files(files)
|
|
28
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
29
|
+
assert "keep" in data
|
|
30
|
+
assert data["keep"] == "hello"
|
|
31
|
+
assert "drop" not in data
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_ignore_list_item_removed(stage_files):
|
|
35
|
+
"""Test that a list item tagged with !ignore is removed from the output."""
|
|
36
|
+
files = {
|
|
37
|
+
"test.yml": """\
|
|
38
|
+
items:
|
|
39
|
+
- one
|
|
40
|
+
- !ignore two
|
|
41
|
+
- three
|
|
42
|
+
""",
|
|
43
|
+
}
|
|
44
|
+
stg = stage_files(files)
|
|
45
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
46
|
+
assert data["items"] == ["one", "three"]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_ignore_standalone_value_becomes_none(stage_files):
|
|
50
|
+
"""Test that a standalone (non-list, non-dict) !ignore value is replaced with None."""
|
|
51
|
+
files = {
|
|
52
|
+
"test.yml": "!ignore standalone",
|
|
53
|
+
}
|
|
54
|
+
stg = stage_files(files)
|
|
55
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
56
|
+
assert data is None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_ignore_multiple_items_in_list(stage_files):
|
|
60
|
+
"""Test that multiple !ignore items in a list are all removed."""
|
|
61
|
+
files = {
|
|
62
|
+
"test.yml": """\
|
|
63
|
+
items:
|
|
64
|
+
- !ignore a
|
|
65
|
+
- b
|
|
66
|
+
- !ignore c
|
|
67
|
+
- d
|
|
68
|
+
""",
|
|
69
|
+
}
|
|
70
|
+
stg = stage_files(files)
|
|
71
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
72
|
+
assert data["items"] == ["b", "d"]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_ignore_multiple_keys_in_dict(stage_files):
|
|
76
|
+
"""Test that multiple !ignore values in a dict are all removed."""
|
|
77
|
+
files = {
|
|
78
|
+
"test.yml": """\
|
|
79
|
+
a: !ignore 1
|
|
80
|
+
b: 2
|
|
81
|
+
c: !ignore 3
|
|
82
|
+
d: 4
|
|
83
|
+
""",
|
|
84
|
+
}
|
|
85
|
+
stg = stage_files(files)
|
|
86
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
87
|
+
assert data == {"b": 2, "d": 4}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_ignore_mapping_value(stage_files):
|
|
91
|
+
"""Test that a mapping tagged with !ignore is removed."""
|
|
92
|
+
files = {
|
|
93
|
+
"test.yml": """\
|
|
94
|
+
keep:
|
|
95
|
+
x: 1
|
|
96
|
+
drop: !ignore
|
|
97
|
+
y: 2
|
|
98
|
+
z: 3
|
|
99
|
+
""",
|
|
100
|
+
}
|
|
101
|
+
stg = stage_files(files)
|
|
102
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
103
|
+
assert "keep" in data
|
|
104
|
+
assert "drop" not in data
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_ignore_sequence_value(stage_files):
|
|
108
|
+
"""Test that a sequence tagged with !ignore is removed when used as a dict value."""
|
|
109
|
+
files = {
|
|
110
|
+
"test.yml": """\
|
|
111
|
+
keep: [1, 2]
|
|
112
|
+
drop: !ignore [3, 4]
|
|
113
|
+
""",
|
|
114
|
+
}
|
|
115
|
+
stg = stage_files(files)
|
|
116
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
117
|
+
assert data == {"keep": [1, 2]}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_ignore_does_not_affect_non_tagged_content(stage_files):
|
|
121
|
+
"""Test that !ignore only affects tagged content and leaves everything else intact."""
|
|
122
|
+
files = {
|
|
123
|
+
"test.yml": """\
|
|
124
|
+
name: yaml-reference
|
|
125
|
+
version: 1.0
|
|
126
|
+
description: !ignore internal note
|
|
127
|
+
tags:
|
|
128
|
+
- alpha
|
|
129
|
+
- !ignore beta
|
|
130
|
+
- gamma
|
|
131
|
+
""",
|
|
132
|
+
}
|
|
133
|
+
stg = stage_files(files)
|
|
134
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
135
|
+
assert data["name"] == "yaml-reference"
|
|
136
|
+
assert data["version"] == 1.0
|
|
137
|
+
assert "description" not in data
|
|
138
|
+
assert data["tags"] == ["alpha", "gamma"]
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def test_ignore_in_referenced_file(stage_files):
|
|
142
|
+
"""Test that !ignore tags in a referenced file are pruned during resolution."""
|
|
143
|
+
files = {
|
|
144
|
+
"root.yml": "contents: !reference { path: ./inner.yml }",
|
|
145
|
+
"inner.yml": """\
|
|
146
|
+
public: visible
|
|
147
|
+
private: !ignore hidden
|
|
148
|
+
""",
|
|
149
|
+
}
|
|
150
|
+
stg = stage_files(files)
|
|
151
|
+
data = load_yaml_with_references(stg / "root.yml")
|
|
152
|
+
assert data["contents"] == {"public": "visible"}
|
|
153
|
+
assert "private" not in data["contents"]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_prune_ignores_standalone(stage_files):
|
|
157
|
+
"""Test prune_ignores() directly on a structure containing Ignore objects."""
|
|
158
|
+
data = {
|
|
159
|
+
"a": Ignore("should be removed"),
|
|
160
|
+
"b": 42,
|
|
161
|
+
"c": [1, Ignore("also removed"), 3],
|
|
162
|
+
}
|
|
163
|
+
result = prune_ignores(data)
|
|
164
|
+
assert "a" not in result
|
|
165
|
+
assert result["b"] == 42
|
|
166
|
+
assert result["c"] == [1, 3]
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def test_ignore_preserves_none_values(stage_files):
|
|
170
|
+
"""Test that existing null/None values in YAML are preserved (not confused with ignored values)."""
|
|
171
|
+
files = {
|
|
172
|
+
"test.yml": """\
|
|
173
|
+
present: ~
|
|
174
|
+
also_present: null
|
|
175
|
+
dropped: !ignore something
|
|
176
|
+
""",
|
|
177
|
+
}
|
|
178
|
+
stg = stage_files(files)
|
|
179
|
+
data = load_yaml_with_references(stg / "test.yml")
|
|
180
|
+
assert "present" in data
|
|
181
|
+
assert data["present"] is None
|
|
182
|
+
assert "also_present" in data
|
|
183
|
+
assert data["also_present"] is None
|
|
184
|
+
assert "dropped" not in data
|