scalio-catalog 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scalio_catalog-0.1.0/PKG-INFO +82 -0
- scalio_catalog-0.1.0/README.md +68 -0
- scalio_catalog-0.1.0/pyproject.toml +34 -0
- scalio_catalog-0.1.0/setup.cfg +4 -0
- scalio_catalog-0.1.0/src/scalio_catalog/__init__.py +19 -0
- scalio_catalog-0.1.0/src/scalio_catalog/catalog.py +296 -0
- scalio_catalog-0.1.0/src/scalio_catalog/cli.py +79 -0
- scalio_catalog-0.1.0/src/scalio_catalog/compare.py +102 -0
- scalio_catalog-0.1.0/src/scalio_catalog/effective.py +218 -0
- scalio_catalog-0.1.0/src/scalio_catalog/pack.py +173 -0
- scalio_catalog-0.1.0/src/scalio_catalog/schema/catalog-v2.schema.json +191 -0
- scalio_catalog-0.1.0/src/scalio_catalog/upgrade.py +127 -0
- scalio_catalog-0.1.0/src/scalio_catalog.egg-info/PKG-INFO +82 -0
- scalio_catalog-0.1.0/src/scalio_catalog.egg-info/SOURCES.txt +22 -0
- scalio_catalog-0.1.0/src/scalio_catalog.egg-info/dependency_links.txt +1 -0
- scalio_catalog-0.1.0/src/scalio_catalog.egg-info/entry_points.txt +2 -0
- scalio_catalog-0.1.0/src/scalio_catalog.egg-info/requires.txt +7 -0
- scalio_catalog-0.1.0/src/scalio_catalog.egg-info/top_level.txt +1 -0
- scalio_catalog-0.1.0/tests/test_catalog.py +112 -0
- scalio_catalog-0.1.0/tests/test_cli.py +44 -0
- scalio_catalog-0.1.0/tests/test_compare.py +62 -0
- scalio_catalog-0.1.0/tests/test_effective.py +84 -0
- scalio_catalog-0.1.0/tests/test_pack.py +91 -0
- scalio_catalog-0.1.0/tests/test_upgrade.py +56 -0
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scalio-catalog
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scalio Blueprint governance catalog (scalio.catalog/v2): check, effective desired state, drift, signed packs
|
|
5
|
+
License: Proprietary
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: cryptography>=42
|
|
9
|
+
Requires-Dist: jsonschema>=4.20
|
|
10
|
+
Requires-Dist: pyyaml>=6
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
13
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
14
|
+
|
|
15
|
+
# scalio-catalog
|
|
16
|
+
|
|
17
|
+
Shared code for the Scalio Blueprint governance catalog: one desired state per tenant, used by Foundry (roll
|
|
18
|
+
out), Citadel (keep, drift) and Rosetta (document). Concept and decisions BP1-BP12:
|
|
19
|
+
https://claude.ai/artifact/8PYfvHPm17KsooerLU95Ar (scalio-docs `concepts/governance-catalog.md`, ADR 0027).
|
|
20
|
+
Format contract: scalio-docs `interfaces/catalog-format.md`.
|
|
21
|
+
|
|
22
|
+
Nothing in this package is secret. The catalog content lives in the private repo `scalio_blueprint`; the signing
|
|
23
|
+
seed stays in 1Password.
|
|
24
|
+
|
|
25
|
+
## Documents
|
|
26
|
+
|
|
27
|
+
| Kind | What | Owner |
|
|
28
|
+
| --- | --- | --- |
|
|
29
|
+
| `ResourceType` | Something in a tenant (name as in Microsoft365DSC/UTCM, e.g. `AADConditionalAccessPolicy`): properties, read/write endpoint, permission | Scalio |
|
|
30
|
+
| `Control` | One security statement: match, desired values, parameters, severity, texts DE/EN, frameworks | Scalio, MSP, customer |
|
|
31
|
+
| `Baseline` | Versioned bundle of controls with default values | Scalio, MSP |
|
|
32
|
+
| `Assignment` | Tenant -> baseline@version, mode, channel, values, exceptions, own controls | customer |
|
|
33
|
+
| `Pack` | Header of a signed package | Scalio, MSP |
|
|
34
|
+
|
|
35
|
+
References: `id` (highest), `id@1.2.3` (exact), `id@^1.2` (same major, at least 1.2). Assignments name an exact
|
|
36
|
+
version.
|
|
37
|
+
|
|
38
|
+
## Use in an app
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from scalio_catalog import Catalog, compare, effective, plan_upgrade, summary, verify
|
|
42
|
+
|
|
43
|
+
catalog = verify(package_bytes, TRUSTED_CATALOG_KEYS) # signed pack from Compass or the image
|
|
44
|
+
eff = effective(catalog, assignment) # baseline + customer layer, master untouched
|
|
45
|
+
results = compare(eff, {c.id: read_resource(c) for c in eff.controls}) # app reads the tenant
|
|
46
|
+
print(summary(results)) # {"counts": ..., "score": 92}
|
|
47
|
+
plan = plan_upgrade(catalog, assignment, "scalio.kmu-standard@1.5.0") # three-way, conflicts listed
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
`compare` statuses: `ok`, `drift`, `missing`, `excepted` (all deviations covered by an active exception),
|
|
51
|
+
`not_read`, `removed` (customer removed it), `incomplete` (a value is missing).
|
|
52
|
+
|
|
53
|
+
Values (later wins): control default, baseline `parameters`, value on the baseline entry, assignment `values`,
|
|
54
|
+
assignment `controlValues.<control>`. Exceptions need a reason, end date and approver; expired ones count as
|
|
55
|
+
deviations again and are listed in `problems`; at most 366 days when `approvedAt` is given.
|
|
56
|
+
|
|
57
|
+
## Command line
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
scalio-catalog validate <folders...> check structure, references, texts
|
|
61
|
+
scalio-catalog effective <assignment> <folders...> [--today 2026-11-01]
|
|
62
|
+
scalio-catalog pack <folder> -o dist sign (CATALOG_SIGNING_KEY or --key-file), --base for other packs
|
|
63
|
+
scalio-catalog verify <file> --key <public> check a package
|
|
64
|
+
scalio-catalog keygen new key: seed to 1Password, public key into the apps
|
|
65
|
+
scalio-catalog schema > catalog-v2.schema.json
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Editing in VS Code
|
|
69
|
+
|
|
70
|
+
Install the YAML extension (Red Hat) and add to the workspace settings of `scalio_blueprint`:
|
|
71
|
+
|
|
72
|
+
```json
|
|
73
|
+
{ "yaml.schemas": { "./schema/catalog-v2.schema.json": ["controls/**/*.yaml", "baselines/**/*.yaml",
|
|
74
|
+
"resource-types/**/*.yaml", "pack.yaml"] } }
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
(`scalio-catalog schema > schema/catalog-v2.schema.json` writes the file.)
|
|
78
|
+
|
|
79
|
+
## Release
|
|
80
|
+
|
|
81
|
+
Bump `pyproject.toml` and `__version__`, CHANGELOG line, merge, then "Freigeben" in the Scalio Dev cockpit (or
|
|
82
|
+
Actions -> Publish with the version). Publishes to PyPI via trusted publishing (no token).
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# scalio-catalog
|
|
2
|
+
|
|
3
|
+
Shared code for the Scalio Blueprint governance catalog: one desired state per tenant, used by Foundry (roll
|
|
4
|
+
out), Citadel (keep, drift) and Rosetta (document). Concept and decisions BP1-BP12:
|
|
5
|
+
https://claude.ai/artifact/8PYfvHPm17KsooerLU95Ar (scalio-docs `concepts/governance-catalog.md`, ADR 0027).
|
|
6
|
+
Format contract: scalio-docs `interfaces/catalog-format.md`.
|
|
7
|
+
|
|
8
|
+
Nothing in this package is secret. The catalog content lives in the private repo `scalio_blueprint`; the signing
|
|
9
|
+
seed stays in 1Password.
|
|
10
|
+
|
|
11
|
+
## Documents
|
|
12
|
+
|
|
13
|
+
| Kind | What | Owner |
|
|
14
|
+
| --- | --- | --- |
|
|
15
|
+
| `ResourceType` | Something in a tenant (name as in Microsoft365DSC/UTCM, e.g. `AADConditionalAccessPolicy`): properties, read/write endpoint, permission | Scalio |
|
|
16
|
+
| `Control` | One security statement: match, desired values, parameters, severity, texts DE/EN, frameworks | Scalio, MSP, customer |
|
|
17
|
+
| `Baseline` | Versioned bundle of controls with default values | Scalio, MSP |
|
|
18
|
+
| `Assignment` | Tenant -> baseline@version, mode, channel, values, exceptions, own controls | customer |
|
|
19
|
+
| `Pack` | Header of a signed package | Scalio, MSP |
|
|
20
|
+
|
|
21
|
+
References: `id` (highest), `id@1.2.3` (exact), `id@^1.2` (same major, at least 1.2). Assignments name an exact
|
|
22
|
+
version.
|
|
23
|
+
|
|
24
|
+
## Use in an app
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
from scalio_catalog import Catalog, compare, effective, plan_upgrade, summary, verify
|
|
28
|
+
|
|
29
|
+
catalog = verify(package_bytes, TRUSTED_CATALOG_KEYS) # signed pack from Compass or the image
|
|
30
|
+
eff = effective(catalog, assignment) # baseline + customer layer, master untouched
|
|
31
|
+
results = compare(eff, {c.id: read_resource(c) for c in eff.controls}) # app reads the tenant
|
|
32
|
+
print(summary(results)) # {"counts": ..., "score": 92}
|
|
33
|
+
plan = plan_upgrade(catalog, assignment, "scalio.kmu-standard@1.5.0") # three-way, conflicts listed
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
`compare` statuses: `ok`, `drift`, `missing`, `excepted` (all deviations covered by an active exception),
|
|
37
|
+
`not_read`, `removed` (customer removed it), `incomplete` (a value is missing).
|
|
38
|
+
|
|
39
|
+
Values (later wins): control default, baseline `parameters`, value on the baseline entry, assignment `values`,
|
|
40
|
+
assignment `controlValues.<control>`. Exceptions need a reason, end date and approver; expired ones count as
|
|
41
|
+
deviations again and are listed in `problems`; at most 366 days when `approvedAt` is given.
|
|
42
|
+
|
|
43
|
+
## Command line
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
scalio-catalog validate <folders...> check structure, references, texts
|
|
47
|
+
scalio-catalog effective <assignment> <folders...> [--today 2026-11-01]
|
|
48
|
+
scalio-catalog pack <folder> -o dist sign (CATALOG_SIGNING_KEY or --key-file), --base for other packs
|
|
49
|
+
scalio-catalog verify <file> --key <public> check a package
|
|
50
|
+
scalio-catalog keygen new key: seed to 1Password, public key into the apps
|
|
51
|
+
scalio-catalog schema > catalog-v2.schema.json
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Editing in VS Code
|
|
55
|
+
|
|
56
|
+
Install the YAML extension (Red Hat) and add to the workspace settings of `scalio_blueprint`:
|
|
57
|
+
|
|
58
|
+
```json
|
|
59
|
+
{ "yaml.schemas": { "./schema/catalog-v2.schema.json": ["controls/**/*.yaml", "baselines/**/*.yaml",
|
|
60
|
+
"resource-types/**/*.yaml", "pack.yaml"] } }
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
(`scalio-catalog schema > schema/catalog-v2.schema.json` writes the file.)
|
|
64
|
+
|
|
65
|
+
## Release
|
|
66
|
+
|
|
67
|
+
Bump `pyproject.toml` and `__version__`, CHANGELOG line, merge, then "Freigeben" in the Scalio Dev cockpit (or
|
|
68
|
+
Actions -> Publish with the version). Publishes to PyPI via trusted publishing (no token).
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scalio-catalog"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Scalio Blueprint governance catalog (scalio.catalog/v2): check, effective desired state, drift, signed packs"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { text = "Proprietary" }
|
|
12
|
+
dependencies = ["cryptography>=42", "jsonschema>=4.20", "pyyaml>=6"]
|
|
13
|
+
|
|
14
|
+
[project.optional-dependencies]
|
|
15
|
+
dev = ["pytest>=8", "ruff>=0.6"]
|
|
16
|
+
|
|
17
|
+
[project.scripts]
|
|
18
|
+
scalio-catalog = "scalio_catalog.cli:main"
|
|
19
|
+
|
|
20
|
+
[tool.setuptools.packages.find]
|
|
21
|
+
where = ["src"]
|
|
22
|
+
|
|
23
|
+
[tool.setuptools.package-data]
|
|
24
|
+
scalio_catalog = ["schema/*.json"]
|
|
25
|
+
|
|
26
|
+
[tool.pytest.ini_options]
|
|
27
|
+
testpaths = ["tests"]
|
|
28
|
+
|
|
29
|
+
[tool.ruff]
|
|
30
|
+
line-length = 120
|
|
31
|
+
target-version = "py311"
|
|
32
|
+
|
|
33
|
+
[tool.ruff.lint]
|
|
34
|
+
select = ["E", "F", "W", "I", "B"]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Scalio Blueprint governance catalog (format scalio.catalog/v2, ADR 0027).
|
|
2
|
+
|
|
3
|
+
Shared by Foundry (roll out), Citadel (keep) and Rosetta (document). Contract: scalio-docs
|
|
4
|
+
interfaces/catalog-format.md.
|
|
5
|
+
"""
|
|
6
|
+
from .catalog import API_VERSION, Catalog, CatalogError, check_document, load_file, load_folder, parse_ref, schema
|
|
7
|
+
from .compare import Deviation, Result, compare, summary
|
|
8
|
+
from .effective import Effective, EffectiveControl, effective, render, require
|
|
9
|
+
from .pack import key_id, load_private_key, new_seed, pack, public_key_b64, verify
|
|
10
|
+
from .upgrade import Change, Plan, auto_update, next_versions, plan_upgrade
|
|
11
|
+
|
|
12
|
+
__version__ = "0.1.0"
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"API_VERSION", "Catalog", "CatalogError", "Change", "Deviation", "Effective", "EffectiveControl", "Plan",
|
|
16
|
+
"Result", "auto_update", "check_document", "compare", "effective", "key_id", "load_file", "load_folder",
|
|
17
|
+
"load_private_key", "new_seed", "next_versions", "pack", "parse_ref", "plan_upgrade", "public_key_b64",
|
|
18
|
+
"render", "require", "schema", "summary", "verify",
|
|
19
|
+
]
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"""Load and check catalog documents (scalio.catalog/v2).
|
|
2
|
+
|
|
3
|
+
A catalog is a set of documents (ResourceType, Control, Baseline, optional Pack and Assignment), usually one YAML
|
|
4
|
+
file each. `Catalog` indexes them by kind, id and version and resolves references like `id@^1.2`.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import datetime as _dt
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from importlib import resources
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Iterable
|
|
15
|
+
|
|
16
|
+
import yaml
|
|
17
|
+
from jsonschema import Draft202012Validator
|
|
18
|
+
|
|
19
|
+
API_VERSION = "scalio.catalog/v2"
|
|
20
|
+
CATALOG_KINDS = ("ResourceType", "Control", "Baseline")
|
|
21
|
+
PLACEHOLDER = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}")
|
|
22
|
+
_REF = re.compile(r"^(?P<id>[A-Za-z0-9][A-Za-z0-9._-]*?)(?:@(?P<caret>\^)?(?P<ver>\d+(?:\.\d+){0,2}))?$")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class CatalogError(Exception):
|
|
26
|
+
"""A document or reference is not valid. `problems` lists every finding."""
|
|
27
|
+
|
|
28
|
+
def __init__(self, message: str, problems: list[str] | None = None):
|
|
29
|
+
super().__init__(message)
|
|
30
|
+
self.problems = problems or [message]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def schema() -> dict:
|
|
34
|
+
"""The JSON schema shipped with the package (also for editors: VS Code yaml.schemas)."""
|
|
35
|
+
return json.loads(resources.files("scalio_catalog").joinpath("schema/catalog-v2.schema.json").read_text("utf-8"))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
_VALIDATOR: Draft202012Validator | None = None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _validator() -> Draft202012Validator:
|
|
42
|
+
global _VALIDATOR
|
|
43
|
+
if _VALIDATOR is None:
|
|
44
|
+
_VALIDATOR = Draft202012Validator(schema(), format_checker=Draft202012Validator.FORMAT_CHECKER)
|
|
45
|
+
return _VALIDATOR
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def plain(value: Any) -> Any:
|
|
49
|
+
"""YAML gives dates and datetimes as objects; the format stores them as ISO strings."""
|
|
50
|
+
if isinstance(value, dict):
|
|
51
|
+
return {str(k): plain(v) for k, v in value.items()}
|
|
52
|
+
if isinstance(value, list):
|
|
53
|
+
return [plain(v) for v in value]
|
|
54
|
+
if isinstance(value, (_dt.date, _dt.datetime)):
|
|
55
|
+
return value.isoformat()
|
|
56
|
+
return value
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def version_key(version: str) -> tuple[int, int, int]:
|
|
60
|
+
a, b, c = (version.split(".") + ["0", "0"])[:3]
|
|
61
|
+
return int(a), int(b), int(c)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def parse_ref(ref: str) -> tuple[str, str | None, bool]:
|
|
65
|
+
"""'id@^1.2' -> ('id', '1.2', True); 'id@1.2.3' -> ('id', '1.2.3', False); 'id' -> ('id', None, False)."""
|
|
66
|
+
m = _REF.match(ref or "")
|
|
67
|
+
if not m:
|
|
68
|
+
raise CatalogError(f"invalid reference '{ref}'")
|
|
69
|
+
return m["id"], m["ver"], bool(m["caret"])
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def ref_matches(version: str, wanted: str | None, caret: bool) -> bool:
|
|
73
|
+
if wanted is None:
|
|
74
|
+
return True
|
|
75
|
+
have = version_key(version)
|
|
76
|
+
parts = [int(p) for p in wanted.split(".")]
|
|
77
|
+
if caret:
|
|
78
|
+
return have[0] == parts[0] and have >= tuple(parts + [0] * (3 - len(parts)))
|
|
79
|
+
return list(have[: len(parts)]) == parts
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def check_texts(doc: Any, where: str = "") -> list[str]:
|
|
83
|
+
"""German texts use 'ss', never the sharp s (Swiss spelling, all Scalio products)."""
|
|
84
|
+
found = []
|
|
85
|
+
if isinstance(doc, dict):
|
|
86
|
+
for k, v in doc.items():
|
|
87
|
+
found += check_texts(v, f"{where}.{k}" if where else str(k))
|
|
88
|
+
elif isinstance(doc, list):
|
|
89
|
+
for i, v in enumerate(doc):
|
|
90
|
+
found += check_texts(v, f"{where}[{i}]")
|
|
91
|
+
elif isinstance(doc, str) and "\u00df" in doc:
|
|
92
|
+
found.append(f"{where}: use 'ss' instead of the sharp s")
|
|
93
|
+
return found
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def check_document(doc: dict) -> list[str]:
|
|
97
|
+
"""Structure (JSON schema) and spelling of one document. Returns the problems, empty when fine."""
|
|
98
|
+
doc = plain(doc)
|
|
99
|
+
problems = [f"{'/'.join(str(p) for p in e.absolute_path) or '(root)'}: {e.message}"
|
|
100
|
+
for e in sorted(_validator().iter_errors(doc), key=lambda e: list(map(str, e.absolute_path)))]
|
|
101
|
+
return problems + check_texts(doc)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def placeholders(value: Any) -> set[str]:
|
|
105
|
+
if isinstance(value, str):
|
|
106
|
+
return set(PLACEHOLDER.findall(value))
|
|
107
|
+
if isinstance(value, dict):
|
|
108
|
+
return set().union(*(placeholders(v) for v in value.values())) if value else set()
|
|
109
|
+
if isinstance(value, list):
|
|
110
|
+
return set().union(*(placeholders(v) for v in value)) if value else set()
|
|
111
|
+
return set()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def label(doc: dict) -> str:
|
|
115
|
+
return f"{doc.get('kind')} {doc.get('id') or doc.get('name')}@{doc.get('version', '-')}"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@dataclass
|
|
119
|
+
class Catalog:
|
|
120
|
+
"""Documents by kind -> id -> version. Assignments and the Pack header are kept apart."""
|
|
121
|
+
|
|
122
|
+
docs: dict[str, dict[str, dict[str, dict]]] = field(default_factory=lambda: {k: {} for k in CATALOG_KINDS})
|
|
123
|
+
assignments: dict[str, dict] = field(default_factory=dict)
|
|
124
|
+
pack: dict | None = None
|
|
125
|
+
|
|
126
|
+
# ------------------------------------------------------------------ building
|
|
127
|
+
|
|
128
|
+
@classmethod
|
|
129
|
+
def from_documents(cls, documents: Iterable[dict], check: bool = True, base: "Catalog | None" = None) -> "Catalog":
|
|
130
|
+
"""`base` is a catalog whose documents references may also point to (e.g. the Scalio base pack for an
|
|
131
|
+
MSP library or a premium pack). It is used for the checks only and not copied in."""
|
|
132
|
+
cat = cls()
|
|
133
|
+
problems: list[str] = []
|
|
134
|
+
for raw in documents:
|
|
135
|
+
doc = plain(raw)
|
|
136
|
+
found = check_document(doc) if check else []
|
|
137
|
+
if found:
|
|
138
|
+
problems += [f"{label(doc)}: {p}" for p in found]
|
|
139
|
+
continue
|
|
140
|
+
problems += cat._add(doc)
|
|
141
|
+
if check and not problems:
|
|
142
|
+
problems += (base.merged(cat) if base else cat).check_references()
|
|
143
|
+
if problems:
|
|
144
|
+
raise CatalogError(f"{len(problems)} problem(s) in the catalog", problems)
|
|
145
|
+
return cat
|
|
146
|
+
|
|
147
|
+
@classmethod
|
|
148
|
+
def from_folder(cls, *folders: str | Path, check: bool = True) -> "Catalog":
|
|
149
|
+
return cls.from_documents(load_folder(*folders), check=check)
|
|
150
|
+
|
|
151
|
+
def _add(self, doc: dict) -> list[str]:
|
|
152
|
+
kind = doc["kind"]
|
|
153
|
+
if kind == "Pack":
|
|
154
|
+
if self.pack is not None:
|
|
155
|
+
return ["more than one Pack header"]
|
|
156
|
+
self.pack = doc
|
|
157
|
+
return []
|
|
158
|
+
if kind == "Assignment":
|
|
159
|
+
if doc["id"] in self.assignments:
|
|
160
|
+
return [f"{label(doc)}: duplicate"]
|
|
161
|
+
self.assignments[doc["id"]] = doc
|
|
162
|
+
return []
|
|
163
|
+
versions = self.docs[kind].setdefault(doc["id"], {})
|
|
164
|
+
if doc["version"] in versions:
|
|
165
|
+
return [f"{label(doc)}: duplicate"]
|
|
166
|
+
versions[doc["version"]] = doc
|
|
167
|
+
return []
|
|
168
|
+
|
|
169
|
+
def merged(self, other: "Catalog") -> "Catalog":
|
|
170
|
+
"""This catalog plus another one (e.g. Scalio master plus an MSP library). Same id+version must not differ."""
|
|
171
|
+
out = Catalog()
|
|
172
|
+
problems = []
|
|
173
|
+
for cat in (self, other):
|
|
174
|
+
for kind in CATALOG_KINDS:
|
|
175
|
+
for versions in cat.docs[kind].values():
|
|
176
|
+
for doc in versions.values():
|
|
177
|
+
have = out.docs[kind].get(doc["id"], {}).get(doc["version"])
|
|
178
|
+
if have is not None and have != doc:
|
|
179
|
+
problems.append(f"{label(doc)}: two different documents with the same version")
|
|
180
|
+
elif have is None:
|
|
181
|
+
out._add(doc)
|
|
182
|
+
out.assignments.update(cat.assignments)
|
|
183
|
+
if problems:
|
|
184
|
+
raise CatalogError("catalogs conflict", problems)
|
|
185
|
+
out.pack = self.pack
|
|
186
|
+
return out
|
|
187
|
+
|
|
188
|
+
# ------------------------------------------------------------------ lookup
|
|
189
|
+
|
|
190
|
+
def documents(self) -> list[dict]:
|
|
191
|
+
out = [self.pack] if self.pack else []
|
|
192
|
+
for kind in CATALOG_KINDS:
|
|
193
|
+
for vid in sorted(self.docs[kind]):
|
|
194
|
+
for ver in sorted(self.docs[kind][vid], key=version_key):
|
|
195
|
+
out.append(self.docs[kind][vid][ver])
|
|
196
|
+
return out + [self.assignments[k] for k in sorted(self.assignments)]
|
|
197
|
+
|
|
198
|
+
def versions(self, kind: str, id_: str) -> list[str]:
|
|
199
|
+
return sorted(self.docs[kind].get(id_, {}), key=version_key)
|
|
200
|
+
|
|
201
|
+
def resolve(self, kind: str, ref: str) -> dict:
|
|
202
|
+
"""Highest version that matches the reference."""
|
|
203
|
+
id_, wanted, caret = parse_ref(ref)
|
|
204
|
+
found = [v for v in self.versions(kind, id_) if ref_matches(v, wanted, caret)]
|
|
205
|
+
if not found:
|
|
206
|
+
raise CatalogError(f"{kind} '{ref}' not found")
|
|
207
|
+
return self.docs[kind][id_][found[-1]]
|
|
208
|
+
|
|
209
|
+
# ------------------------------------------------------------------ cross checks
|
|
210
|
+
|
|
211
|
+
def check_references(self) -> list[str]:
|
|
212
|
+
problems = []
|
|
213
|
+
for versions in self.docs["Control"].values():
|
|
214
|
+
for c in versions.values():
|
|
215
|
+
problems += [f"{label(c)}: {p}" for p in self._check_control(c)]
|
|
216
|
+
for versions in self.docs["Baseline"].values():
|
|
217
|
+
for b in versions.values():
|
|
218
|
+
for entry in b["controls"]:
|
|
219
|
+
ref = entry if isinstance(entry, str) else entry["ref"]
|
|
220
|
+
try:
|
|
221
|
+
self.resolve("Control", ref)
|
|
222
|
+
except CatalogError as e:
|
|
223
|
+
problems.append(f"{label(b)}: {e}")
|
|
224
|
+
for a in self.assignments.values():
|
|
225
|
+
try:
|
|
226
|
+
self.resolve("Baseline", a["baseline"])
|
|
227
|
+
except CatalogError as e:
|
|
228
|
+
problems.append(f"{label(a)}: {e}")
|
|
229
|
+
for ref in a.get("add") or []:
|
|
230
|
+
try:
|
|
231
|
+
self.resolve("Control", ref)
|
|
232
|
+
except CatalogError as e:
|
|
233
|
+
problems.append(f"{label(a)}: {e}")
|
|
234
|
+
return problems
|
|
235
|
+
|
|
236
|
+
def _check_control(self, c: dict) -> list[str]:
|
|
237
|
+
problems = []
|
|
238
|
+
params = c.get("parameters") or {}
|
|
239
|
+
used = placeholders(c["resource"]["match"]) | placeholders(c["desired"])
|
|
240
|
+
problems += [f"placeholder '{{{{{p}}}}}' is not a declared parameter" for p in sorted(used - set(params))]
|
|
241
|
+
for name, p in params.items():
|
|
242
|
+
if p["type"] == "enum" and not p.get("values"):
|
|
243
|
+
problems.append(f"parameter '{name}': enum needs values")
|
|
244
|
+
if "default" in p and p["type"] == "enum" and p["default"] not in (p.get("values") or []):
|
|
245
|
+
problems.append(f"parameter '{name}': default is not one of the values")
|
|
246
|
+
try:
|
|
247
|
+
rt = self.resolve("ResourceType", c["resource"]["type"])
|
|
248
|
+
except CatalogError as e:
|
|
249
|
+
return problems + [str(e)]
|
|
250
|
+
props = rt["properties"]
|
|
251
|
+
for key, value in {**c["resource"]["match"], **c["desired"]}.items():
|
|
252
|
+
prop = props.get(key)
|
|
253
|
+
if prop is None:
|
|
254
|
+
problems.append(f"'{key}' is not a property of {rt['id']}")
|
|
255
|
+
continue
|
|
256
|
+
if placeholders(value):
|
|
257
|
+
continue
|
|
258
|
+
problems += [f"'{key}': {p}" for p in type_problems(prop, value)]
|
|
259
|
+
return problems
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def type_problems(prop: dict, value: Any) -> list[str]:
|
|
263
|
+
t = prop["type"]
|
|
264
|
+
ok = {
|
|
265
|
+
"string": isinstance(value, str),
|
|
266
|
+
"integer": isinstance(value, int) and not isinstance(value, bool),
|
|
267
|
+
"boolean": isinstance(value, bool),
|
|
268
|
+
"enum": isinstance(value, str),
|
|
269
|
+
"list": isinstance(value, list),
|
|
270
|
+
"object": isinstance(value, dict),
|
|
271
|
+
}[t]
|
|
272
|
+
if not ok:
|
|
273
|
+
return [f"expected {t}, got {type(value).__name__}"]
|
|
274
|
+
if t == "enum" and value not in prop.get("values", []):
|
|
275
|
+
return [f"'{value}' is not one of {prop.get('values')}"]
|
|
276
|
+
return []
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def load_file(path: str | Path) -> list[dict]:
|
|
280
|
+
"""All documents of one YAML or JSON file (YAML may hold several, separated by ---)."""
|
|
281
|
+
text = Path(path).read_text("utf-8")
|
|
282
|
+
if str(path).endswith(".json"):
|
|
283
|
+
data = json.loads(text)
|
|
284
|
+
return data if isinstance(data, list) else [data]
|
|
285
|
+
return [plain(d) for d in yaml.safe_load_all(text) if d is not None]
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def load_folder(*folders: str | Path) -> list[dict]:
|
|
289
|
+
docs = []
|
|
290
|
+
for folder in folders:
|
|
291
|
+
folder = Path(folder)
|
|
292
|
+
files = [folder] if folder.is_file() else sorted(
|
|
293
|
+
p for p in folder.rglob("*") if p.suffix in (".yaml", ".yml", ".json") and p.is_file())
|
|
294
|
+
for f in files:
|
|
295
|
+
docs += load_file(f)
|
|
296
|
+
return docs
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Command line: scalio-catalog validate|pack|verify|keygen|effective|schema."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import datetime as _dt
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from .catalog import Catalog, CatalogError, load_file, schema
|
|
12
|
+
from .effective import effective
|
|
13
|
+
from .pack import key_id, load_private_key, new_seed, pack, public_key_b64, verify
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _base(paths) -> Catalog | None:
|
|
17
|
+
return Catalog.from_folder(*paths) if paths else None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def main(argv=None) -> int:
|
|
21
|
+
ap = argparse.ArgumentParser(prog="scalio-catalog", description=__doc__)
|
|
22
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
23
|
+
v = sub.add_parser("validate", help="check folders or files")
|
|
24
|
+
v.add_argument("paths", nargs="+")
|
|
25
|
+
p = sub.add_parser("pack", help="check and sign a pack folder (key from CATALOG_SIGNING_KEY or --key-file)")
|
|
26
|
+
p.add_argument("folder")
|
|
27
|
+
p.add_argument("-o", "--out", default="dist")
|
|
28
|
+
p.add_argument("--key-file")
|
|
29
|
+
p.add_argument("--base", nargs="*", default=[], help="folders other packs refer to")
|
|
30
|
+
c = sub.add_parser("verify", help="check a signed package")
|
|
31
|
+
c.add_argument("file")
|
|
32
|
+
c.add_argument("--key", action="append", required=True, help="trusted public key (base64)")
|
|
33
|
+
sub.add_parser("keygen", help="new signing key: store the seed in 1Password, embed the public key")
|
|
34
|
+
e = sub.add_parser("effective", help="effective desired state of an assignment as JSON")
|
|
35
|
+
e.add_argument("assignment")
|
|
36
|
+
e.add_argument("paths", nargs="+", help="catalog folders")
|
|
37
|
+
e.add_argument("--today")
|
|
38
|
+
sub.add_parser("schema", help="print the JSON schema (for editors)")
|
|
39
|
+
a = ap.parse_args(argv)
|
|
40
|
+
try:
|
|
41
|
+
if a.cmd == "validate":
|
|
42
|
+
cat = Catalog.from_folder(*a.paths)
|
|
43
|
+
n = sum(len(v) for k in cat.docs.values() for v in k.values())
|
|
44
|
+
print(f"ok: {n} document(s), {len(cat.assignments)} assignment(s)")
|
|
45
|
+
elif a.cmd == "pack":
|
|
46
|
+
seed = Path(a.key_file).read_text().strip() if a.key_file else os.environ.get("CATALOG_SIGNING_KEY", "")
|
|
47
|
+
if not seed:
|
|
48
|
+
raise CatalogError("no signing key: set CATALOG_SIGNING_KEY or pass --key-file")
|
|
49
|
+
priv = load_private_key(seed)
|
|
50
|
+
name, data = pack(a.folder, priv, _base(a.base))
|
|
51
|
+
out = Path(a.out)
|
|
52
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
53
|
+
(out / name).write_bytes(data)
|
|
54
|
+
print(f"{out / name} key {key_id(public_key_b64(priv))}")
|
|
55
|
+
elif a.cmd == "verify":
|
|
56
|
+
cat = verify(Path(a.file).read_bytes(), a.key)
|
|
57
|
+
print(f"ok: {cat.pack['name']} {cat.pack['version']}")
|
|
58
|
+
elif a.cmd == "keygen":
|
|
59
|
+
seed = new_seed()
|
|
60
|
+
pub = public_key_b64(load_private_key(seed))
|
|
61
|
+
print(f"seed (secret, 1Password only): {seed}\npublic key: {pub}\nkey id: {key_id(pub)}")
|
|
62
|
+
elif a.cmd == "effective":
|
|
63
|
+
cat = Catalog.from_folder(*a.paths)
|
|
64
|
+
docs = load_file(a.assignment)
|
|
65
|
+
today = _dt.date.fromisoformat(a.today) if a.today else None
|
|
66
|
+
print(json.dumps(effective(cat, Catalog.from_documents(docs, base=cat).assignments[docs[0]["id"]],
|
|
67
|
+
today).as_dict(), indent=2, ensure_ascii=False))
|
|
68
|
+
elif a.cmd == "schema":
|
|
69
|
+
print(json.dumps(schema(), indent=2, ensure_ascii=False))
|
|
70
|
+
except CatalogError as err:
|
|
71
|
+
print(f"error: {err}", file=sys.stderr)
|
|
72
|
+
for line in err.problems:
|
|
73
|
+
print(f" - {line}", file=sys.stderr)
|
|
74
|
+
return 1
|
|
75
|
+
return 0
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
if __name__ == "__main__":
|
|
79
|
+
raise SystemExit(main())
|