s-layer-semantic-formats 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- s_layer_semantic_formats-0.1.0/LICENSE +21 -0
- s_layer_semantic_formats-0.1.0/PKG-INFO +204 -0
- s_layer_semantic_formats-0.1.0/README.md +183 -0
- s_layer_semantic_formats-0.1.0/pyproject.toml +97 -0
- s_layer_semantic_formats-0.1.0/pyproject.toml.orig +87 -0
- s_layer_semantic_formats-0.1.0/src/s_layer_semantic_formats/__init__.py +105 -0
- s_layer_semantic_formats-0.1.0/src/s_layer_semantic_formats/_api.py +296 -0
- s_layer_semantic_formats-0.1.0/src/s_layer_semantic_formats/_errors.py +56 -0
- s_layer_semantic_formats-0.1.0/src/s_layer_semantic_formats/_models.py +768 -0
- s_layer_semantic_formats-0.1.0/src/s_layer_semantic_formats/_schema.py +50 -0
- s_layer_semantic_formats-0.1.0/src/s_layer_semantic_formats/py.typed +0 -0
- s_layer_semantic_formats-0.1.0/src/s_layer_semantic_formats/s_layer.schema.json +830 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Paul Garner
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: s-layer-semantic-formats
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: The S-LAYER semantic format vocabulary: typed models, normal form and JSON Schema
|
|
5
|
+
Author: Anentropic
|
|
6
|
+
Author-email: Anentropic <ego@anentropic.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Dist: pydantic>=2.11
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Project-URL: Homepage, https://github.com/s-layer/semantic-formats
|
|
18
|
+
Project-URL: Repository, https://github.com/s-layer/semantic-formats
|
|
19
|
+
Project-URL: Issues, https://github.com/s-layer/semantic-formats/issues
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# s-layer-semantic-formats
|
|
23
|
+
|
|
24
|
+
The shared definition of the S-LAYER semantic format vocabulary: the `s_layer` block that
|
|
25
|
+
says what a metric, fact or dimension *means* (a currency in USD, a fraction shown as a
|
|
26
|
+
percentage, a duration in seconds, a monthly date), written once in dbt `config.meta`.
|
|
27
|
+
|
|
28
|
+
- [dbt-s-layer](https://github.com/s-layer/dbt-s-layer) validates blocks and writes them into
|
|
29
|
+
each warehouse.
|
|
30
|
+
- [semolina](https://github.com/s-layer/semolina) reads them back.
|
|
31
|
+
|
|
32
|
+
This package holds what both sides must agree on:
|
|
33
|
+
|
|
34
|
+
- typed pydantic v2 models for the block, with `validate_block()` for a member type;
|
|
35
|
+
- the normal form that defines equality (`normalize()`);
|
|
36
|
+
- the compact `S_LAYER_SPEC` tag payload used by Snowflake and DuckDB semantic views, and its
|
|
37
|
+
inverse;
|
|
38
|
+
- the JSON Schema, generated from the models and shipped as
|
|
39
|
+
`s_layer_semantic_formats/s_layer.schema.json`.
|
|
40
|
+
|
|
41
|
+
It contains no warehouse code and no dbt code.
|
|
42
|
+
|
|
43
|
+
## The block
|
|
44
|
+
|
|
45
|
+
```yaml
|
|
46
|
+
metrics:
|
|
47
|
+
- name: revenue
|
|
48
|
+
config:
|
|
49
|
+
meta:
|
|
50
|
+
s_layer:
|
|
51
|
+
value: {kind: currency, currency: USD}
|
|
52
|
+
display: {decimals: {max: 0}, compact: auto}
|
|
53
|
+
synonyms: [sales, turnover]
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
| Part | Keys |
|
|
57
|
+
| ------------- | ---------------------------------------------------------------------------- |
|
|
58
|
+
| `value` | `kind` (required) plus the kind's own keys, such as `currency` or `unit` |
|
|
59
|
+
| `display` | `decimals {min, max}`, `compact`, `negative`, `grouping`, `date_style` |
|
|
60
|
+
| `time` | `grain`, `fiscal_year_start` (date and datetime dimensions only) |
|
|
61
|
+
| `aggregation` | `additivity`, plus `by` when `semi_additive` (metrics only) |
|
|
62
|
+
| `label` | only where the platform has no label slot |
|
|
63
|
+
| `synonyms` | only where the platform has no synonyms slot |
|
|
64
|
+
| `x-<vendor>` | extension keys, carried verbatim but not interpreted |
|
|
65
|
+
|
|
66
|
+
Kinds: `number`, `count`, `currency`, `percentage`, `duration`, `bytes`, `quantity`, `score`
|
|
67
|
+
(metrics, and some facts) and `category`, `geo`, `date`, `datetime`, `identifier`, `boolean`
|
|
68
|
+
(dimensions). `KIND_MEMBER_TYPES` has the exact table.
|
|
69
|
+
|
|
70
|
+
## Usage
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from s_layer_semantic_formats import (
|
|
74
|
+
SLayerValidationError,
|
|
75
|
+
from_compact_json,
|
|
76
|
+
normalize,
|
|
77
|
+
spec_tag_value,
|
|
78
|
+
validate_block,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
block = validate_block(
|
|
82
|
+
{
|
|
83
|
+
"value": {"kind": "currency", "currency": "USD"},
|
|
84
|
+
"display": {"decimals": {"max": 0}, "compact": "auto"},
|
|
85
|
+
},
|
|
86
|
+
"metric",
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
normalize(block)
|
|
90
|
+
# {'display': {'compact': 'auto', 'decimals': [0, 0],
|
|
91
|
+
# 'grouping': True, 'negative': 'minus'},
|
|
92
|
+
# 'value': {'currency': 'USD', 'kind': 'currency'}}
|
|
93
|
+
|
|
94
|
+
# The S_LAYER_SPEC tag value (raises SpecTooLongError past 256 chars)
|
|
95
|
+
spec = spec_tag_value(block)
|
|
96
|
+
# '{"display":{"compact":"auto","decimals":[0,0],"grouping":true,"negative":"minus"},"value":{"currency":"USD"}}'
|
|
97
|
+
|
|
98
|
+
# Reading the tags back gives an equal block
|
|
99
|
+
assert from_compact_json("currency", spec) == block
|
|
100
|
+
|
|
101
|
+
try:
|
|
102
|
+
validate_block({"value": {"kind": "currency"}}, "dimension")
|
|
103
|
+
except SLayerValidationError as error:
|
|
104
|
+
print(error)
|
|
105
|
+
# invalid s_layer block (1 problem):
|
|
106
|
+
# value.currency: required key is missing
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`SLayerValidationError.issues` lists every problem as an `Issue(path, message)`.
|
|
110
|
+
|
|
111
|
+
## Public API
|
|
112
|
+
|
|
113
|
+
| Name | What it does |
|
|
114
|
+
| -------------------------------------------------------- | --------------------------------------------------------------- |
|
|
115
|
+
| `validate_block(data, member_type)` | Parse a block and check its kind applies to the member type |
|
|
116
|
+
| `parse_block(data)` | Parse a block without the member-type check |
|
|
117
|
+
| `member_type_issues(block, member_type)` | The member-type problems of a parsed block, as a list |
|
|
118
|
+
| `normalize(block)` | The normal form as JSON-ready data |
|
|
119
|
+
| `to_compact_json(block)` | The `S_LAYER_SPEC` payload |
|
|
120
|
+
| `spec_tag_value(block, limit=256)` | `to_compact_json`, raising `SpecTooLongError` when too long |
|
|
121
|
+
| `from_compact_json(kind, spec, *, label, synonyms, ...)` | Rebuild a block from `S_LAYER_KIND`, `S_LAYER_SPEC` and label |
|
|
122
|
+
| `implied_additivity(kind)` | `"non_additive"` for percentage and score, else `None` |
|
|
123
|
+
| `json_schema()`, `load_schema()`, `SCHEMA_PATH` | The generated schema, and the committed copy |
|
|
124
|
+
| `SLayerBlock` and the section and value models | The typed block; `block.kind`, `block.additivity`, `.extensions` |
|
|
125
|
+
|
|
126
|
+
Blocks compare equal (`==`, `hash`) on their normal form, so `{max: 2}` and
|
|
127
|
+
`{min: 0, max: 2}` are the same value.
|
|
128
|
+
|
|
129
|
+
## Decisions not in the design doc
|
|
130
|
+
|
|
131
|
+
The design doc is a draft. Where it is silent, this package picks the simplest rule it could
|
|
132
|
+
find:
|
|
133
|
+
|
|
134
|
+
1. **Defaults the normal form fills in are only the ones the doc states**: percentage
|
|
135
|
+
`scale: fraction`, bytes `unit: B`, decimals `min: 0`, and on numeric kinds
|
|
136
|
+
`negative: minus` and `grouping: true`; `identifier` gets `grouping: false`. Every other
|
|
137
|
+
optional key (`compact`, `date_style`, `grain`, `fiscal_year_start`, `code`, number
|
|
138
|
+
`unit`) is left out when absent, so a missing `compact` is not equal to `compact: none`.
|
|
139
|
+
2. **Implied additivity is not written into the block.** `percentage` and `score` imply
|
|
140
|
+
`non_additive`, exposed by `implied_additivity()` and `SLayerBlock.additivity`, but the
|
|
141
|
+
normal form and `S_LAYER_SPEC` only carry an `aggregation` someone wrote ("derive, don't
|
|
142
|
+
duplicate"). A stated `aggregation` on those kinds must be `non_additive`.
|
|
143
|
+
3. **Required kind keys**: `duration.unit`, `score.range` and `geo.level` are required, as
|
|
144
|
+
their table rows have no `?` and no default. `boolean.labels` needs both `true` and
|
|
145
|
+
`false`; unquoted YAML `true:` / `false:` keys (read as booleans) are accepted too.
|
|
146
|
+
4. **Allowed values the doc leaves open**: bytes units are `B`, `KB` … `EB` and `KiB` …
|
|
147
|
+
`EiB`. A number's optional `unit` is any non-empty string, like quantity's UCUM code.
|
|
148
|
+
Currency is checked against `^[A-Z]{3}$` only. A geo `code` is any non-empty string.
|
|
149
|
+
5. **Score range**: two finite numbers with `min < max`; whole numbers are written as
|
|
150
|
+
integers in the normal form, so `[0.0, 10]` and `[0, 10]` are equal.
|
|
151
|
+
6. **Decimals**: integers 0-20, `max` is required when `decimals` is given, `min <= max`.
|
|
152
|
+
Both `{min, max}` and `[min, max]` are accepted; the normal form is the pair.
|
|
153
|
+
7. **Display keys must fit the kind**: `decimals`, `compact` and `negative` only on numeric
|
|
154
|
+
kinds; `grouping` on numeric kinds and on `identifier` (where only `false` is allowed);
|
|
155
|
+
`date_style` only on `date` and `datetime`.
|
|
156
|
+
8. **`time` only on `date` and `datetime`**, and a `date` cannot take a grain finer than a
|
|
157
|
+
day (`second`, `minute`, `hour`).
|
|
158
|
+
9. **`aggregation` only on metrics** (checked by `validate_block`). `by` is required exactly
|
|
159
|
+
when `semi_additive`.
|
|
160
|
+
10. **Lists** (`category.order`, `aggregation.by`, `synonyms`) are non-empty lists of
|
|
161
|
+
distinct non-empty strings. Their order is kept, since it can carry meaning.
|
|
162
|
+
11. **`label`** is 1-255 characters, which fits both Databricks `display_name` (255) and the
|
|
163
|
+
256-character tag limit. Whether a platform has a native slot (and so whether `label` or
|
|
164
|
+
`synonyms` belong in the block at all) is left to dbt-s-layer.
|
|
165
|
+
12. **Extension keys** are allowed on the block and on `value`, `display`, `time` and
|
|
166
|
+
`aggregation`, not inside `decimals`, `labels` or `range`. A key is an extension when it
|
|
167
|
+
starts with lower-case `x-` and has at least one more character. Values can be any JSON
|
|
168
|
+
(so no NaN or Infinity) and are kept as they are; only object key order is sorted in the normal form.
|
|
169
|
+
13. **`null` means absent** for any optional key, and an empty `display: {}` or `time: {}`
|
|
170
|
+
is the same as leaving it out.
|
|
171
|
+
14. **Strict scalar types**: no coercion from strings or between bools and numbers
|
|
172
|
+
(`grouping: "yes"`, `fiscal_year_start: "4"` and `decimals.max: 2.0` are errors).
|
|
173
|
+
15. **`S_LAYER_SPEC` keeps the block's nesting** (`{"display": {...}, "value": {...}}`)
|
|
174
|
+
minus `value.kind`, `label` and `synonyms`; an empty `value` object is dropped, so a bare
|
|
175
|
+
`date` block's spec is `{}`. Non-ASCII text is written as is, and the 256 limit counts
|
|
176
|
+
characters and is inclusive. `from_compact_json` rejects a spec that contains `kind`,
|
|
177
|
+
`label` or `synonyms`, or NaN or Infinity. It treats an empty label or synonyms list
|
|
178
|
+
as none, and rejects a bare string of synonyms.
|
|
179
|
+
16. **Equality compares the canonical JSON text** of the normal form, so extension values
|
|
180
|
+
`1`, `1.0` and `true` are different.
|
|
181
|
+
17. **The JSON Schema checks shape only.** Which kinds apply to which member type, and the
|
|
182
|
+
rules between sections (items 2, 7, 8, 9), are enforced by the Python models and
|
|
183
|
+
`validate_block`. So are a few rules within one key that the schema does not state: `2.0`
|
|
184
|
+
where an integer is required (item 14), `min <= max` for decimals and `min < max` for a score
|
|
185
|
+
range (items 5, 6), `by` being present exactly when `semi_additive` (item 9), and finite
|
|
186
|
+
extension values (item 12). A block that passes the schema can still fail
|
|
187
|
+
`validate_block`. The schema accepts `null` for optional keys, matching item 13.
|
|
188
|
+
|
|
189
|
+
## Development
|
|
190
|
+
|
|
191
|
+
```sh
|
|
192
|
+
uv sync --dev
|
|
193
|
+
prek install # git hooks
|
|
194
|
+
uv run pytest # tests, including hypothesis property tests
|
|
195
|
+
prek run --all-files # ruff, basedpyright, uv lock and schema checks
|
|
196
|
+
just schema # regenerate s_layer.schema.json after changing the models
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
Without `just`, regenerate the schema with `uv run python scripts/regen_schema.py`
|
|
200
|
+
(`--check` only compares). See [DEVELOP.md](DEVELOP.md) for the full guide.
|
|
201
|
+
|
|
202
|
+
## License
|
|
203
|
+
|
|
204
|
+
MIT
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# s-layer-semantic-formats
|
|
2
|
+
|
|
3
|
+
The shared definition of the S-LAYER semantic format vocabulary: the `s_layer` block that
|
|
4
|
+
says what a metric, fact or dimension *means* (a currency in USD, a fraction shown as a
|
|
5
|
+
percentage, a duration in seconds, a monthly date), written once in dbt `config.meta`.
|
|
6
|
+
|
|
7
|
+
- [dbt-s-layer](https://github.com/s-layer/dbt-s-layer) validates blocks and writes them into
|
|
8
|
+
each warehouse.
|
|
9
|
+
- [semolina](https://github.com/s-layer/semolina) reads them back.
|
|
10
|
+
|
|
11
|
+
This package holds what both sides must agree on:
|
|
12
|
+
|
|
13
|
+
- typed pydantic v2 models for the block, with `validate_block()` for a member type;
|
|
14
|
+
- the normal form that defines equality (`normalize()`);
|
|
15
|
+
- the compact `S_LAYER_SPEC` tag payload used by Snowflake and DuckDB semantic views, and its
|
|
16
|
+
inverse;
|
|
17
|
+
- the JSON Schema, generated from the models and shipped as
|
|
18
|
+
`s_layer_semantic_formats/s_layer.schema.json`.
|
|
19
|
+
|
|
20
|
+
It contains no warehouse code and no dbt code.
|
|
21
|
+
|
|
22
|
+
## The block
|
|
23
|
+
|
|
24
|
+
```yaml
|
|
25
|
+
metrics:
|
|
26
|
+
- name: revenue
|
|
27
|
+
config:
|
|
28
|
+
meta:
|
|
29
|
+
s_layer:
|
|
30
|
+
value: {kind: currency, currency: USD}
|
|
31
|
+
display: {decimals: {max: 0}, compact: auto}
|
|
32
|
+
synonyms: [sales, turnover]
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
| Part | Keys |
|
|
36
|
+
| ------------- | ---------------------------------------------------------------------------- |
|
|
37
|
+
| `value` | `kind` (required) plus the kind's own keys, such as `currency` or `unit` |
|
|
38
|
+
| `display` | `decimals {min, max}`, `compact`, `negative`, `grouping`, `date_style` |
|
|
39
|
+
| `time` | `grain`, `fiscal_year_start` (date and datetime dimensions only) |
|
|
40
|
+
| `aggregation` | `additivity`, plus `by` when `semi_additive` (metrics only) |
|
|
41
|
+
| `label` | only where the platform has no label slot |
|
|
42
|
+
| `synonyms` | only where the platform has no synonyms slot |
|
|
43
|
+
| `x-<vendor>` | extension keys, carried verbatim but not interpreted |
|
|
44
|
+
|
|
45
|
+
Kinds: `number`, `count`, `currency`, `percentage`, `duration`, `bytes`, `quantity`, `score`
|
|
46
|
+
(metrics, and some facts) and `category`, `geo`, `date`, `datetime`, `identifier`, `boolean`
|
|
47
|
+
(dimensions). `KIND_MEMBER_TYPES` has the exact table.
|
|
48
|
+
|
|
49
|
+
## Usage
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from s_layer_semantic_formats import (
|
|
53
|
+
SLayerValidationError,
|
|
54
|
+
from_compact_json,
|
|
55
|
+
normalize,
|
|
56
|
+
spec_tag_value,
|
|
57
|
+
validate_block,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
block = validate_block(
|
|
61
|
+
{
|
|
62
|
+
"value": {"kind": "currency", "currency": "USD"},
|
|
63
|
+
"display": {"decimals": {"max": 0}, "compact": "auto"},
|
|
64
|
+
},
|
|
65
|
+
"metric",
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
normalize(block)
|
|
69
|
+
# {'display': {'compact': 'auto', 'decimals': [0, 0],
|
|
70
|
+
# 'grouping': True, 'negative': 'minus'},
|
|
71
|
+
# 'value': {'currency': 'USD', 'kind': 'currency'}}
|
|
72
|
+
|
|
73
|
+
# The S_LAYER_SPEC tag value (raises SpecTooLongError past 256 chars)
|
|
74
|
+
spec = spec_tag_value(block)
|
|
75
|
+
# '{"display":{"compact":"auto","decimals":[0,0],"grouping":true,"negative":"minus"},"value":{"currency":"USD"}}'
|
|
76
|
+
|
|
77
|
+
# Reading the tags back gives an equal block
|
|
78
|
+
assert from_compact_json("currency", spec) == block
|
|
79
|
+
|
|
80
|
+
try:
|
|
81
|
+
validate_block({"value": {"kind": "currency"}}, "dimension")
|
|
82
|
+
except SLayerValidationError as error:
|
|
83
|
+
print(error)
|
|
84
|
+
# invalid s_layer block (1 problem):
|
|
85
|
+
# value.currency: required key is missing
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
`SLayerValidationError.issues` lists every problem as an `Issue(path, message)`.
|
|
89
|
+
|
|
90
|
+
## Public API
|
|
91
|
+
|
|
92
|
+
| Name | What it does |
|
|
93
|
+
| -------------------------------------------------------- | --------------------------------------------------------------- |
|
|
94
|
+
| `validate_block(data, member_type)` | Parse a block and check its kind applies to the member type |
|
|
95
|
+
| `parse_block(data)` | Parse a block without the member-type check |
|
|
96
|
+
| `member_type_issues(block, member_type)` | The member-type problems of a parsed block, as a list |
|
|
97
|
+
| `normalize(block)` | The normal form as JSON-ready data |
|
|
98
|
+
| `to_compact_json(block)` | The `S_LAYER_SPEC` payload |
|
|
99
|
+
| `spec_tag_value(block, limit=256)` | `to_compact_json`, raising `SpecTooLongError` when too long |
|
|
100
|
+
| `from_compact_json(kind, spec, *, label, synonyms, ...)` | Rebuild a block from `S_LAYER_KIND`, `S_LAYER_SPEC` and label |
|
|
101
|
+
| `implied_additivity(kind)` | `"non_additive"` for percentage and score, else `None` |
|
|
102
|
+
| `json_schema()`, `load_schema()`, `SCHEMA_PATH` | The generated schema, and the committed copy |
|
|
103
|
+
| `SLayerBlock` and the section and value models | The typed block; `block.kind`, `block.additivity`, `.extensions` |
|
|
104
|
+
|
|
105
|
+
Blocks compare equal (`==`, `hash`) on their normal form, so `{max: 2}` and
|
|
106
|
+
`{min: 0, max: 2}` are the same value.
|
|
107
|
+
|
|
108
|
+
## Decisions not in the design doc
|
|
109
|
+
|
|
110
|
+
The design doc is a draft. Where it is silent, this package picks the simplest rule it could
|
|
111
|
+
find:
|
|
112
|
+
|
|
113
|
+
1. **Defaults the normal form fills in are only the ones the doc states**: percentage
|
|
114
|
+
`scale: fraction`, bytes `unit: B`, decimals `min: 0`, and on numeric kinds
|
|
115
|
+
`negative: minus` and `grouping: true`; `identifier` gets `grouping: false`. Every other
|
|
116
|
+
optional key (`compact`, `date_style`, `grain`, `fiscal_year_start`, `code`, number
|
|
117
|
+
`unit`) is left out when absent, so a missing `compact` is not equal to `compact: none`.
|
|
118
|
+
2. **Implied additivity is not written into the block.** `percentage` and `score` imply
|
|
119
|
+
`non_additive`, exposed by `implied_additivity()` and `SLayerBlock.additivity`, but the
|
|
120
|
+
normal form and `S_LAYER_SPEC` only carry an `aggregation` someone wrote ("derive, don't
|
|
121
|
+
duplicate"). A stated `aggregation` on those kinds must be `non_additive`.
|
|
122
|
+
3. **Required kind keys**: `duration.unit`, `score.range` and `geo.level` are required, as
|
|
123
|
+
their table rows have no `?` and no default. `boolean.labels` needs both `true` and
|
|
124
|
+
`false`; unquoted YAML `true:` / `false:` keys (read as booleans) are accepted too.
|
|
125
|
+
4. **Allowed values the doc leaves open**: bytes units are `B`, `KB` … `EB` and `KiB` …
|
|
126
|
+
`EiB`. A number's optional `unit` is any non-empty string, like quantity's UCUM code.
|
|
127
|
+
Currency is checked against `^[A-Z]{3}$` only. A geo `code` is any non-empty string.
|
|
128
|
+
5. **Score range**: two finite numbers with `min < max`; whole numbers are written as
|
|
129
|
+
integers in the normal form, so `[0.0, 10]` and `[0, 10]` are equal.
|
|
130
|
+
6. **Decimals**: integers 0-20, `max` is required when `decimals` is given, `min <= max`.
|
|
131
|
+
Both `{min, max}` and `[min, max]` are accepted; the normal form is the pair.
|
|
132
|
+
7. **Display keys must fit the kind**: `decimals`, `compact` and `negative` only on numeric
|
|
133
|
+
kinds; `grouping` on numeric kinds and on `identifier` (where only `false` is allowed);
|
|
134
|
+
`date_style` only on `date` and `datetime`.
|
|
135
|
+
8. **`time` only on `date` and `datetime`**, and a `date` cannot take a grain finer than a
|
|
136
|
+
day (`second`, `minute`, `hour`).
|
|
137
|
+
9. **`aggregation` only on metrics** (checked by `validate_block`). `by` is required exactly
|
|
138
|
+
when `semi_additive`.
|
|
139
|
+
10. **Lists** (`category.order`, `aggregation.by`, `synonyms`) are non-empty lists of
|
|
140
|
+
distinct non-empty strings. Their order is kept, since it can carry meaning.
|
|
141
|
+
11. **`label`** is 1-255 characters, which fits both Databricks `display_name` (255) and the
|
|
142
|
+
256-character tag limit. Whether a platform has a native slot (and so whether `label` or
|
|
143
|
+
`synonyms` belong in the block at all) is left to dbt-s-layer.
|
|
144
|
+
12. **Extension keys** are allowed on the block and on `value`, `display`, `time` and
|
|
145
|
+
`aggregation`, not inside `decimals`, `labels` or `range`. A key is an extension when it
|
|
146
|
+
starts with lower-case `x-` and has at least one more character. Values can be any JSON
|
|
147
|
+
(so no NaN or Infinity) and are kept as they are; only object key order is sorted in the normal form.
|
|
148
|
+
13. **`null` means absent** for any optional key, and an empty `display: {}` or `time: {}`
|
|
149
|
+
is the same as leaving it out.
|
|
150
|
+
14. **Strict scalar types**: no coercion from strings or between bools and numbers
|
|
151
|
+
(`grouping: "yes"`, `fiscal_year_start: "4"` and `decimals.max: 2.0` are errors).
|
|
152
|
+
15. **`S_LAYER_SPEC` keeps the block's nesting** (`{"display": {...}, "value": {...}}`)
|
|
153
|
+
minus `value.kind`, `label` and `synonyms`; an empty `value` object is dropped, so a bare
|
|
154
|
+
`date` block's spec is `{}`. Non-ASCII text is written as is, and the 256 limit counts
|
|
155
|
+
characters and is inclusive. `from_compact_json` rejects a spec that contains `kind`,
|
|
156
|
+
`label` or `synonyms`, or NaN or Infinity. It treats an empty label or synonyms list
|
|
157
|
+
as none, and rejects a bare string of synonyms.
|
|
158
|
+
16. **Equality compares the canonical JSON text** of the normal form, so extension values
|
|
159
|
+
`1`, `1.0` and `true` are different.
|
|
160
|
+
17. **The JSON Schema checks shape only.** Which kinds apply to which member type, and the
|
|
161
|
+
rules between sections (items 2, 7, 8, 9), are enforced by the Python models and
|
|
162
|
+
`validate_block`. So are a few rules within one key that the schema does not state: `2.0`
|
|
163
|
+
where an integer is required (item 14), `min <= max` for decimals and `min < max` for a score
|
|
164
|
+
range (items 5, 6), `by` being present exactly when `semi_additive` (item 9), and finite
|
|
165
|
+
extension values (item 12). A block that passes the schema can still fail
|
|
166
|
+
`validate_block`. The schema accepts `null` for optional keys, matching item 13.
|
|
167
|
+
|
|
168
|
+
## Development
|
|
169
|
+
|
|
170
|
+
```sh
|
|
171
|
+
uv sync --dev
|
|
172
|
+
prek install # git hooks
|
|
173
|
+
uv run pytest # tests, including hypothesis property tests
|
|
174
|
+
prek run --all-files # ruff, basedpyright, uv lock and schema checks
|
|
175
|
+
just schema # regenerate s_layer.schema.json after changing the models
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Without `just`, regenerate the schema with `uv run python scripts/regen_schema.py`
|
|
179
|
+
(`--check` only compares). See [DEVELOP.md](DEVELOP.md) for the full guide.
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
MIT
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "s-layer-semantic-formats"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "The S-LAYER semantic format vocabulary: typed models, normal form and JSON Schema"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.11"
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Programming Language :: Python :: 3",
|
|
11
|
+
"Programming Language :: Python :: 3.11",
|
|
12
|
+
"Programming Language :: Python :: 3.12",
|
|
13
|
+
"Programming Language :: Python :: 3.13",
|
|
14
|
+
"Programming Language :: Python :: 3.14",
|
|
15
|
+
"Typing :: Typed",
|
|
16
|
+
]
|
|
17
|
+
dependencies = ["pydantic>=2.11"]
|
|
18
|
+
|
|
19
|
+
[[project.authors]]
|
|
20
|
+
name = "Anentropic"
|
|
21
|
+
email = "ego@anentropic.com"
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://github.com/s-layer/semantic-formats"
|
|
25
|
+
Repository = "https://github.com/s-layer/semantic-formats"
|
|
26
|
+
Issues = "https://github.com/s-layer/semantic-formats/issues"
|
|
27
|
+
|
|
28
|
+
[build-system]
|
|
29
|
+
requires = ["uv_build>=0.9.18,<1.0.0"]
|
|
30
|
+
build-backend = "uv_build"
|
|
31
|
+
|
|
32
|
+
[dependency-groups]
|
|
33
|
+
dev = [
|
|
34
|
+
"basedpyright>=1.38.0",
|
|
35
|
+
"hypothesis>=6.100",
|
|
36
|
+
"ipython>=9.10.0",
|
|
37
|
+
"jsonschema>=4.23",
|
|
38
|
+
"pdbpp>=0.12.0.post1",
|
|
39
|
+
"pytest>=8.0.0",
|
|
40
|
+
"pytest-cov>=6.0.0",
|
|
41
|
+
"ruff>=0.15.1",
|
|
42
|
+
"types-jsonschema>=4.26.0.20261006",
|
|
43
|
+
]
|
|
44
|
+
docs = [
|
|
45
|
+
"sphinx>=8.0.0",
|
|
46
|
+
"shibuya",
|
|
47
|
+
"sphinx-design>=0.6.0",
|
|
48
|
+
"sphinx-autoapi>=3.0.0",
|
|
49
|
+
"sphinx-autobuild>=2024.0.0",
|
|
50
|
+
"myst-parser>=4.0.0",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
[tool.pyright]
|
|
54
|
+
pythonVersion = "3.11"
|
|
55
|
+
typeCheckingMode = "strict"
|
|
56
|
+
include = [
|
|
57
|
+
"src",
|
|
58
|
+
"tests",
|
|
59
|
+
"scripts",
|
|
60
|
+
]
|
|
61
|
+
reportPrivateUsage = false
|
|
62
|
+
reportUnnecessaryTypeIgnoreComment = "error"
|
|
63
|
+
|
|
64
|
+
[tool.ruff]
|
|
65
|
+
target-version = "py311"
|
|
66
|
+
line-length = 100
|
|
67
|
+
src = ["src"]
|
|
68
|
+
|
|
69
|
+
[tool.ruff.lint]
|
|
70
|
+
select = [
|
|
71
|
+
"E",
|
|
72
|
+
"F",
|
|
73
|
+
"W",
|
|
74
|
+
"I",
|
|
75
|
+
"UP",
|
|
76
|
+
"B",
|
|
77
|
+
"SIM",
|
|
78
|
+
"TCH",
|
|
79
|
+
"D",
|
|
80
|
+
"TID",
|
|
81
|
+
"BLE",
|
|
82
|
+
]
|
|
83
|
+
ignore = [
|
|
84
|
+
"D1",
|
|
85
|
+
"D202",
|
|
86
|
+
"D203",
|
|
87
|
+
"D212",
|
|
88
|
+
"D401",
|
|
89
|
+
"D413",
|
|
90
|
+
]
|
|
91
|
+
|
|
92
|
+
[tool.ruff.lint.per-file-ignores]
|
|
93
|
+
"tests/**" = ["BLE001"]
|
|
94
|
+
|
|
95
|
+
[tool.pytest.ini_options]
|
|
96
|
+
addopts = "-v --strict-markers"
|
|
97
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "s-layer-semantic-formats"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "The S-LAYER semantic format vocabulary: typed models, normal form and JSON Schema"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Anentropic", email = "ego@anentropic.com" }
|
|
10
|
+
]
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Programming Language :: Python :: 3.14",
|
|
18
|
+
"Typing :: Typed",
|
|
19
|
+
]
|
|
20
|
+
dependencies = [
|
|
21
|
+
"pydantic>=2.11",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/s-layer/semantic-formats"
|
|
26
|
+
Repository = "https://github.com/s-layer/semantic-formats"
|
|
27
|
+
Issues = "https://github.com/s-layer/semantic-formats/issues"
|
|
28
|
+
|
|
29
|
+
[build-system]
|
|
30
|
+
requires = ["uv_build>=0.9.18,<1.0.0"]
|
|
31
|
+
build-backend = "uv_build"
|
|
32
|
+
|
|
33
|
+
[dependency-groups]
|
|
34
|
+
dev = [
|
|
35
|
+
"basedpyright>=1.38.0",
|
|
36
|
+
"hypothesis>=6.100",
|
|
37
|
+
"ipython>=9.10.0",
|
|
38
|
+
"jsonschema>=4.23",
|
|
39
|
+
"pdbpp>=0.12.0.post1",
|
|
40
|
+
"pytest>=8.0.0",
|
|
41
|
+
"pytest-cov>=6.0.0",
|
|
42
|
+
"ruff>=0.15.1",
|
|
43
|
+
"types-jsonschema>=4.26.0.20261006",
|
|
44
|
+
]
|
|
45
|
+
docs = [
|
|
46
|
+
"sphinx>=8.0.0",
|
|
47
|
+
"shibuya",
|
|
48
|
+
"sphinx-design>=0.6.0",
|
|
49
|
+
"sphinx-autoapi>=3.0.0",
|
|
50
|
+
"sphinx-autobuild>=2024.0.0",
|
|
51
|
+
"myst-parser>=4.0.0",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
[tool.pyright]
|
|
55
|
+
# The minimum supported version, not the dev version: an explicit pythonVersion
|
|
56
|
+
# overrides the running interpreter, so setting it to the dev version lets newer
|
|
57
|
+
# stdlib and typing APIs type-check clean on *every* CI leg -- including the
|
|
58
|
+
# minimum-version leg whose job is to catch them -- and fail at runtime instead.
|
|
59
|
+
# Kept in step with requires-python and ruff's target-version below.
|
|
60
|
+
pythonVersion = "3.11"
|
|
61
|
+
typeCheckingMode = "strict"
|
|
62
|
+
include = ["src", "tests", "scripts"]
|
|
63
|
+
reportPrivateUsage = false
|
|
64
|
+
reportUnnecessaryTypeIgnoreComment = "error"
|
|
65
|
+
|
|
66
|
+
[tool.ruff]
|
|
67
|
+
target-version = "py311"
|
|
68
|
+
line-length = 100
|
|
69
|
+
src = ["src"]
|
|
70
|
+
|
|
71
|
+
[tool.ruff.lint]
|
|
72
|
+
select = ["E", "F", "W", "I", "UP", "B", "SIM", "TCH", "D", "TID", "BLE"]
|
|
73
|
+
ignore = [
|
|
74
|
+
"D1", # don't require docstrings everywhere
|
|
75
|
+
"D202", # allow blank line after docstring (common in tests)
|
|
76
|
+
"D203", # conflicts with D211 (no blank line before class docstring)
|
|
77
|
+
"D212", # conflicts with D213 (multiline summary on second line)
|
|
78
|
+
"D401", # imperative mood is awkward for dunder methods
|
|
79
|
+
"D413", # blank line after last section not required
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
[tool.ruff.lint.per-file-ignores]
|
|
83
|
+
"tests/**" = ["BLE001"]
|
|
84
|
+
|
|
85
|
+
[tool.pytest.ini_options]
|
|
86
|
+
addopts = "-v --strict-markers"
|
|
87
|
+
testpaths = ["tests"]
|