dpm-xl 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dpm_xl-0.1.0/LICENSE +21 -0
- dpm_xl-0.1.0/PKG-INFO +194 -0
- dpm_xl-0.1.0/README.md +172 -0
- dpm_xl-0.1.0/pyproject.toml +126 -0
- dpm_xl-0.1.0/pyproject.toml.orig +82 -0
- dpm_xl-0.1.0/src/dpmxl/__init__.py +83 -0
- dpm_xl-0.1.0/src/dpmxl/ast.py +413 -0
- dpm_xl-0.1.0/src/dpmxl/cli.py +161 -0
- dpm_xl-0.1.0/src/dpmxl/columnar.py +1440 -0
- dpm_xl-0.1.0/src/dpmxl/evaluator.py +1010 -0
- dpm_xl-0.1.0/src/dpmxl/explain.py +567 -0
- dpm_xl-0.1.0/src/dpmxl/parser.py +729 -0
- dpm_xl-0.1.0/src/dpmxl/provider.py +353 -0
- dpm_xl-0.1.0/src/dpmxl/py.typed +0 -0
- dpm_xl-0.1.0/src/dpmxl/recordset.py +350 -0
- dpm_xl-0.1.0/src/dpmxl/references.py +78 -0
- dpm_xl-0.1.0/src/dpmxl/values.py +345 -0
dpm_xl-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Luuk Hopman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
dpm_xl-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dpm-xl
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Parser and evaluator for the EBA DPM-XL validation-rule language
|
|
5
|
+
Keywords: DPM,DPM-XL,EBA,XBRL,validation rules
|
|
6
|
+
Author: Luuk Hopman
|
|
7
|
+
Author-email: Luuk Hopman <github@luukhopman.nl>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Topic :: Office/Business :: Financial
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Dist: polars>=2.0,<3 ; extra == 'columnar'
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Project-URL: Homepage, https://github.com/luukhopman/dpm-xl
|
|
18
|
+
Project-URL: Source, https://github.com/luukhopman/dpm-xl
|
|
19
|
+
Project-URL: Issues, https://github.com/luukhopman/dpm-xl/issues
|
|
20
|
+
Provides-Extra: columnar
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# dpm-xl
|
|
24
|
+
|
|
25
|
+
A Python parser and evaluator for **DPM-XL**, the expression language of EBA
|
|
26
|
+
validation rules and similar checks. The core has no dependencies; the
|
|
27
|
+
optional `columnar` extra adds a polars evaluator for reports with millions of
|
|
28
|
+
open-table rows.
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from dpmxl import evaluate_rule, explain
|
|
32
|
+
|
|
33
|
+
expression = (
|
|
34
|
+
"with {tC_01.00, c*, default: 0, interval: true}: {r0010} = {r0020} + {r0030}"
|
|
35
|
+
)
|
|
36
|
+
result = evaluate_rule(expression, provider, precondition="{v_C_01.00}")
|
|
37
|
+
for outcome in result.failures:
|
|
38
|
+
print(explain(result.evaluation, expression, outcome.key).summary)
|
|
39
|
+
# 1,204 should equal 1,211 (1,180 + 31): difference 7, at least 5.5 given the
|
|
40
|
+
# reported precision
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The package does not read report files and knows nothing about a particular
|
|
44
|
+
DPM release. Data comes through a small provider protocol (see below), so the
|
|
45
|
+
same evaluator works against an xBRL-CSV upload, a database, or test fixtures.
|
|
46
|
+
|
|
47
|
+
dpm-xl is an independent project, not affiliated with or endorsed by the
|
|
48
|
+
European Banking Authority.
|
|
49
|
+
|
|
50
|
+
## Modules
|
|
51
|
+
|
|
52
|
+
| Module | Purpose |
|
|
53
|
+
| --- | --- |
|
|
54
|
+
| `dpmxl.parser` | Tokenizer and recursive-descent parser. Nodes keep source spans. |
|
|
55
|
+
| `dpmxl.ast` | Expression tree, selections, axis selectors and item literals. |
|
|
56
|
+
| `dpmxl.values` | `Number` with interval tolerance, comparison and formatting. |
|
|
57
|
+
| `dpmxl.recordset` | Keyed record sets and the natural join used by every operator. |
|
|
58
|
+
| `dpmxl.evaluator` | Evaluation, preconditions and per-record outcomes. |
|
|
59
|
+
| `dpmxl.explain` | Fills a failing record's formula with operand values. |
|
|
60
|
+
| `dpmxl.references` | Static selections (with `with` context applied) and `{v_…}` templates. |
|
|
61
|
+
| `dpmxl.provider` | `DataProvider` protocol and the in-memory `MemoryProvider`. |
|
|
62
|
+
| `dpmxl.columnar` | The same semantics on polars frames (optional `columnar` extra). |
|
|
63
|
+
| `dpmxl.cli` | `dpm-xl check` and `dpm-xl eval`. |
|
|
64
|
+
|
|
65
|
+
## Supported language
|
|
66
|
+
|
|
67
|
+
- `with {context} [where …]: body`, nested contexts replacing inherited parts.
|
|
68
|
+
- Selections `{tTABLE, r…, c…, s…, default: v, interval: b}` with single codes,
|
|
69
|
+
inclusive ranges (`r0010-0050`), lists (`(c0010, c0030)`) and wildcards.
|
|
70
|
+
- `{v_TABLE}` filing-indicator tests and `{vName}` report parameters.
|
|
71
|
+
- Literals: numbers, `4.5%`, `"text"`/`'text'`, `#2026-12-31#`, `true`/`false`,
|
|
72
|
+
`null`, items `[eba_qEC:qx01]`.
|
|
73
|
+
- Operators `+ - * /`, `= != <> < <= > >=`, `in {…}`, `not`, `and`, `or`, `xor`,
|
|
74
|
+
`if … then … [else …] endif`.
|
|
75
|
+
- Aggregates `sum`, `count`, `avg`, `min`, `max`, `median` (and the
|
|
76
|
+
specification's `min_aggr`/`max_aggr` spellings) with `group by`;
|
|
77
|
+
scalar `abs`, `isnull`, `nvl`, `len`, `match`, `min`/`max` with several
|
|
78
|
+
operands, `exp`, `ln`, `sqrt`, `power`; `filter(data, condition)`.
|
|
79
|
+
- Clauses `[where …]`, `[get key]`, `[sub key = value]`, `[rename a to b]`.
|
|
80
|
+
|
|
81
|
+
`time_shift` needs the previous reporting period and is reported as
|
|
82
|
+
`unsupported`. Every distinct EBA rule and precondition expression in releases
|
|
83
|
+
3.4–4.4 parses, except two source expressions with typos (`trueendif`,
|
|
84
|
+
`ornot`), which are reported as syntax errors.
|
|
85
|
+
|
|
86
|
+
## Semantics
|
|
87
|
+
|
|
88
|
+
- **Records and keys.** A selection yields one record per data point. Its key
|
|
89
|
+
holds every closed axis that selects more than one coordinate (`r`, `c`, `s`;
|
|
90
|
+
an axis that is not written counts when the table has several coordinates on
|
|
91
|
+
it) plus open key components such as `qEBB`. Binary operators natural-join on
|
|
92
|
+
shared keys; a scalar broadcasts. Operands with unrelated keys are an error.
|
|
93
|
+
- **Missing facts** take the selection's `default`; with no default they are
|
|
94
|
+
null. Providers must therefore return every defined cell of a closed
|
|
95
|
+
selection, reported or not.
|
|
96
|
+
- **Interval arithmetic.** With `interval: true`, numbers follow the Eurofiling
|
|
97
|
+
interval arithmetic functions (`iaf:*`, version 1.1.1) that EBA validation
|
|
98
|
+
rules are based on. A fact reported with `decimals = d` stands for
|
|
99
|
+
`value ± 0.5·10^-d` (`INF` is exact; constants and defaults are exact). Sums
|
|
100
|
+
and differences add thresholds; products use `|A·δB| + |B·δA| + δA·δB`;
|
|
101
|
+
quotients take the largest deviation of the four endpoint quotients
|
|
102
|
+
(skipping a zero divisor); `abs` and negation keep the threshold; `min` and
|
|
103
|
+
`max` keep the chosen value's threshold. `a = b` holds when
|
|
104
|
+
`|a − b| ≤ δa + δb` (the intervals overlap), `a ≤ b` when
|
|
105
|
+
`a − b ≤ δa + δb`, `a ≥ b` when `a ≥ b − (δa + δb)`, and likewise for the
|
|
106
|
+
strict comparisons. With `interval: false` facts are exact.
|
|
107
|
+
Explanations report each side's interval and the *gap*: how far apart the
|
|
108
|
+
closest values within those intervals still are.
|
|
109
|
+
- **Null.** Null propagates through arithmetic, comparisons and functions
|
|
110
|
+
(except `isnull` and `nvl`). `and`/`or`/`not` use three-valued (Kleene)
|
|
111
|
+
logic. A record whose result is null is "not evaluated", never a failure.
|
|
112
|
+
Division by zero is a run-time error, as the specification requires.
|
|
113
|
+
- **Conditionals.** `if` takes the `else` branch when the condition is null;
|
|
114
|
+
without `else`, a Boolean `then` yields true where the condition is false or
|
|
115
|
+
null. The result has the condition's record structure; branch keys the
|
|
116
|
+
condition lacks widen it, so each condition record applies to every branch
|
|
117
|
+
record sharing its keys (the columnar evaluator hands that case to the
|
|
118
|
+
reference evaluator).
|
|
119
|
+
- **Aggregates** over no records are null, except `count`, which is 0;
|
|
120
|
+
`count` counts non-null values. `max` and `min` also order dates, text and
|
|
121
|
+
items by their text (ISO dates in date order, items by category and code),
|
|
122
|
+
as presence checks expect; Booleans, and numbers mixed with other
|
|
123
|
+
values in one group, are an error.
|
|
124
|
+
- **Clauses.** `[get key]` omits records whose fact is null; a `where` or
|
|
125
|
+
`sub` inside a `with` body overrides the context's clause on the same key.
|
|
126
|
+
Booleans order `false < true`.
|
|
127
|
+
- **Key names** in `where`, `group by`, `sub` and `get` match with or without
|
|
128
|
+
the DPM 2.0 `q` prefix and ignore case (`EBB` finds `qEBB`).
|
|
129
|
+
- **Items** compare by code and category, ignoring the `eba_` prefix and a
|
|
130
|
+
category missing on either side.
|
|
131
|
+
- **Preconditions** make a rule `not_applicable` unless they yield at least one
|
|
132
|
+
true record.
|
|
133
|
+
|
|
134
|
+
A `RuleResult` has status `passed`, `failed`, `not_applicable`, `error`
|
|
135
|
+
(syntax, unknown table, type mismatch) or `unsupported`, plus one `Outcome`
|
|
136
|
+
per record with the data points it used.
|
|
137
|
+
|
|
138
|
+
These semantics follow the EBA *DPM-XL language specification v0.9*
|
|
139
|
+
(operators and data model documents) and the Eurofiling interval arithmetic
|
|
140
|
+
functions. Where the specification is ambiguous or contradicts itself (sum of
|
|
141
|
+
no records, `count` over nulls, key-name aliases, the radius formulas for
|
|
142
|
+
products and quotients), the choice above is the one that avoids false
|
|
143
|
+
failures.
|
|
144
|
+
|
|
145
|
+
## Columnar evaluation
|
|
146
|
+
|
|
147
|
+
`dpmxl.columnar.evaluate_rule_columnar(expression, provider)` evaluates a rule
|
|
148
|
+
on polars frames from a `FrameProvider` (`select_frame(query)` returns one row
|
|
149
|
+
per data point with `k:<key>` columns and the `num`, `thr`, `txt`, `flag` value
|
|
150
|
+
columns). Values are exact `Decimal(38, 12)`, keys are joined like
|
|
151
|
+
`recordset.combine`, and a `FrameCache` shares selections across rules within
|
|
152
|
+
a row budget. Anything the module does not implement, including arithmetic
|
|
153
|
+
that would overflow, raises `ColumnarUnsupported` so the caller falls back to
|
|
154
|
+
the reference evaluator. `tests/test_columnar.py` checks both evaluators agree
|
|
155
|
+
case by case. `PointsFrameProvider` adapts any `DataProvider`.
|
|
156
|
+
|
|
157
|
+
## Provider contract
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
class DataProvider(Protocol):
|
|
161
|
+
def select(self, query: CellQuery) -> Sequence[DataPoint]: ...
|
|
162
|
+
def table_reported(self, table: str) -> bool | None: ...
|
|
163
|
+
def parameter(self, name: str) -> Value: ...
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
`select` resolves the table and axis selectors (`axis_matches` and
|
|
167
|
+
`normalize_code` implement range and code matching) and returns all defined
|
|
168
|
+
cells of the closed axes with `value=None` where nothing was reported. For an
|
|
169
|
+
open table it returns one point per reported key combination, with the key
|
|
170
|
+
components in `DataPoint.keys`. `DataPoint.ref` is an opaque identity that
|
|
171
|
+
comes back in outcomes, so a caller can highlight cells.
|
|
172
|
+
|
|
173
|
+
## CLI
|
|
174
|
+
|
|
175
|
+
```bash
|
|
176
|
+
dpm-xl check rules.json # parse every expression/precondition in a rule list
|
|
177
|
+
dpm-xl check "with {tC_01.00}: {r0010} >= 0" --references
|
|
178
|
+
dpm-xl eval "with {tC_01.00, c0010, default: 0, interval: true}: {r0010} = {r0020} + {r0030}" data.json
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
`data.json` holds `tables` (`code`, `rows`, `columns`, `sheets`, `open_keys`),
|
|
182
|
+
`facts` (`table`, `row`, `column`, `sheet`, `keys`, `value`, `decimals`,
|
|
183
|
+
`type`), `reported` and `parameters`.
|
|
184
|
+
|
|
185
|
+
## Development
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
uv sync
|
|
189
|
+
uv run pytest
|
|
190
|
+
uv run ruff check . && uv run ruff format --check . && uv run ty check
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
`tests/data/eba_expressions.txt` is a sample of about 300 EBA rule
|
|
194
|
+
expressions covering every construct above. Every line must parse.
|
dpm_xl-0.1.0/README.md
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# dpm-xl
|
|
2
|
+
|
|
3
|
+
A Python parser and evaluator for **DPM-XL**, the expression language of EBA
|
|
4
|
+
validation rules and similar checks. The core has no dependencies; the
|
|
5
|
+
optional `columnar` extra adds a polars evaluator for reports with millions of
|
|
6
|
+
open-table rows.
|
|
7
|
+
|
|
8
|
+
```python
|
|
9
|
+
from dpmxl import evaluate_rule, explain
|
|
10
|
+
|
|
11
|
+
expression = (
|
|
12
|
+
"with {tC_01.00, c*, default: 0, interval: true}: {r0010} = {r0020} + {r0030}"
|
|
13
|
+
)
|
|
14
|
+
result = evaluate_rule(expression, provider, precondition="{v_C_01.00}")
|
|
15
|
+
for outcome in result.failures:
|
|
16
|
+
print(explain(result.evaluation, expression, outcome.key).summary)
|
|
17
|
+
# 1,204 should equal 1,211 (1,180 + 31): difference 7, at least 5.5 given the
|
|
18
|
+
# reported precision
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
The package does not read report files and knows nothing about a particular
|
|
22
|
+
DPM release. Data comes through a small provider protocol (see below), so the
|
|
23
|
+
same evaluator works against an xBRL-CSV upload, a database, or test fixtures.
|
|
24
|
+
|
|
25
|
+
dpm-xl is an independent project, not affiliated with or endorsed by the
|
|
26
|
+
European Banking Authority.
|
|
27
|
+
|
|
28
|
+
## Modules
|
|
29
|
+
|
|
30
|
+
| Module | Purpose |
|
|
31
|
+
| --- | --- |
|
|
32
|
+
| `dpmxl.parser` | Tokenizer and recursive-descent parser. Nodes keep source spans. |
|
|
33
|
+
| `dpmxl.ast` | Expression tree, selections, axis selectors and item literals. |
|
|
34
|
+
| `dpmxl.values` | `Number` with interval tolerance, comparison and formatting. |
|
|
35
|
+
| `dpmxl.recordset` | Keyed record sets and the natural join used by every operator. |
|
|
36
|
+
| `dpmxl.evaluator` | Evaluation, preconditions and per-record outcomes. |
|
|
37
|
+
| `dpmxl.explain` | Fills a failing record's formula with operand values. |
|
|
38
|
+
| `dpmxl.references` | Static selections (with `with` context applied) and `{v_…}` templates. |
|
|
39
|
+
| `dpmxl.provider` | `DataProvider` protocol and the in-memory `MemoryProvider`. |
|
|
40
|
+
| `dpmxl.columnar` | The same semantics on polars frames (optional `columnar` extra). |
|
|
41
|
+
| `dpmxl.cli` | `dpm-xl check` and `dpm-xl eval`. |
|
|
42
|
+
|
|
43
|
+
## Supported language
|
|
44
|
+
|
|
45
|
+
- `with {context} [where …]: body`, nested contexts replacing inherited parts.
|
|
46
|
+
- Selections `{tTABLE, r…, c…, s…, default: v, interval: b}` with single codes,
|
|
47
|
+
inclusive ranges (`r0010-0050`), lists (`(c0010, c0030)`) and wildcards.
|
|
48
|
+
- `{v_TABLE}` filing-indicator tests and `{vName}` report parameters.
|
|
49
|
+
- Literals: numbers, `4.5%`, `"text"`/`'text'`, `#2026-12-31#`, `true`/`false`,
|
|
50
|
+
`null`, items `[eba_qEC:qx01]`.
|
|
51
|
+
- Operators `+ - * /`, `= != <> < <= > >=`, `in {…}`, `not`, `and`, `or`, `xor`,
|
|
52
|
+
`if … then … [else …] endif`.
|
|
53
|
+
- Aggregates `sum`, `count`, `avg`, `min`, `max`, `median` (and the
|
|
54
|
+
specification's `min_aggr`/`max_aggr` spellings) with `group by`;
|
|
55
|
+
scalar `abs`, `isnull`, `nvl`, `len`, `match`, `min`/`max` with several
|
|
56
|
+
operands, `exp`, `ln`, `sqrt`, `power`; `filter(data, condition)`.
|
|
57
|
+
- Clauses `[where …]`, `[get key]`, `[sub key = value]`, `[rename a to b]`.
|
|
58
|
+
|
|
59
|
+
`time_shift` needs the previous reporting period and is reported as
|
|
60
|
+
`unsupported`. Every distinct EBA rule and precondition expression in releases
|
|
61
|
+
3.4–4.4 parses, except two source expressions with typos (`trueendif`,
|
|
62
|
+
`ornot`), which are reported as syntax errors.
|
|
63
|
+
|
|
64
|
+
## Semantics
|
|
65
|
+
|
|
66
|
+
- **Records and keys.** A selection yields one record per data point. Its key
|
|
67
|
+
holds every closed axis that selects more than one coordinate (`r`, `c`, `s`;
|
|
68
|
+
an axis that is not written counts when the table has several coordinates on
|
|
69
|
+
it) plus open key components such as `qEBB`. Binary operators natural-join on
|
|
70
|
+
shared keys; a scalar broadcasts. Operands with unrelated keys are an error.
|
|
71
|
+
- **Missing facts** take the selection's `default`; with no default they are
|
|
72
|
+
null. Providers must therefore return every defined cell of a closed
|
|
73
|
+
selection, reported or not.
|
|
74
|
+
- **Interval arithmetic.** With `interval: true`, numbers follow the Eurofiling
|
|
75
|
+
interval arithmetic functions (`iaf:*`, version 1.1.1) that EBA validation
|
|
76
|
+
rules are based on. A fact reported with `decimals = d` stands for
|
|
77
|
+
`value ± 0.5·10^-d` (`INF` is exact; constants and defaults are exact). Sums
|
|
78
|
+
and differences add thresholds; products use `|A·δB| + |B·δA| + δA·δB`;
|
|
79
|
+
quotients take the largest deviation of the four endpoint quotients
|
|
80
|
+
(skipping a zero divisor); `abs` and negation keep the threshold; `min` and
|
|
81
|
+
`max` keep the chosen value's threshold. `a = b` holds when
|
|
82
|
+
`|a − b| ≤ δa + δb` (the intervals overlap), `a ≤ b` when
|
|
83
|
+
`a − b ≤ δa + δb`, `a ≥ b` when `a ≥ b − (δa + δb)`, and likewise for the
|
|
84
|
+
strict comparisons. With `interval: false` facts are exact.
|
|
85
|
+
Explanations report each side's interval and the *gap*: how far apart the
|
|
86
|
+
closest values within those intervals still are.
|
|
87
|
+
- **Null.** Null propagates through arithmetic, comparisons and functions
|
|
88
|
+
(except `isnull` and `nvl`). `and`/`or`/`not` use three-valued (Kleene)
|
|
89
|
+
logic. A record whose result is null is "not evaluated", never a failure.
|
|
90
|
+
Division by zero is a run-time error, as the specification requires.
|
|
91
|
+
- **Conditionals.** `if` takes the `else` branch when the condition is null;
|
|
92
|
+
without `else`, a Boolean `then` yields true where the condition is false or
|
|
93
|
+
null. The result has the condition's record structure; branch keys the
|
|
94
|
+
condition lacks widen it, so each condition record applies to every branch
|
|
95
|
+
record sharing its keys (the columnar evaluator hands that case to the
|
|
96
|
+
reference evaluator).
|
|
97
|
+
- **Aggregates** over no records are null, except `count`, which is 0;
|
|
98
|
+
`count` counts non-null values. `max` and `min` also order dates, text and
|
|
99
|
+
items by their text (ISO dates in date order, items by category and code),
|
|
100
|
+
as presence checks expect; Booleans, and numbers mixed with other
|
|
101
|
+
values in one group, are an error.
|
|
102
|
+
- **Clauses.** `[get key]` omits records whose fact is null; a `where` or
|
|
103
|
+
`sub` inside a `with` body overrides the context's clause on the same key.
|
|
104
|
+
Booleans order `false < true`.
|
|
105
|
+
- **Key names** in `where`, `group by`, `sub` and `get` match with or without
|
|
106
|
+
the DPM 2.0 `q` prefix and ignore case (`EBB` finds `qEBB`).
|
|
107
|
+
- **Items** compare by code and category, ignoring the `eba_` prefix and a
|
|
108
|
+
category missing on either side.
|
|
109
|
+
- **Preconditions** make a rule `not_applicable` unless they yield at least one
|
|
110
|
+
true record.
|
|
111
|
+
|
|
112
|
+
A `RuleResult` has status `passed`, `failed`, `not_applicable`, `error`
|
|
113
|
+
(syntax, unknown table, type mismatch) or `unsupported`, plus one `Outcome`
|
|
114
|
+
per record with the data points it used.
|
|
115
|
+
|
|
116
|
+
These semantics follow the EBA *DPM-XL language specification v0.9*
|
|
117
|
+
(operators and data model documents) and the Eurofiling interval arithmetic
|
|
118
|
+
functions. Where the specification is ambiguous or contradicts itself (sum of
|
|
119
|
+
no records, `count` over nulls, key-name aliases, the radius formulas for
|
|
120
|
+
products and quotients), the choice above is the one that avoids false
|
|
121
|
+
failures.
|
|
122
|
+
|
|
123
|
+
## Columnar evaluation
|
|
124
|
+
|
|
125
|
+
`dpmxl.columnar.evaluate_rule_columnar(expression, provider)` evaluates a rule
|
|
126
|
+
on polars frames from a `FrameProvider` (`select_frame(query)` returns one row
|
|
127
|
+
per data point with `k:<key>` columns and the `num`, `thr`, `txt`, `flag` value
|
|
128
|
+
columns). Values are exact `Decimal(38, 12)`, keys are joined like
|
|
129
|
+
`recordset.combine`, and a `FrameCache` shares selections across rules within
|
|
130
|
+
a row budget. Anything the module does not implement, including arithmetic
|
|
131
|
+
that would overflow, raises `ColumnarUnsupported` so the caller falls back to
|
|
132
|
+
the reference evaluator. `tests/test_columnar.py` checks both evaluators agree
|
|
133
|
+
case by case. `PointsFrameProvider` adapts any `DataProvider`.
|
|
134
|
+
|
|
135
|
+
## Provider contract
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
class DataProvider(Protocol):
|
|
139
|
+
def select(self, query: CellQuery) -> Sequence[DataPoint]: ...
|
|
140
|
+
def table_reported(self, table: str) -> bool | None: ...
|
|
141
|
+
def parameter(self, name: str) -> Value: ...
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
`select` resolves the table and axis selectors (`axis_matches` and
|
|
145
|
+
`normalize_code` implement range and code matching) and returns all defined
|
|
146
|
+
cells of the closed axes with `value=None` where nothing was reported. For an
|
|
147
|
+
open table it returns one point per reported key combination, with the key
|
|
148
|
+
components in `DataPoint.keys`. `DataPoint.ref` is an opaque identity that
|
|
149
|
+
comes back in outcomes, so a caller can highlight cells.
|
|
150
|
+
|
|
151
|
+
## CLI
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
dpm-xl check rules.json # parse every expression/precondition in a rule list
|
|
155
|
+
dpm-xl check "with {tC_01.00}: {r0010} >= 0" --references
|
|
156
|
+
dpm-xl eval "with {tC_01.00, c0010, default: 0, interval: true}: {r0010} = {r0020} + {r0030}" data.json
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
`data.json` holds `tables` (`code`, `rows`, `columns`, `sheets`, `open_keys`),
|
|
160
|
+
`facts` (`table`, `row`, `column`, `sheet`, `keys`, `value`, `decimals`,
|
|
161
|
+
`type`), `reported` and `parameters`.
|
|
162
|
+
|
|
163
|
+
## Development
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
uv sync
|
|
167
|
+
uv run pytest
|
|
168
|
+
uv run ruff check . && uv run ruff format --check . && uv run ty check
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
`tests/data/eba_expressions.txt` is a sample of about 300 EBA rule
|
|
172
|
+
expressions covering every construct above. Every line must parse.
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.12.5,<0.13"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "dpm-xl"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Parser and evaluator for the EBA DPM-XL validation-rule language"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
keywords = [
|
|
14
|
+
"DPM",
|
|
15
|
+
"DPM-XL",
|
|
16
|
+
"EBA",
|
|
17
|
+
"XBRL",
|
|
18
|
+
"validation rules",
|
|
19
|
+
]
|
|
20
|
+
classifiers = [
|
|
21
|
+
"Development Status :: 3 - Alpha",
|
|
22
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
23
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
24
|
+
"Topic :: Office/Business :: Financial",
|
|
25
|
+
"Typing :: Typed",
|
|
26
|
+
]
|
|
27
|
+
dependencies = []
|
|
28
|
+
|
|
29
|
+
[[project.authors]]
|
|
30
|
+
name = "Luuk Hopman"
|
|
31
|
+
email = "github@luukhopman.nl"
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/luukhopman/dpm-xl"
|
|
35
|
+
Source = "https://github.com/luukhopman/dpm-xl"
|
|
36
|
+
Issues = "https://github.com/luukhopman/dpm-xl/issues"
|
|
37
|
+
|
|
38
|
+
[project.scripts]
|
|
39
|
+
dpm-xl = "dpmxl.cli:main"
|
|
40
|
+
|
|
41
|
+
[project.optional-dependencies]
|
|
42
|
+
columnar = ["polars>=2.0,<3"]
|
|
43
|
+
|
|
44
|
+
[dependency-groups]
|
|
45
|
+
test = [
|
|
46
|
+
"pytest",
|
|
47
|
+
"pytest-cov>=7.1,<8",
|
|
48
|
+
]
|
|
49
|
+
lint = [
|
|
50
|
+
"ruff",
|
|
51
|
+
"ty",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
[[dependency-groups.dev]]
|
|
55
|
+
include-group = "test"
|
|
56
|
+
|
|
57
|
+
[[dependency-groups.dev]]
|
|
58
|
+
include-group = "lint"
|
|
59
|
+
|
|
60
|
+
[[tool.uv.index]]
|
|
61
|
+
url = "https://pypi.org/simple"
|
|
62
|
+
default = true
|
|
63
|
+
|
|
64
|
+
[tool.uv.build-backend]
|
|
65
|
+
module-name = "dpmxl"
|
|
66
|
+
module-root = "src"
|
|
67
|
+
|
|
68
|
+
[tool.ruff]
|
|
69
|
+
target-version = "py311"
|
|
70
|
+
line-length = 88
|
|
71
|
+
|
|
72
|
+
[tool.ruff.lint]
|
|
73
|
+
preview = true
|
|
74
|
+
select = [
|
|
75
|
+
"B",
|
|
76
|
+
"C4",
|
|
77
|
+
"D",
|
|
78
|
+
"DOC",
|
|
79
|
+
"E",
|
|
80
|
+
"ERA",
|
|
81
|
+
"F",
|
|
82
|
+
"I",
|
|
83
|
+
"PIE",
|
|
84
|
+
"PTH",
|
|
85
|
+
"RET",
|
|
86
|
+
"RSE",
|
|
87
|
+
"RUF",
|
|
88
|
+
"S",
|
|
89
|
+
"SIM",
|
|
90
|
+
"UP",
|
|
91
|
+
"W",
|
|
92
|
+
]
|
|
93
|
+
ignore = [
|
|
94
|
+
"incorrect-blank-line-before-class",
|
|
95
|
+
"multi-line-summary-second-line",
|
|
96
|
+
"docstring-extraneous-exception",
|
|
97
|
+
"assert",
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
[tool.ruff.lint.per-file-ignores]
|
|
101
|
+
"tests/*" = [
|
|
102
|
+
"D",
|
|
103
|
+
"S",
|
|
104
|
+
"line-too-long",
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
[tool.ruff.lint.pydocstyle]
|
|
108
|
+
convention = "google"
|
|
109
|
+
|
|
110
|
+
[tool.ty.src]
|
|
111
|
+
include = [
|
|
112
|
+
"src",
|
|
113
|
+
"tests",
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
[tool.pytest.ini_options]
|
|
117
|
+
testpaths = ["tests"]
|
|
118
|
+
addopts = [
|
|
119
|
+
"--strict-config",
|
|
120
|
+
"--strict-markers",
|
|
121
|
+
"--import-mode=importlib",
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
[tool.coverage.run]
|
|
125
|
+
source = ["dpmxl"]
|
|
126
|
+
branch = true
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.12.5,<0.13"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "dpm-xl"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Parser and evaluator for the EBA DPM-XL validation-rule language"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
authors = [{ name = "Luuk Hopman", email = "github@luukhopman.nl" }]
|
|
14
|
+
keywords = ["DPM", "DPM-XL", "EBA", "XBRL", "validation rules"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
18
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
19
|
+
"Topic :: Office/Business :: Financial",
|
|
20
|
+
"Typing :: Typed",
|
|
21
|
+
]
|
|
22
|
+
dependencies = []
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/luukhopman/dpm-xl"
|
|
26
|
+
Source = "https://github.com/luukhopman/dpm-xl"
|
|
27
|
+
Issues = "https://github.com/luukhopman/dpm-xl/issues"
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
dpm-xl = "dpmxl.cli:main"
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
columnar = [
|
|
34
|
+
"polars>=2.0,<3",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[dependency-groups]
|
|
38
|
+
test = ["pytest", "pytest-cov>=7.1,<8"]
|
|
39
|
+
lint = ["ruff", "ty"]
|
|
40
|
+
dev = [{ include-group = "test" }, { include-group = "lint" }]
|
|
41
|
+
|
|
42
|
+
[[tool.uv.index]]
|
|
43
|
+
url = "https://pypi.org/simple"
|
|
44
|
+
default = true
|
|
45
|
+
|
|
46
|
+
[tool.ruff]
|
|
47
|
+
target-version = "py311"
|
|
48
|
+
line-length = 88
|
|
49
|
+
|
|
50
|
+
[tool.ruff.lint]
|
|
51
|
+
preview = true
|
|
52
|
+
select = [
|
|
53
|
+
"B", "C4", "D", "DOC", "E", "ERA", "F", "I", "PIE", "PTH", "RET", "RSE",
|
|
54
|
+
"RUF", "S", "SIM", "UP", "W",
|
|
55
|
+
]
|
|
56
|
+
ignore = [
|
|
57
|
+
"incorrect-blank-line-before-class",
|
|
58
|
+
"multi-line-summary-second-line",
|
|
59
|
+
"docstring-extraneous-exception",
|
|
60
|
+
"assert",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
[tool.ruff.lint.per-file-ignores]
|
|
64
|
+
"tests/*" = ["D", "S", "line-too-long"]
|
|
65
|
+
|
|
66
|
+
[tool.ruff.lint.pydocstyle]
|
|
67
|
+
convention = "google"
|
|
68
|
+
|
|
69
|
+
[tool.ty.src]
|
|
70
|
+
include = ["src", "tests"]
|
|
71
|
+
|
|
72
|
+
[tool.uv.build-backend]
|
|
73
|
+
module-name = "dpmxl"
|
|
74
|
+
module-root = "src"
|
|
75
|
+
|
|
76
|
+
[tool.pytest.ini_options]
|
|
77
|
+
testpaths = ["tests"]
|
|
78
|
+
addopts = ["--strict-config", "--strict-markers", "--import-mode=importlib"]
|
|
79
|
+
|
|
80
|
+
[tool.coverage.run]
|
|
81
|
+
source = ["dpmxl"]
|
|
82
|
+
branch = true
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Parse and evaluate EBA DPM-XL validation expressions.
|
|
2
|
+
|
|
3
|
+
Typical use::
|
|
4
|
+
|
|
5
|
+
from dpmxl import evaluate_rule, explain, parse
|
|
6
|
+
|
|
7
|
+
result = evaluate_rule(expression, provider, precondition=precondition)
|
|
8
|
+
for outcome in result.failures:
|
|
9
|
+
print(explain(result.evaluation, expression, outcome.key).summary)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from .ast import AxisSelector, Item, Node, SelectionSpec
|
|
13
|
+
from .evaluator import (
|
|
14
|
+
Evaluation,
|
|
15
|
+
Evaluator,
|
|
16
|
+
Outcome,
|
|
17
|
+
RuleResult,
|
|
18
|
+
SelectionCache,
|
|
19
|
+
evaluate_rule,
|
|
20
|
+
parse_cached,
|
|
21
|
+
)
|
|
22
|
+
from .explain import Explanation, Operand, explain
|
|
23
|
+
from .parser import DpmXlSyntaxError, parse, parse_selection
|
|
24
|
+
from .provider import (
|
|
25
|
+
CellQuery,
|
|
26
|
+
DataPoint,
|
|
27
|
+
DataProvider,
|
|
28
|
+
MemoryProvider,
|
|
29
|
+
TableLayout,
|
|
30
|
+
axis_matcher,
|
|
31
|
+
axis_matches,
|
|
32
|
+
normalize_code,
|
|
33
|
+
)
|
|
34
|
+
from .recordset import (
|
|
35
|
+
EvaluationError,
|
|
36
|
+
Recordset,
|
|
37
|
+
UnsupportedError,
|
|
38
|
+
iter_points,
|
|
39
|
+
unique_points,
|
|
40
|
+
)
|
|
41
|
+
from .references import Reference, references, reported_templates, tables
|
|
42
|
+
from .values import Number, Value, compare, format_value
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"AxisSelector",
|
|
46
|
+
"CellQuery",
|
|
47
|
+
"DataPoint",
|
|
48
|
+
"DataProvider",
|
|
49
|
+
"DpmXlSyntaxError",
|
|
50
|
+
"Evaluation",
|
|
51
|
+
"EvaluationError",
|
|
52
|
+
"Evaluator",
|
|
53
|
+
"Explanation",
|
|
54
|
+
"Item",
|
|
55
|
+
"MemoryProvider",
|
|
56
|
+
"Node",
|
|
57
|
+
"Number",
|
|
58
|
+
"Operand",
|
|
59
|
+
"Outcome",
|
|
60
|
+
"Recordset",
|
|
61
|
+
"Reference",
|
|
62
|
+
"RuleResult",
|
|
63
|
+
"SelectionCache",
|
|
64
|
+
"SelectionSpec",
|
|
65
|
+
"TableLayout",
|
|
66
|
+
"UnsupportedError",
|
|
67
|
+
"Value",
|
|
68
|
+
"axis_matcher",
|
|
69
|
+
"axis_matches",
|
|
70
|
+
"compare",
|
|
71
|
+
"evaluate_rule",
|
|
72
|
+
"explain",
|
|
73
|
+
"format_value",
|
|
74
|
+
"iter_points",
|
|
75
|
+
"normalize_code",
|
|
76
|
+
"parse",
|
|
77
|
+
"parse_cached",
|
|
78
|
+
"parse_selection",
|
|
79
|
+
"references",
|
|
80
|
+
"reported_templates",
|
|
81
|
+
"tables",
|
|
82
|
+
"unique_points",
|
|
83
|
+
]
|