dpmcore 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dpmcore-0.2.0 → dpmcore-0.2.1}/PKG-INFO +52 -24
- {dpmcore-0.2.0 → dpmcore-0.2.1}/README.md +33 -7
- {dpmcore-0.2.0 → dpmcore-0.2.1}/pyproject.toml +41 -28
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/__init__.py +1 -1
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/cli/main.py +144 -15
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/constructor.py +7 -4
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/ml_generation.py +37 -27
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/module_dependencies.py +1 -1
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/nodes.py +18 -4
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/dpm_xlLexer.g4 +61 -37
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/dpm_xlParser.g4 +24 -14
- dpmcore-0.2.1/src/dpmcore/dpm_xl/grammar/generated/dpm_xlLexer.interp +574 -0
- dpmcore-0.2.1/src/dpmcore/dpm_xl/grammar/generated/dpm_xlLexer.py +1368 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlLexer.tokens +36 -35
- dpmcore-0.2.1/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParser.interp +339 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParser.py +671 -706
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParser.tokens +36 -35
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParserListener.py +19 -19
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParserVisitor.py +11 -11
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/model_queries.py +702 -272
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/aggregate.py +45 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/semantic_analyzer.py +12 -3
- dpmcore-0.2.1/src/dpmcore/dpm_xl/utils/db_ast.py +914 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/utils/serialization.py +2 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/errors.py +22 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/_open_keys.py +49 -17
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/ast_generator.py +1257 -192
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_export/exporter.py +14 -4
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_export/visitors.py +152 -57
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/data_dictionary.py +2 -4
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/database_update.py +10 -2
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/dpm_xl.py +2 -1
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/layout_exporter/excel_writer.py +5 -2
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/layout_exporter/models.py +5 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/layout_exporter/queries.py +89 -36
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/meili_build.py +5 -3
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/scope_calculator.py +458 -35
- dpmcore-0.2.1/src/dpmcore/services/script_fixups.py +408 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/semantic.py +11 -3
- dpmcore-0.2.0/src/dpmcore/dpm_xl/grammar/generated/dpm_xlLexer.interp +0 -557
- dpmcore-0.2.0/src/dpmcore/dpm_xl/grammar/generated/dpm_xlLexer.py +0 -1314
- dpmcore-0.2.0/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParser.interp +0 -337
- {dpmcore-0.2.0 → dpmcore-0.2.1}/LICENSE +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/cli/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/connection.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/data/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/data/module_schema_mapping.csv +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/admin.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/apps.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/models/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/models/auxiliary.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/models/glossary.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/models/infrastructure.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/models/operations.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/models/packaging.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/models/rendering.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/models/variables.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/routers.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/django/urls.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/domain_membership.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/module_analyzer.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/operands.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/template.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/visitor.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/ast/where_clause.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/generated/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/grammar/generated/listeners.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/arithmetic.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/base.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/boolean.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/clause.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/comparison.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/conditional.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/string.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/operators/time.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/symbols.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/types/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/types/promotion.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/types/scalar.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/types/time.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/utils/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/utils/data_handlers.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/utils/filters.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/utils/operands_mapping.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/utils/operator_mapping.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/utils/range_resolution.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/utils/scopes_calculator.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/utils/tokens.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/dpm_xl/warning_collector.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/loaders/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/loaders/migration.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/_compat.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/auxiliary.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/base.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/glossary.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/infrastructure.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/operations.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/packaging.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/query_utils.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/release_sort_order.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/rendering.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/supercategories.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/orm/variables.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/py.typed +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/server/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/server/app.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/server/envelope.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/server/params.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/server/routers/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/server/routers/scope.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/server/routers/scripts.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/server/routers/structure.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/_parameters.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/_precondition_codes.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_export/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_export/queries.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_graph/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_graph/_template.html +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_graph/assets/VENDOR.md +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_graph/assets/cytoscape-dagre.min.js +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_graph/assets/cytoscape.min.js +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_graph/assets/dagre.min.js +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_graph/engine.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/calculations_graph/service.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/ecb_validations_import.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/explorer.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/export_csv.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/expression_metadata.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/hierarchy.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/layout_exporter/__init__.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/layout_exporter/processing.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/layout_exporter/service.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/meili_json.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/schema_validation.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/structure.py +0 -0
- {dpmcore-0.2.0 → dpmcore-0.2.1}/src/dpmcore/services/syntax.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dpmcore
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Python library for DPM 2.0 Refit regulatory reporting (ORM, services, REST API)
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -25,31 +25,33 @@ Provides-Extra: postgres
|
|
|
25
25
|
Provides-Extra: server
|
|
26
26
|
Provides-Extra: sqlserver
|
|
27
27
|
Requires-Dist: antlr4-python3-runtime (>=4.9,<4.10)
|
|
28
|
-
Requires-Dist: click (>=8.
|
|
29
|
-
Requires-Dist: click (>=8.
|
|
30
|
-
Requires-Dist: django (>=
|
|
31
|
-
Requires-Dist: django (>=
|
|
32
|
-
Requires-Dist: fastapi (>=0.
|
|
33
|
-
Requires-Dist: fastapi (>=0.
|
|
34
|
-
Requires-Dist: mssql-django (>=1.
|
|
35
|
-
Requires-Dist: mssql-django (>=1.
|
|
36
|
-
Requires-Dist: numpy (>=
|
|
37
|
-
Requires-Dist: numpy (>=
|
|
38
|
-
Requires-Dist: numpy (>=
|
|
28
|
+
Requires-Dist: click (>=8.1) ; extra == "all"
|
|
29
|
+
Requires-Dist: click (>=8.1) ; extra == "cli"
|
|
30
|
+
Requires-Dist: django (>=4.2) ; extra == "all"
|
|
31
|
+
Requires-Dist: django (>=4.2) ; extra == "django"
|
|
32
|
+
Requires-Dist: fastapi (>=0.101) ; extra == "all"
|
|
33
|
+
Requires-Dist: fastapi (>=0.101) ; extra == "server"
|
|
34
|
+
Requires-Dist: mssql-django (>=1.4) ; extra == "all"
|
|
35
|
+
Requires-Dist: mssql-django (>=1.4) ; extra == "django"
|
|
36
|
+
Requires-Dist: numpy (>=1.26,<3.0) ; extra == "all"
|
|
37
|
+
Requires-Dist: numpy (>=1.26,<3.0) ; extra == "data"
|
|
38
|
+
Requires-Dist: numpy (>=1.26,<3.0) ; extra == "migration"
|
|
39
39
|
Requires-Dist: openpyxl (>=3.1) ; extra == "all"
|
|
40
40
|
Requires-Dist: openpyxl (>=3.1) ; extra == "export"
|
|
41
41
|
Requires-Dist: pandas (>=2.1.4,<3.0) ; extra == "all"
|
|
42
42
|
Requires-Dist: pandas (>=2.1.4,<3.0) ; extra == "data"
|
|
43
43
|
Requires-Dist: pandas (>=2.1.4,<3.0) ; extra == "migration"
|
|
44
|
-
Requires-Dist: psycopg2-binary (>=2.9
|
|
45
|
-
Requires-Dist: psycopg2-binary (>=2.9
|
|
44
|
+
Requires-Dist: psycopg2-binary (>=2.9,<3.0) ; extra == "all"
|
|
45
|
+
Requires-Dist: psycopg2-binary (>=2.9,<3.0) ; extra == "postgres"
|
|
46
|
+
Requires-Dist: pydantic (>=2.0,<3.0) ; extra == "all"
|
|
47
|
+
Requires-Dist: pydantic (>=2.0,<3.0) ; extra == "server"
|
|
46
48
|
Requires-Dist: pyodbc (>=5.0) ; extra == "all"
|
|
47
49
|
Requires-Dist: pyodbc (>=5.0) ; extra == "sqlserver"
|
|
48
50
|
Requires-Dist: rich (>=13.7) ; extra == "all"
|
|
49
51
|
Requires-Dist: rich (>=13.7) ; extra == "cli"
|
|
50
|
-
Requires-Dist: sqlalchemy (>=
|
|
51
|
-
Requires-Dist: uvicorn[standard] (>=0.
|
|
52
|
-
Requires-Dist: uvicorn[standard] (>=0.
|
|
52
|
+
Requires-Dist: sqlalchemy (>=1.4.50,<3.0)
|
|
53
|
+
Requires-Dist: uvicorn[standard] (>=0.27) ; extra == "all"
|
|
54
|
+
Requires-Dist: uvicorn[standard] (>=0.27) ; extra == "server"
|
|
53
55
|
Project-URL: Bug Tracker, https://github.com/Meaningful-Data/dpmcore/issues
|
|
54
56
|
Project-URL: Documentation, https://docs.dpmcore.meaningfuldata.eu
|
|
55
57
|
Project-URL: Homepage, https://dpmcore.meaningfuldata.eu
|
|
@@ -263,7 +265,11 @@ dpmcore export-script \
|
|
|
263
265
|
Severities come from `OperationScope.Severity` in the database rather than
|
|
264
266
|
a manual `--severity` override. `--output` is optional for a single target;
|
|
265
267
|
when omitted, the script is written to
|
|
266
|
-
`<module_code>-<module_version>.json` in the current directory.
|
|
268
|
+
`<module_code>-<module_version>.json` in the current directory. Unlike
|
|
269
|
+
`script()`'s Python return value above, the file `export-script` writes
|
|
270
|
+
holds only `enriched_ast`'s content — `failed_operations` is printed to
|
|
271
|
+
the console instead (skipped-validation reasons, next to the discovered
|
|
272
|
+
count), not written to the file.
|
|
267
273
|
|
|
268
274
|
**Bulk export (`--all-modules`/`--all-versions`):**
|
|
269
275
|
|
|
@@ -298,6 +304,28 @@ three are pairwise mutually exclusive — a bare `--release` (no
|
|
|
298
304
|
single version active at that release, instead of naming a version
|
|
299
305
|
directly.
|
|
300
306
|
|
|
307
|
+
**Patching known EBA data-quality issues:**
|
|
308
|
+
|
|
309
|
+
A fixed set of ~30 validations reproduce a source-data error faithfully —
|
|
310
|
+
an empty/zero default that should be absent, a literal typed `Integer`
|
|
311
|
+
where the engine expects `Boolean`, etc. — because the underlying DPM
|
|
312
|
+
database data is wrong for them, not because of a generation bug. EBA
|
|
313
|
+
doesn't amend already-published releases, so these stay wrong
|
|
314
|
+
indefinitely; `fix-script` patches an already-generated script's JSON in
|
|
315
|
+
place. Run it after every `export-script`/`generate-script` run, not
|
|
316
|
+
just once:
|
|
317
|
+
|
|
318
|
+
```bash
|
|
319
|
+
# A single script file
|
|
320
|
+
dpmcore fix-script --input-path ./script.json
|
|
321
|
+
|
|
322
|
+
# Every script in a directory (e.g. after a bulk export-script sweep)
|
|
323
|
+
dpmcore fix-script --input-path ./scripts/ --bulk
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
Only rewrites a file when something in it actually matched a known
|
|
327
|
+
issue; prints which validations were patched per file.
|
|
328
|
+
|
|
301
329
|
**Calculations export:**
|
|
302
330
|
|
|
303
331
|
Export a module version's *calculations* — the operations linked to it
|
|
@@ -502,7 +530,7 @@ from dpmcore.services.database_update import DatabaseUpdateService
|
|
|
502
530
|
|
|
503
531
|
result = DatabaseUpdateService().update(
|
|
504
532
|
target="sqlite:///dpm.db", # SQLite path or URL, PostgreSQL/SQL Server URL
|
|
505
|
-
access_file="/path/to/dpm.accdb", #
|
|
533
|
+
access_file="/path/to/dpm.accdb", # or source_dir="..." for pre-exported CSVs
|
|
506
534
|
ecb_validations_file="ecb.csv", # optional
|
|
507
535
|
dry_run=False,
|
|
508
536
|
keep_staging=False,
|
|
@@ -515,8 +543,8 @@ print(f"Updated {result.target_type} database — "
|
|
|
515
543
|
Or from the command line:
|
|
516
544
|
|
|
517
545
|
```bash
|
|
518
|
-
# Update a SQLite database from
|
|
519
|
-
dpmcore update-db --target dpm.db
|
|
546
|
+
# Update a SQLite database from a pre-exported CSV directory
|
|
547
|
+
dpmcore update-db --target dpm.db --source-dir ./csv_export
|
|
520
548
|
|
|
521
549
|
# Update from an Access file
|
|
522
550
|
dpmcore update-db --target dpm.db --access-file /path/to/DPM_v4_2_1.accdb
|
|
@@ -572,7 +600,7 @@ from dpmcore.services.meili_build import MeiliBuildService
|
|
|
572
600
|
# From a directory of pre-exported CSV tables
|
|
573
601
|
result = MeiliBuildService().build(
|
|
574
602
|
output_file="operations.json",
|
|
575
|
-
source_dir="
|
|
603
|
+
source_dir="./csv_export",
|
|
576
604
|
)
|
|
577
605
|
print(f"Wrote {result.operations_written} operations to {result.output_file}")
|
|
578
606
|
|
|
@@ -588,7 +616,7 @@ Or from the command line:
|
|
|
588
616
|
|
|
589
617
|
```bash
|
|
590
618
|
# From a pre-exported CSV directory
|
|
591
|
-
dpmcore build-meili-json --source-dir
|
|
619
|
+
dpmcore build-meili-json --source-dir ./csv_export --output operations.json
|
|
592
620
|
|
|
593
621
|
# Directly from an Access file
|
|
594
622
|
dpmcore build-meili-json --access-file /path/to/dpm.accdb --output operations.json
|
|
@@ -766,7 +794,7 @@ src/dpmcore/
|
|
|
766
794
|
│ └── routers/ scope, scripts, structure
|
|
767
795
|
├── django/ Django integration app (models, admin, views)
|
|
768
796
|
├── cli/
|
|
769
|
-
│ └── main.py Click CLI (migrate, export-csv, build-meili-json, update-db, serve, generate-script, export-script, export-layout)
|
|
797
|
+
│ └── main.py Click CLI (migrate, export-csv, build-meili-json, update-db, serve, generate-script, export-script, fix-script, export-layout)
|
|
770
798
|
└── dpm_xl/ DPM-XL engine internals
|
|
771
799
|
├── grammar/ ANTLR4 grammar + generated parser
|
|
772
800
|
├── ast/ AST nodes, visitor, operands
|
|
@@ -205,7 +205,11 @@ dpmcore export-script \
|
|
|
205
205
|
Severities come from `OperationScope.Severity` in the database rather than
|
|
206
206
|
a manual `--severity` override. `--output` is optional for a single target;
|
|
207
207
|
when omitted, the script is written to
|
|
208
|
-
`<module_code>-<module_version>.json` in the current directory.
|
|
208
|
+
`<module_code>-<module_version>.json` in the current directory. Unlike
|
|
209
|
+
`script()`'s Python return value above, the file `export-script` writes
|
|
210
|
+
holds only `enriched_ast`'s content — `failed_operations` is printed to
|
|
211
|
+
the console instead (skipped-validation reasons, next to the discovered
|
|
212
|
+
count), not written to the file.
|
|
209
213
|
|
|
210
214
|
**Bulk export (`--all-modules`/`--all-versions`):**
|
|
211
215
|
|
|
@@ -240,6 +244,28 @@ three are pairwise mutually exclusive — a bare `--release` (no
|
|
|
240
244
|
single version active at that release, instead of naming a version
|
|
241
245
|
directly.
|
|
242
246
|
|
|
247
|
+
**Patching known EBA data-quality issues:**
|
|
248
|
+
|
|
249
|
+
A fixed set of ~30 validations reproduce a source-data error faithfully —
|
|
250
|
+
an empty/zero default that should be absent, a literal typed `Integer`
|
|
251
|
+
where the engine expects `Boolean`, etc. — because the underlying DPM
|
|
252
|
+
database data is wrong for them, not because of a generation bug. EBA
|
|
253
|
+
doesn't amend already-published releases, so these stay wrong
|
|
254
|
+
indefinitely; `fix-script` patches an already-generated script's JSON in
|
|
255
|
+
place. Run it after every `export-script`/`generate-script` run, not
|
|
256
|
+
just once:
|
|
257
|
+
|
|
258
|
+
```bash
|
|
259
|
+
# A single script file
|
|
260
|
+
dpmcore fix-script --input-path ./script.json
|
|
261
|
+
|
|
262
|
+
# Every script in a directory (e.g. after a bulk export-script sweep)
|
|
263
|
+
dpmcore fix-script --input-path ./scripts/ --bulk
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
Only rewrites a file when something in it actually matched a known
|
|
267
|
+
issue; prints which validations were patched per file.
|
|
268
|
+
|
|
243
269
|
**Calculations export:**
|
|
244
270
|
|
|
245
271
|
Export a module version's *calculations* — the operations linked to it
|
|
@@ -444,7 +470,7 @@ from dpmcore.services.database_update import DatabaseUpdateService
|
|
|
444
470
|
|
|
445
471
|
result = DatabaseUpdateService().update(
|
|
446
472
|
target="sqlite:///dpm.db", # SQLite path or URL, PostgreSQL/SQL Server URL
|
|
447
|
-
access_file="/path/to/dpm.accdb", #
|
|
473
|
+
access_file="/path/to/dpm.accdb", # or source_dir="..." for pre-exported CSVs
|
|
448
474
|
ecb_validations_file="ecb.csv", # optional
|
|
449
475
|
dry_run=False,
|
|
450
476
|
keep_staging=False,
|
|
@@ -457,8 +483,8 @@ print(f"Updated {result.target_type} database — "
|
|
|
457
483
|
Or from the command line:
|
|
458
484
|
|
|
459
485
|
```bash
|
|
460
|
-
# Update a SQLite database from
|
|
461
|
-
dpmcore update-db --target dpm.db
|
|
486
|
+
# Update a SQLite database from a pre-exported CSV directory
|
|
487
|
+
dpmcore update-db --target dpm.db --source-dir ./csv_export
|
|
462
488
|
|
|
463
489
|
# Update from an Access file
|
|
464
490
|
dpmcore update-db --target dpm.db --access-file /path/to/DPM_v4_2_1.accdb
|
|
@@ -514,7 +540,7 @@ from dpmcore.services.meili_build import MeiliBuildService
|
|
|
514
540
|
# From a directory of pre-exported CSV tables
|
|
515
541
|
result = MeiliBuildService().build(
|
|
516
542
|
output_file="operations.json",
|
|
517
|
-
source_dir="
|
|
543
|
+
source_dir="./csv_export",
|
|
518
544
|
)
|
|
519
545
|
print(f"Wrote {result.operations_written} operations to {result.output_file}")
|
|
520
546
|
|
|
@@ -530,7 +556,7 @@ Or from the command line:
|
|
|
530
556
|
|
|
531
557
|
```bash
|
|
532
558
|
# From a pre-exported CSV directory
|
|
533
|
-
dpmcore build-meili-json --source-dir
|
|
559
|
+
dpmcore build-meili-json --source-dir ./csv_export --output operations.json
|
|
534
560
|
|
|
535
561
|
# Directly from an Access file
|
|
536
562
|
dpmcore build-meili-json --access-file /path/to/dpm.accdb --output operations.json
|
|
@@ -708,7 +734,7 @@ src/dpmcore/
|
|
|
708
734
|
│ └── routers/ scope, scripts, structure
|
|
709
735
|
├── django/ Django integration app (models, admin, views)
|
|
710
736
|
├── cli/
|
|
711
|
-
│ └── main.py Click CLI (migrate, export-csv, build-meili-json, update-db, serve, generate-script, export-script, export-layout)
|
|
737
|
+
│ └── main.py Click CLI (migrate, export-csv, build-meili-json, update-db, serve, generate-script, export-script, fix-script, export-layout)
|
|
712
738
|
└── dpm_xl/ DPM-XL engine internals
|
|
713
739
|
├── grammar/ ANTLR4 grammar + generated parser
|
|
714
740
|
├── ast/ AST nodes, visitor, operands
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dpmcore"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.1"
|
|
4
4
|
description = "Python library for DPM 2.0 Refit regulatory reporting (ORM, services, REST API)"
|
|
5
5
|
license = { text = "Apache-2.0" }
|
|
6
6
|
readme = "README.md"
|
|
@@ -31,7 +31,7 @@ dependencies = [
|
|
|
31
31
|
# Floor is 1.4.50 to match the python3-sqlalchemy apt package on
|
|
32
32
|
# Ubuntu 24.04 (Noble); the ORM is compatible with both 1.4 and 2.0
|
|
33
33
|
# (see issue #104). 2.0 remains the canonical target for dev/wheel.
|
|
34
|
-
"sqlalchemy>=
|
|
34
|
+
"sqlalchemy>=1.4.50,<3.0",
|
|
35
35
|
# Runtime must match the ANTLR version used to generate the grammar
|
|
36
36
|
# in src/dpmcore/dpm_xl/grammar/generated/. Regenerating with a newer
|
|
37
37
|
# ANTLR is tracked separately.
|
|
@@ -45,30 +45,43 @@ documentation = "https://docs.dpmcore.meaningfuldata.eu"
|
|
|
45
45
|
"Bug Tracker" = "https://github.com/Meaningful-Data/dpmcore/issues"
|
|
46
46
|
|
|
47
47
|
[project.optional-dependencies]
|
|
48
|
-
# Floors
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
48
|
+
# Floors are the versions Ubuntu 24.04 (Noble) ships as apt packages, so
|
|
49
|
+
# dpmcore can be installed on top of the system Python without pip
|
|
50
|
+
# upgrading them (ubuntu_test_24_04.yml runs the suite against exactly
|
|
51
|
+
# those versions and checks these floors). Newer distros (Debian 13,
|
|
52
|
+
# Ubuntu 26.04) ship newer versions, which the ranges also accept. Do not
|
|
53
|
+
# raise a floor just because a newer release exists: the lock file is
|
|
54
|
+
# what pins the versions CI and development use.
|
|
55
|
+
#
|
|
56
|
+
# pydantic is imported directly by the server (v2 API: field_validator);
|
|
57
|
+
# Noble's apt python3-pydantic is 1.10, so it always comes from pip.
|
|
58
|
+
server = [
|
|
59
|
+
"fastapi>=0.101",
|
|
60
|
+
"pydantic>=2.0,<3.0",
|
|
61
|
+
"uvicorn[standard]>=0.27",
|
|
62
|
+
]
|
|
63
|
+
django = ["django>=4.2", "mssql-django>=1.4"]
|
|
64
|
+
postgres = ["psycopg2-binary>=2.9,<3.0"]
|
|
55
65
|
sqlserver = ["pyodbc>=5.0"]
|
|
56
|
-
# pandas
|
|
57
|
-
#
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
66
|
+
# pandas < 3.0: the 3.0 behaviour changes (copy-on-write, string dtype)
|
|
67
|
+
# have not been validated. numpy 1.26 is Noble's; numpy is imported
|
|
68
|
+
# directly (scope calculator, scalar types) so it is declared here
|
|
69
|
+
# rather than left to pandas.
|
|
70
|
+
migration = ["pandas>=2.1.4,<3.0", "numpy>=1.26,<3.0"]
|
|
71
|
+
data = ["pandas>=2.1.4,<3.0", "numpy>=1.26,<3.0"]
|
|
72
|
+
cli = ["click>=8.1", "rich>=13.7"]
|
|
61
73
|
export = ["openpyxl>=3.1"]
|
|
62
74
|
all = [
|
|
63
|
-
"fastapi>=0.
|
|
64
|
-
"
|
|
65
|
-
"
|
|
66
|
-
"
|
|
67
|
-
"
|
|
75
|
+
"fastapi>=0.101",
|
|
76
|
+
"pydantic>=2.0,<3.0",
|
|
77
|
+
"uvicorn[standard]>=0.27",
|
|
78
|
+
"django>=4.2",
|
|
79
|
+
"mssql-django>=1.4",
|
|
80
|
+
"psycopg2-binary>=2.9,<3.0",
|
|
68
81
|
"pyodbc>=5.0",
|
|
69
82
|
"pandas>=2.1.4,<3.0",
|
|
70
|
-
"numpy>=
|
|
71
|
-
"click>=8.
|
|
83
|
+
"numpy>=1.26,<3.0",
|
|
84
|
+
"click>=8.1",
|
|
72
85
|
"rich>=13.7",
|
|
73
86
|
"openpyxl>=3.1",
|
|
74
87
|
]
|
|
@@ -80,15 +93,15 @@ dpmcore = "dpmcore.cli.main:main"
|
|
|
80
93
|
packages = [{ include = "dpmcore", from = "src" }]
|
|
81
94
|
|
|
82
95
|
[tool.poetry.group.dev.dependencies]
|
|
83
|
-
coverage = ">=7.16.
|
|
84
|
-
ruff = ">=0.16.
|
|
85
|
-
mypy = ">=2.3.
|
|
96
|
+
coverage = ">=7.16.1,<8.0"
|
|
97
|
+
ruff = ">=0.16.9"
|
|
98
|
+
mypy = ">=2.3.1"
|
|
86
99
|
pytest = ">=9.1.1"
|
|
87
100
|
pytest-asyncio = ">=0.24"
|
|
88
101
|
pytest-cov = ">=6.0"
|
|
89
102
|
pyroma = ">=4.2"
|
|
90
103
|
httpx = ">=0.28,<1.0"
|
|
91
|
-
pytest-django = ">=4.
|
|
104
|
+
pytest-django = ">=4.14.0"
|
|
92
105
|
django-stubs = ">=6.0.9"
|
|
93
106
|
pandas-stubs = ">=2.2.2,<3.0"
|
|
94
107
|
# django and numpy are not imported by the dev tooling directly; they are
|
|
@@ -97,9 +110,9 @@ pandas-stubs = ">=2.2.2,<3.0"
|
|
|
97
110
|
# without a cap the dev group resolves the newest release while the `django`
|
|
98
111
|
# and `data`/`migration` extras resolve an older one -- Poetry then locks
|
|
99
112
|
# both with overlapping python markers and installs them over each other
|
|
100
|
-
# (issue #321).
|
|
101
|
-
#
|
|
102
|
-
#
|
|
113
|
+
# (issue #321). These ranges sit inside the (wider, distro-aligned) extra
|
|
114
|
+
# ranges, so the lock resolves a single version of each; they only steer
|
|
115
|
+
# the lock and are not part of the published metadata.
|
|
103
116
|
django = ">=5.2.17,<6.0"
|
|
104
117
|
numpy = ">=2.2.0,<2.3"
|
|
105
118
|
|
|
@@ -184,7 +184,8 @@ def export_csv(source: str, output_dir: str) -> None:
|
|
|
184
184
|
"--source-dir",
|
|
185
185
|
type=click.Path(exists=True, file_okay=False, path_type=str),
|
|
186
186
|
default=None,
|
|
187
|
-
help="Directory containing exported CSV tables.
|
|
187
|
+
help="Directory containing exported CSV tables. Required unless "
|
|
188
|
+
"--access-file is given.",
|
|
188
189
|
)
|
|
189
190
|
@click.option(
|
|
190
191
|
"--access-file",
|
|
@@ -192,7 +193,8 @@ def export_csv(source: str, output_dir: str) -> None:
|
|
|
192
193
|
default=None,
|
|
193
194
|
help=(
|
|
194
195
|
"Access .accdb / .mdb file. Exported to a temporary"
|
|
195
|
-
" CSV directory before building."
|
|
196
|
+
" CSV directory before building. Required unless --source-dir "
|
|
197
|
+
"is given."
|
|
196
198
|
),
|
|
197
199
|
)
|
|
198
200
|
@click.option(
|
|
@@ -268,7 +270,14 @@ def build_meili_json(
|
|
|
268
270
|
"--access-file",
|
|
269
271
|
type=click.Path(exists=True, dir_okay=False, path_type=str),
|
|
270
272
|
default=None,
|
|
271
|
-
help="
|
|
273
|
+
help="Access file. Mutually exclusive with --source-dir; one is required.",
|
|
274
|
+
)
|
|
275
|
+
@click.option(
|
|
276
|
+
"--source-dir",
|
|
277
|
+
type=click.Path(exists=True, file_okay=False, path_type=str),
|
|
278
|
+
default=None,
|
|
279
|
+
help="Directory containing exported CSV tables. Mutually exclusive "
|
|
280
|
+
"with --access-file; one is required.",
|
|
272
281
|
)
|
|
273
282
|
@click.option(
|
|
274
283
|
"--ecb-validations-file",
|
|
@@ -291,6 +300,7 @@ def build_meili_json(
|
|
|
291
300
|
def update_db(
|
|
292
301
|
target: str,
|
|
293
302
|
access_file: str | None,
|
|
303
|
+
source_dir: str | None,
|
|
294
304
|
ecb_validations_file: str | None,
|
|
295
305
|
dry_run: bool,
|
|
296
306
|
keep_staging: bool,
|
|
@@ -317,13 +327,17 @@ def update_db(
|
|
|
317
327
|
f"Updating [cyan]{target}[/cyan] from Access file "
|
|
318
328
|
f"[cyan]{access_file}[/cyan]..."
|
|
319
329
|
)
|
|
320
|
-
|
|
321
|
-
console.print(
|
|
330
|
+
elif source_dir is not None:
|
|
331
|
+
console.print(
|
|
332
|
+
f"Updating [cyan]{target}[/cyan] from CSV directory "
|
|
333
|
+
f"[cyan]{source_dir}[/cyan]..."
|
|
334
|
+
)
|
|
322
335
|
|
|
323
336
|
try:
|
|
324
337
|
result = DatabaseUpdateService().update(
|
|
325
338
|
target=target,
|
|
326
339
|
access_file=access_file,
|
|
340
|
+
source_dir=source_dir,
|
|
327
341
|
ecb_validations_file=ecb_validations_file,
|
|
328
342
|
dry_run=dry_run,
|
|
329
343
|
keep_staging=keep_staging,
|
|
@@ -346,7 +360,7 @@ def update_db(
|
|
|
346
360
|
if result.used_access_file:
|
|
347
361
|
console.print("[green]Source loaded from Access file[/green]")
|
|
348
362
|
else:
|
|
349
|
-
console.print("[green]Source loaded from
|
|
363
|
+
console.print("[green]Source loaded from CSV directory[/green]")
|
|
350
364
|
|
|
351
365
|
if result.ecb_validations_imported:
|
|
352
366
|
console.print("[green]ECB validations imported[/green]")
|
|
@@ -641,6 +655,18 @@ def export_script(
|
|
|
641
655
|
``--all-modules``/``--all-versions`` to sweep many at once.
|
|
642
656
|
``--release`` on its own (no ``--module-version``/``--all-versions``)
|
|
643
657
|
selects each targeted module's version active at that release instead.
|
|
658
|
+
|
|
659
|
+
The output file holds only the ``enriched_ast`` content — the same
|
|
660
|
+
``{namespace: ...}`` shape mdpm's own export produces — not the
|
|
661
|
+
``success``/``error``/``failed_operations`` wrapper ``script()``
|
|
662
|
+
returns internally. Skipped validations are reported on the console
|
|
663
|
+
instead, not written into the file.
|
|
664
|
+
|
|
665
|
+
Run ``dpmcore fix-script`` on the output afterwards: a fixed set of
|
|
666
|
+
validations carry a known EBA source-data error that this command
|
|
667
|
+
faithfully reproduces (it's not a generation bug), and EBA doesn't
|
|
668
|
+
amend published releases, so it stays wrong indefinitely until
|
|
669
|
+
patched.
|
|
644
670
|
"""
|
|
645
671
|
import json
|
|
646
672
|
from pathlib import Path
|
|
@@ -721,7 +747,7 @@ def export_script(
|
|
|
721
747
|
|
|
722
748
|
out_path = out_dir / f"{code}-{version}.json"
|
|
723
749
|
out_path.write_text(
|
|
724
|
-
json.dumps(result, indent=2, default=str),
|
|
750
|
+
json.dumps(result["enriched_ast"], indent=2, default=str),
|
|
725
751
|
encoding="utf-8",
|
|
726
752
|
)
|
|
727
753
|
n_ops, n_skipped, n_dep = _script_result_counts(result)
|
|
@@ -732,6 +758,7 @@ def export_script(
|
|
|
732
758
|
f"{n_skipped} skipped, "
|
|
733
759
|
f"{n_dep} dependency modules)"
|
|
734
760
|
)
|
|
761
|
+
_report_skipped_operations(console, result)
|
|
735
762
|
succeeded.append((code, version))
|
|
736
763
|
|
|
737
764
|
_print_sweep_summary(
|
|
@@ -766,7 +793,8 @@ def export_script(
|
|
|
766
793
|
)
|
|
767
794
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
768
795
|
out_path.write_text(
|
|
769
|
-
json.dumps(result, indent=2, default=str),
|
|
796
|
+
json.dumps(result["enriched_ast"], indent=2, default=str),
|
|
797
|
+
encoding="utf-8",
|
|
770
798
|
)
|
|
771
799
|
|
|
772
800
|
n_ops, n_skipped, n_dep = _script_result_counts(result)
|
|
@@ -867,7 +895,7 @@ def _print_sweep_summary(
|
|
|
867
895
|
if total_skipped:
|
|
868
896
|
console.print(
|
|
869
897
|
f"[yellow]{total_skipped} validations skipped[/yellow] for "
|
|
870
|
-
"semantic errors — see
|
|
898
|
+
"semantic errors — see the per-target detail printed above "
|
|
871
899
|
"for the reason per validation."
|
|
872
900
|
)
|
|
873
901
|
|
|
@@ -877,10 +905,11 @@ def _report_skipped_operations(
|
|
|
877
905
|
) -> None:
|
|
878
906
|
"""Print why each skipped validation was left out of the script.
|
|
879
907
|
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
908
|
+
A console line that only counts what made it gives no hint that
|
|
909
|
+
anything was dropped (#355) — the output file no longer carries
|
|
910
|
+
``failed_operations`` for ``export-script`` (only the ``enriched_ast``
|
|
911
|
+
content, matching mdpm's own script shape), so this is the only place
|
|
912
|
+
skipped validations are reported for that command.
|
|
884
913
|
|
|
885
914
|
Reasons are escaped before printing: a message naming an item, e.g.
|
|
886
915
|
``[eba_AS:x2]``, reads as rich markup and would otherwise be
|
|
@@ -898,10 +927,110 @@ def _report_skipped_operations(
|
|
|
898
927
|
for code, reason in list(failed_ops.items())[:limit]:
|
|
899
928
|
console.print(f" [yellow]{escape(code)}[/yellow]: {escape(reason)}")
|
|
900
929
|
if len(failed_ops) > limit:
|
|
930
|
+
console.print(f" ... and {len(failed_ops) - limit} more skipped")
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
@main.command("fix-script")
|
|
934
|
+
@click.option(
|
|
935
|
+
"--input-path",
|
|
936
|
+
required=True,
|
|
937
|
+
type=click.Path(exists=True),
|
|
938
|
+
help="A single script .json file, or a directory of them with --bulk.",
|
|
939
|
+
)
|
|
940
|
+
@click.option(
|
|
941
|
+
"--bulk",
|
|
942
|
+
is_flag=True,
|
|
943
|
+
default=False,
|
|
944
|
+
help="Treat --input-path as a directory and fix every .json file "
|
|
945
|
+
"in it, instead of a single file.",
|
|
946
|
+
)
|
|
947
|
+
def fix_script(input_path: str, bulk: bool) -> None:
|
|
948
|
+
"""Patch known EBA source-data errors into already-generated scripts.
|
|
949
|
+
|
|
950
|
+
Rewrites each file in place. Ported from mdpm's
|
|
951
|
+
``mdm-fix-json-values.py``: see
|
|
952
|
+
``dpmcore.services.script_fixups`` for the fixed validations and
|
|
953
|
+
why they're wrong at the data level, not a generation bug — EBA
|
|
954
|
+
doesn't amend past releases, so run this after every
|
|
955
|
+
``export-script`` generation, not just once.
|
|
956
|
+
"""
|
|
957
|
+
try:
|
|
958
|
+
from rich.console import Console
|
|
959
|
+
except ImportError:
|
|
960
|
+
click.echo(
|
|
961
|
+
"Install 'rich' for pretty output: pip install dpmcore[cli]",
|
|
962
|
+
err=True,
|
|
963
|
+
)
|
|
964
|
+
sys.exit(1)
|
|
965
|
+
|
|
966
|
+
console = Console()
|
|
967
|
+
json_files = _resolve_fix_script_targets(console, input_path, bulk)
|
|
968
|
+
|
|
969
|
+
total_fixed = 0
|
|
970
|
+
for json_file in json_files:
|
|
971
|
+
total_fixed += _fix_script_file(console, json_file)
|
|
972
|
+
|
|
973
|
+
if not total_fixed:
|
|
974
|
+
console.print("[yellow]No known data-quality issues found.[/yellow]")
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
def _resolve_fix_script_targets(
|
|
978
|
+
console: Any, input_path: str, bulk: bool
|
|
979
|
+
) -> list[Any]:
|
|
980
|
+
"""Resolve ``fix-script``'s ``--input-path``/``--bulk`` to files."""
|
|
981
|
+
from pathlib import Path
|
|
982
|
+
|
|
983
|
+
path = Path(input_path)
|
|
984
|
+
if bulk:
|
|
985
|
+
if not path.is_dir():
|
|
986
|
+
console.print(
|
|
987
|
+
f"[red]--bulk requires --input-path to be a "
|
|
988
|
+
f"directory:[/red] {input_path}"
|
|
989
|
+
)
|
|
990
|
+
sys.exit(1)
|
|
991
|
+
json_files = sorted(path.glob("*.json"))
|
|
992
|
+
if not json_files:
|
|
993
|
+
console.print(f"[red]No .json files found in:[/red] {input_path}")
|
|
994
|
+
sys.exit(1)
|
|
995
|
+
return json_files
|
|
996
|
+
|
|
997
|
+
if not path.is_file() or path.suffix != ".json":
|
|
901
998
|
console.print(
|
|
902
|
-
f"
|
|
903
|
-
"'failed_operations' in the output file."
|
|
999
|
+
f"[red]--input-path is not a .json file:[/red] {input_path}"
|
|
904
1000
|
)
|
|
1001
|
+
sys.exit(1)
|
|
1002
|
+
return [path]
|
|
1003
|
+
|
|
1004
|
+
|
|
1005
|
+
def _fix_script_file(console: Any, json_file: Any) -> int:
|
|
1006
|
+
"""Apply known fixups to one script file, rewriting it in place.
|
|
1007
|
+
|
|
1008
|
+
Returns the number of validations changed (0 leaves the file
|
|
1009
|
+
untouched, so an already-fixed file is never rewritten).
|
|
1010
|
+
"""
|
|
1011
|
+
import json
|
|
1012
|
+
|
|
1013
|
+
from dpmcore.services.script_fixups import fix_module_operations
|
|
1014
|
+
|
|
1015
|
+
data = json.loads(json_file.read_text(encoding="utf-8"))
|
|
1016
|
+
changed_codes: list[str] = []
|
|
1017
|
+
for module_data in data.values():
|
|
1018
|
+
operations = module_data.get("operations")
|
|
1019
|
+
if operations:
|
|
1020
|
+
changed_codes.extend(fix_module_operations(operations))
|
|
1021
|
+
if not changed_codes:
|
|
1022
|
+
return 0
|
|
1023
|
+
|
|
1024
|
+
json_file.write_text(
|
|
1025
|
+
json.dumps(data, indent=2, ensure_ascii=False, default=str),
|
|
1026
|
+
encoding="utf-8",
|
|
1027
|
+
)
|
|
1028
|
+
console.print(
|
|
1029
|
+
f"[green]Fixed[/green] {json_file} "
|
|
1030
|
+
f"({len(changed_codes)} validations: "
|
|
1031
|
+
f"{', '.join(sorted(set(changed_codes)))})"
|
|
1032
|
+
)
|
|
1033
|
+
return len(changed_codes)
|
|
905
1034
|
|
|
906
1035
|
|
|
907
1036
|
def _script_result_counts(result: dict[str, Any]) -> tuple[int, int, int]:
|
|
@@ -410,13 +410,16 @@ class ASTVisitor(dpm_xlParserVisitor):
|
|
|
410
410
|
elif first == "current data point":
|
|
411
411
|
return WindowBoundary(bound_type="current_data_point")
|
|
412
412
|
else:
|
|
413
|
-
# INTEGER_LITERAL
|
|
413
|
+
# INTEGER_LITERAL TIME_PERIOD? preceding|following
|
|
414
414
|
n = int(first)
|
|
415
|
-
|
|
415
|
+
last = self._symbol_text(ctx_list[-1])
|
|
416
416
|
bound_type = (
|
|
417
|
-
"n_preceding" if
|
|
417
|
+
"n_preceding" if last == "preceding" else "n_following"
|
|
418
|
+
)
|
|
419
|
+
period = (
|
|
420
|
+
self._symbol_text(ctx_list[1]) if len(ctx_list) == 3 else None
|
|
418
421
|
)
|
|
419
|
-
return WindowBoundary(bound_type=bound_type, n=n)
|
|
422
|
+
return WindowBoundary(bound_type=bound_type, n=n, period=period)
|
|
420
423
|
|
|
421
424
|
def visitGroupingClause(
|
|
422
425
|
self, ctx: dpm_xlParser.GroupingClauseContext
|