dpmcore 0.2.0__tar.gz → 0.2.1rc2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/PKG-INFO +39 -13
  2. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/README.md +33 -7
  3. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/pyproject.toml +10 -10
  4. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/__init__.py +1 -1
  5. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/cli/main.py +144 -15
  6. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/module_dependencies.py +1 -1
  7. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/nodes.py +1 -1
  8. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/dpm_xlLexer.g4 +63 -37
  9. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/dpm_xlParser.g4 +24 -14
  10. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/model_queries.py +702 -272
  11. dpmcore-0.2.1rc2/src/dpmcore/dpm_xl/utils/db_ast.py +907 -0
  12. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/_open_keys.py +49 -17
  13. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/ast_generator.py +1257 -192
  14. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/data_dictionary.py +2 -4
  15. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/database_update.py +10 -2
  16. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/layout_exporter/excel_writer.py +5 -2
  17. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/layout_exporter/models.py +5 -0
  18. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/layout_exporter/queries.py +89 -36
  19. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/meili_build.py +5 -3
  20. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/scope_calculator.py +458 -35
  21. dpmcore-0.2.1rc2/src/dpmcore/services/script_fixups.py +408 -0
  22. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/LICENSE +0 -0
  23. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/cli/__init__.py +0 -0
  24. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/connection.py +0 -0
  25. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/data/__init__.py +0 -0
  26. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/data/module_schema_mapping.csv +0 -0
  27. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/__init__.py +0 -0
  28. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/admin.py +0 -0
  29. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/apps.py +0 -0
  30. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/models/__init__.py +0 -0
  31. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/models/auxiliary.py +0 -0
  32. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/models/glossary.py +0 -0
  33. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/models/infrastructure.py +0 -0
  34. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/models/operations.py +0 -0
  35. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/models/packaging.py +0 -0
  36. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/models/rendering.py +0 -0
  37. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/models/variables.py +0 -0
  38. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/routers.py +0 -0
  39. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/django/urls.py +0 -0
  40. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/__init__.py +0 -0
  41. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/__init__.py +0 -0
  42. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/constructor.py +0 -0
  43. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/domain_membership.py +0 -0
  44. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/ml_generation.py +0 -0
  45. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/module_analyzer.py +0 -0
  46. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/operands.py +0 -0
  47. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/template.py +0 -0
  48. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/visitor.py +0 -0
  49. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/ast/where_clause.py +0 -0
  50. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/__init__.py +0 -0
  51. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/__init__.py +0 -0
  52. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlLexer.interp +0 -0
  53. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlLexer.py +0 -0
  54. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlLexer.tokens +0 -0
  55. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParser.interp +0 -0
  56. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParser.py +0 -0
  57. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParser.tokens +0 -0
  58. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParserListener.py +0 -0
  59. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/dpm_xlParserVisitor.py +0 -0
  60. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/grammar/generated/listeners.py +0 -0
  61. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/__init__.py +0 -0
  62. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/aggregate.py +0 -0
  63. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/arithmetic.py +0 -0
  64. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/base.py +0 -0
  65. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/boolean.py +0 -0
  66. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/clause.py +0 -0
  67. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/comparison.py +0 -0
  68. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/conditional.py +0 -0
  69. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/string.py +0 -0
  70. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/operators/time.py +0 -0
  71. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/semantic_analyzer.py +0 -0
  72. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/symbols.py +0 -0
  73. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/types/__init__.py +0 -0
  74. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/types/promotion.py +0 -0
  75. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/types/scalar.py +0 -0
  76. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/types/time.py +0 -0
  77. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/utils/__init__.py +0 -0
  78. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/utils/data_handlers.py +0 -0
  79. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/utils/filters.py +0 -0
  80. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/utils/operands_mapping.py +0 -0
  81. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/utils/operator_mapping.py +0 -0
  82. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/utils/range_resolution.py +0 -0
  83. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/utils/scopes_calculator.py +0 -0
  84. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/utils/serialization.py +0 -0
  85. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/utils/tokens.py +0 -0
  86. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/dpm_xl/warning_collector.py +0 -0
  87. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/errors.py +0 -0
  88. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/loaders/__init__.py +0 -0
  89. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/loaders/migration.py +0 -0
  90. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/__init__.py +0 -0
  91. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/_compat.py +0 -0
  92. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/auxiliary.py +0 -0
  93. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/base.py +0 -0
  94. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/glossary.py +0 -0
  95. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/infrastructure.py +0 -0
  96. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/operations.py +0 -0
  97. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/packaging.py +0 -0
  98. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/query_utils.py +0 -0
  99. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/release_sort_order.py +0 -0
  100. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/rendering.py +0 -0
  101. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/supercategories.py +0 -0
  102. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/orm/variables.py +0 -0
  103. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/py.typed +0 -0
  104. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/server/__init__.py +0 -0
  105. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/server/app.py +0 -0
  106. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/server/envelope.py +0 -0
  107. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/server/params.py +0 -0
  108. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/server/routers/__init__.py +0 -0
  109. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/server/routers/scope.py +0 -0
  110. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/server/routers/scripts.py +0 -0
  111. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/server/routers/structure.py +0 -0
  112. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/__init__.py +0 -0
  113. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/_parameters.py +0 -0
  114. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/_precondition_codes.py +0 -0
  115. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_export/__init__.py +0 -0
  116. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_export/exporter.py +0 -0
  117. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_export/queries.py +0 -0
  118. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_export/visitors.py +0 -0
  119. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_graph/__init__.py +0 -0
  120. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_graph/_template.html +0 -0
  121. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_graph/assets/VENDOR.md +0 -0
  122. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_graph/assets/cytoscape-dagre.min.js +0 -0
  123. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_graph/assets/cytoscape.min.js +0 -0
  124. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_graph/assets/dagre.min.js +0 -0
  125. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_graph/engine.py +0 -0
  126. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/calculations_graph/service.py +0 -0
  127. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/dpm_xl.py +0 -0
  128. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/ecb_validations_import.py +0 -0
  129. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/explorer.py +0 -0
  130. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/export_csv.py +0 -0
  131. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/expression_metadata.py +0 -0
  132. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/hierarchy.py +0 -0
  133. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/layout_exporter/__init__.py +0 -0
  134. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/layout_exporter/processing.py +0 -0
  135. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/layout_exporter/service.py +0 -0
  136. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/meili_json.py +0 -0
  137. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/schema_validation.py +0 -0
  138. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/semantic.py +0 -0
  139. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/structure.py +0 -0
  140. {dpmcore-0.2.0 → dpmcore-0.2.1rc2}/src/dpmcore/services/syntax.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dpmcore
3
- Version: 0.2.0
3
+ Version: 0.2.1rc2
4
4
  Summary: Python library for DPM 2.0 Refit regulatory reporting (ORM, services, REST API)
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -41,15 +41,15 @@ Requires-Dist: openpyxl (>=3.1) ; extra == "export"
41
41
  Requires-Dist: pandas (>=2.1.4,<3.0) ; extra == "all"
42
42
  Requires-Dist: pandas (>=2.1.4,<3.0) ; extra == "data"
43
43
  Requires-Dist: pandas (>=2.1.4,<3.0) ; extra == "migration"
44
- Requires-Dist: psycopg2-binary (>=2.9.12,<3.0) ; extra == "all"
45
- Requires-Dist: psycopg2-binary (>=2.9.12,<3.0) ; extra == "postgres"
44
+ Requires-Dist: psycopg2-binary (>=2.9.13,<3.0) ; extra == "all"
45
+ Requires-Dist: psycopg2-binary (>=2.9.13,<3.0) ; extra == "postgres"
46
46
  Requires-Dist: pyodbc (>=5.0) ; extra == "all"
47
47
  Requires-Dist: pyodbc (>=5.0) ; extra == "sqlserver"
48
48
  Requires-Dist: rich (>=13.7) ; extra == "all"
49
49
  Requires-Dist: rich (>=13.7) ; extra == "cli"
50
- Requires-Dist: sqlalchemy (>=2.0.52,<3.0)
51
- Requires-Dist: uvicorn[standard] (>=0.52.1) ; extra == "all"
52
- Requires-Dist: uvicorn[standard] (>=0.52.1) ; extra == "server"
50
+ Requires-Dist: sqlalchemy (>=2.0.54,<3.0)
51
+ Requires-Dist: uvicorn[standard] (>=0.53.0) ; extra == "all"
52
+ Requires-Dist: uvicorn[standard] (>=0.53.0) ; extra == "server"
53
53
  Project-URL: Bug Tracker, https://github.com/Meaningful-Data/dpmcore/issues
54
54
  Project-URL: Documentation, https://docs.dpmcore.meaningfuldata.eu
55
55
  Project-URL: Homepage, https://dpmcore.meaningfuldata.eu
@@ -263,7 +263,11 @@ dpmcore export-script \
263
263
  Severities come from `OperationScope.Severity` in the database rather than
264
264
  a manual `--severity` override. `--output` is optional for a single target;
265
265
  when omitted, the script is written to
266
- `<module_code>-<module_version>.json` in the current directory.
266
+ `<module_code>-<module_version>.json` in the current directory. Unlike
267
+ `script()`'s Python return value above, the file `export-script` writes
268
+ holds only `enriched_ast`'s content — `failed_operations` is printed to
269
+ the console instead (skipped-validation reasons, next to the discovered
270
+ count), not written to the file.
267
271
 
268
272
  **Bulk export (`--all-modules`/`--all-versions`):**
269
273
 
@@ -298,6 +302,28 @@ three are pairwise mutually exclusive — a bare `--release` (no
298
302
  single version active at that release, instead of naming a version
299
303
  directly.
300
304
 
305
+ **Patching known EBA data-quality issues:**
306
+
307
+ A fixed set of ~30 validations reproduce a source-data error faithfully —
308
+ an empty/zero default that should be absent, a literal typed `Integer`
309
+ where the engine expects `Boolean`, etc. — because the underlying DPM
310
+ database data is wrong for them, not because of a generation bug. EBA
311
+ doesn't amend already-published releases, so these stay wrong
312
+ indefinitely; `fix-script` patches an already-generated script's JSON in
313
+ place. Run it after every `export-script`/`generate-script` run, not
314
+ just once:
315
+
316
+ ```bash
317
+ # A single script file
318
+ dpmcore fix-script --input-path ./script.json
319
+
320
+ # Every script in a directory (e.g. after a bulk export-script sweep)
321
+ dpmcore fix-script --input-path ./scripts/ --bulk
322
+ ```
323
+
324
+ Only rewrites a file when something in it actually matched a known
325
+ issue; prints which validations were patched per file.
326
+
301
327
  **Calculations export:**
302
328
 
303
329
  Export a module version's *calculations* — the operations linked to it
@@ -502,7 +528,7 @@ from dpmcore.services.database_update import DatabaseUpdateService
502
528
 
503
529
  result = DatabaseUpdateService().update(
504
530
  target="sqlite:///dpm.db", # SQLite path or URL, PostgreSQL/SQL Server URL
505
- access_file="/path/to/dpm.accdb", # optional; omit to read from data/DPM/
531
+ access_file="/path/to/dpm.accdb", # or source_dir="..." for pre-exported CSVs
506
532
  ecb_validations_file="ecb.csv", # optional
507
533
  dry_run=False,
508
534
  keep_staging=False,
@@ -515,8 +541,8 @@ print(f"Updated {result.target_type} database — "
515
541
  Or from the command line:
516
542
 
517
543
  ```bash
518
- # Update a SQLite database from data/DPM/ CSV files
519
- dpmcore update-db --target dpm.db
544
+ # Update a SQLite database from a pre-exported CSV directory
545
+ dpmcore update-db --target dpm.db --source-dir ./csv_export
520
546
 
521
547
  # Update from an Access file
522
548
  dpmcore update-db --target dpm.db --access-file /path/to/DPM_v4_2_1.accdb
@@ -572,7 +598,7 @@ from dpmcore.services.meili_build import MeiliBuildService
572
598
  # From a directory of pre-exported CSV tables
573
599
  result = MeiliBuildService().build(
574
600
  output_file="operations.json",
575
- source_dir="data/DPM",
601
+ source_dir="./csv_export",
576
602
  )
577
603
  print(f"Wrote {result.operations_written} operations to {result.output_file}")
578
604
 
@@ -588,7 +614,7 @@ Or from the command line:
588
614
 
589
615
  ```bash
590
616
  # From a pre-exported CSV directory
591
- dpmcore build-meili-json --source-dir data/DPM --output operations.json
617
+ dpmcore build-meili-json --source-dir ./csv_export --output operations.json
592
618
 
593
619
  # Directly from an Access file
594
620
  dpmcore build-meili-json --access-file /path/to/dpm.accdb --output operations.json
@@ -766,7 +792,7 @@ src/dpmcore/
766
792
  │ └── routers/ scope, scripts, structure
767
793
  ├── django/ Django integration app (models, admin, views)
768
794
  ├── cli/
769
- │ └── main.py Click CLI (migrate, export-csv, build-meili-json, update-db, serve, generate-script, export-script, export-layout)
795
+ │ └── main.py Click CLI (migrate, export-csv, build-meili-json, update-db, serve, generate-script, export-script, fix-script, export-layout)
770
796
  └── dpm_xl/ DPM-XL engine internals
771
797
  ├── grammar/ ANTLR4 grammar + generated parser
772
798
  ├── ast/ AST nodes, visitor, operands
@@ -205,7 +205,11 @@ dpmcore export-script \
205
205
  Severities come from `OperationScope.Severity` in the database rather than
206
206
  a manual `--severity` override. `--output` is optional for a single target;
207
207
  when omitted, the script is written to
208
- `<module_code>-<module_version>.json` in the current directory.
208
+ `<module_code>-<module_version>.json` in the current directory. Unlike
209
+ `script()`'s Python return value above, the file `export-script` writes
210
+ holds only `enriched_ast`'s content — `failed_operations` is printed to
211
+ the console instead (skipped-validation reasons, next to the discovered
212
+ count), not written to the file.
209
213
 
210
214
  **Bulk export (`--all-modules`/`--all-versions`):**
211
215
 
@@ -240,6 +244,28 @@ three are pairwise mutually exclusive — a bare `--release` (no
240
244
  single version active at that release, instead of naming a version
241
245
  directly.
242
246
 
247
+ **Patching known EBA data-quality issues:**
248
+
249
+ A fixed set of ~30 validations reproduce a source-data error faithfully —
250
+ an empty/zero default that should be absent, a literal typed `Integer`
251
+ where the engine expects `Boolean`, etc. — because the underlying DPM
252
+ database data is wrong for them, not because of a generation bug. EBA
253
+ doesn't amend already-published releases, so these stay wrong
254
+ indefinitely; `fix-script` patches an already-generated script's JSON in
255
+ place. Run it after every `export-script`/`generate-script` run, not
256
+ just once:
257
+
258
+ ```bash
259
+ # A single script file
260
+ dpmcore fix-script --input-path ./script.json
261
+
262
+ # Every script in a directory (e.g. after a bulk export-script sweep)
263
+ dpmcore fix-script --input-path ./scripts/ --bulk
264
+ ```
265
+
266
+ Only rewrites a file when something in it actually matched a known
267
+ issue; prints which validations were patched per file.
268
+
243
269
  **Calculations export:**
244
270
 
245
271
  Export a module version's *calculations* — the operations linked to it
@@ -444,7 +470,7 @@ from dpmcore.services.database_update import DatabaseUpdateService
444
470
 
445
471
  result = DatabaseUpdateService().update(
446
472
  target="sqlite:///dpm.db", # SQLite path or URL, PostgreSQL/SQL Server URL
447
- access_file="/path/to/dpm.accdb", # optional; omit to read from data/DPM/
473
+ access_file="/path/to/dpm.accdb", # or source_dir="..." for pre-exported CSVs
448
474
  ecb_validations_file="ecb.csv", # optional
449
475
  dry_run=False,
450
476
  keep_staging=False,
@@ -457,8 +483,8 @@ print(f"Updated {result.target_type} database — "
457
483
  Or from the command line:
458
484
 
459
485
  ```bash
460
- # Update a SQLite database from data/DPM/ CSV files
461
- dpmcore update-db --target dpm.db
486
+ # Update a SQLite database from a pre-exported CSV directory
487
+ dpmcore update-db --target dpm.db --source-dir ./csv_export
462
488
 
463
489
  # Update from an Access file
464
490
  dpmcore update-db --target dpm.db --access-file /path/to/DPM_v4_2_1.accdb
@@ -514,7 +540,7 @@ from dpmcore.services.meili_build import MeiliBuildService
514
540
  # From a directory of pre-exported CSV tables
515
541
  result = MeiliBuildService().build(
516
542
  output_file="operations.json",
517
- source_dir="data/DPM",
543
+ source_dir="./csv_export",
518
544
  )
519
545
  print(f"Wrote {result.operations_written} operations to {result.output_file}")
520
546
 
@@ -530,7 +556,7 @@ Or from the command line:
530
556
 
531
557
  ```bash
532
558
  # From a pre-exported CSV directory
533
- dpmcore build-meili-json --source-dir data/DPM --output operations.json
559
+ dpmcore build-meili-json --source-dir ./csv_export --output operations.json
534
560
 
535
561
  # Directly from an Access file
536
562
  dpmcore build-meili-json --access-file /path/to/dpm.accdb --output operations.json
@@ -708,7 +734,7 @@ src/dpmcore/
708
734
  │ └── routers/ scope, scripts, structure
709
735
  ├── django/ Django integration app (models, admin, views)
710
736
  ├── cli/
711
- │ └── main.py Click CLI (migrate, export-csv, build-meili-json, update-db, serve, generate-script, export-script, export-layout)
737
+ │ └── main.py Click CLI (migrate, export-csv, build-meili-json, update-db, serve, generate-script, export-script, fix-script, export-layout)
712
738
  └── dpm_xl/ DPM-XL engine internals
713
739
  ├── grammar/ ANTLR4 grammar + generated parser
714
740
  ├── ast/ AST nodes, visitor, operands
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dpmcore"
3
- version = "0.2.0"
3
+ version = "0.2.1rc2"
4
4
  description = "Python library for DPM 2.0 Refit regulatory reporting (ORM, services, REST API)"
5
5
  license = { text = "Apache-2.0" }
6
6
  readme = "README.md"
@@ -31,7 +31,7 @@ dependencies = [
31
31
  # Floor is 1.4.50 to match the python3-sqlalchemy apt package on
32
32
  # Ubuntu 24.04 (Noble); the ORM is compatible with both 1.4 and 2.0
33
33
  # (see issue #104). 2.0 remains the canonical target for dev/wheel.
34
- "sqlalchemy>=2.0.52,<3.0",
34
+ "sqlalchemy>=2.0.54,<3.0",
35
35
  # Runtime must match the ANTLR version used to generate the grammar
36
36
  # in src/dpmcore/dpm_xl/grammar/generated/. Regenerating with a newer
37
37
  # ANTLR is tracked separately.
@@ -49,9 +49,9 @@ documentation = "https://docs.dpmcore.meaningfuldata.eu"
49
49
  # ubuntu_test_24_04.yml can install this library with system packages
50
50
  # via `pip install --no-dependencies`. Version bumps above these floors
51
51
  # should stay within semver compatibility.
52
- server = ["fastapi>=0.141.1", "uvicorn[standard]>=0.52.1"]
52
+ server = ["fastapi>=0.141.1", "uvicorn[standard]>=0.53.0"]
53
53
  django = ["django>=5.2.17", "mssql-django>=1.8.0"]
54
- postgres = ["psycopg2-binary>=2.9.12,<3.0"]
54
+ postgres = ["psycopg2-binary>=2.9.13,<3.0"]
55
55
  sqlserver = ["pyodbc>=5.0"]
56
56
  # pandas / numpy ranges mirror vtlengine's published pins so that
57
57
  # dpmcore + vtlengine can be co-installed without resolver conflicts.
@@ -61,10 +61,10 @@ cli = ["click>=8.5.0", "rich>=13.7"]
61
61
  export = ["openpyxl>=3.1"]
62
62
  all = [
63
63
  "fastapi>=0.141.1",
64
- "uvicorn[standard]>=0.52.1",
64
+ "uvicorn[standard]>=0.53.0",
65
65
  "django>=5.2.17",
66
66
  "mssql-django>=1.8.0",
67
- "psycopg2-binary>=2.9.12,<3.0",
67
+ "psycopg2-binary>=2.9.13,<3.0",
68
68
  "pyodbc>=5.0",
69
69
  "pandas>=2.1.4,<3.0",
70
70
  "numpy>=2.2.0,<2.3",
@@ -80,15 +80,15 @@ dpmcore = "dpmcore.cli.main:main"
80
80
  packages = [{ include = "dpmcore", from = "src" }]
81
81
 
82
82
  [tool.poetry.group.dev.dependencies]
83
- coverage = ">=7.16.0,<8.0"
84
- ruff = ">=0.16.2"
85
- mypy = ">=2.3.0"
83
+ coverage = ">=7.16.1,<8.0"
84
+ ruff = ">=0.16.8"
85
+ mypy = ">=2.3.1"
86
86
  pytest = ">=9.1.1"
87
87
  pytest-asyncio = ">=0.24"
88
88
  pytest-cov = ">=6.0"
89
89
  pyroma = ">=4.2"
90
90
  httpx = ">=0.28,<1.0"
91
- pytest-django = ">=4.13.0"
91
+ pytest-django = ">=4.14.0"
92
92
  django-stubs = ">=6.0.9"
93
93
  pandas-stubs = ">=2.2.2,<3.0"
94
94
  # django and numpy are not imported by the dev tooling directly; they are
@@ -1,6 +1,6 @@
1
1
  """Your opinionated Python DPM library."""
2
2
 
3
- __version__ = "0.2.0"
3
+ __version__ = "0.2.1rc2"
4
4
 
5
5
  from dpmcore.connection import DpmConnection, connect
6
6
 
@@ -184,7 +184,8 @@ def export_csv(source: str, output_dir: str) -> None:
184
184
  "--source-dir",
185
185
  type=click.Path(exists=True, file_okay=False, path_type=str),
186
186
  default=None,
187
- help="Directory containing exported CSV tables. Defaults to data/DPM.",
187
+ help="Directory containing exported CSV tables. Required unless "
188
+ "--access-file is given.",
188
189
  )
189
190
  @click.option(
190
191
  "--access-file",
@@ -192,7 +193,8 @@ def export_csv(source: str, output_dir: str) -> None:
192
193
  default=None,
193
194
  help=(
194
195
  "Access .accdb / .mdb file. Exported to a temporary"
195
- " CSV directory before building."
196
+ " CSV directory before building. Required unless --source-dir "
197
+ "is given."
196
198
  ),
197
199
  )
198
200
  @click.option(
@@ -268,7 +270,14 @@ def build_meili_json(
268
270
  "--access-file",
269
271
  type=click.Path(exists=True, dir_okay=False, path_type=str),
270
272
  default=None,
271
- help="Optional Access file. If omitted, data/DPM CSVs are used.",
273
+ help="Access file. Mutually exclusive with --source-dir; one is required.",
274
+ )
275
+ @click.option(
276
+ "--source-dir",
277
+ type=click.Path(exists=True, file_okay=False, path_type=str),
278
+ default=None,
279
+ help="Directory containing exported CSV tables. Mutually exclusive "
280
+ "with --access-file; one is required.",
272
281
  )
273
282
  @click.option(
274
283
  "--ecb-validations-file",
@@ -291,6 +300,7 @@ def build_meili_json(
291
300
  def update_db(
292
301
  target: str,
293
302
  access_file: str | None,
303
+ source_dir: str | None,
294
304
  ecb_validations_file: str | None,
295
305
  dry_run: bool,
296
306
  keep_staging: bool,
@@ -317,13 +327,17 @@ def update_db(
317
327
  f"Updating [cyan]{target}[/cyan] from Access file "
318
328
  f"[cyan]{access_file}[/cyan]..."
319
329
  )
320
- else:
321
- console.print(f"Updating [cyan]{target}[/cyan] from data/DPM CSVs...")
330
+ elif source_dir is not None:
331
+ console.print(
332
+ f"Updating [cyan]{target}[/cyan] from CSV directory "
333
+ f"[cyan]{source_dir}[/cyan]..."
334
+ )
322
335
 
323
336
  try:
324
337
  result = DatabaseUpdateService().update(
325
338
  target=target,
326
339
  access_file=access_file,
340
+ source_dir=source_dir,
327
341
  ecb_validations_file=ecb_validations_file,
328
342
  dry_run=dry_run,
329
343
  keep_staging=keep_staging,
@@ -346,7 +360,7 @@ def update_db(
346
360
  if result.used_access_file:
347
361
  console.print("[green]Source loaded from Access file[/green]")
348
362
  else:
349
- console.print("[green]Source loaded from data/DPM CSVs[/green]")
363
+ console.print("[green]Source loaded from CSV directory[/green]")
350
364
 
351
365
  if result.ecb_validations_imported:
352
366
  console.print("[green]ECB validations imported[/green]")
@@ -641,6 +655,18 @@ def export_script(
641
655
  ``--all-modules``/``--all-versions`` to sweep many at once.
642
656
  ``--release`` on its own (no ``--module-version``/``--all-versions``)
643
657
  selects each targeted module's version active at that release instead.
658
+
659
+ The output file holds only the ``enriched_ast`` content — the same
660
+ ``{namespace: ...}`` shape mdpm's own export produces — not the
661
+ ``success``/``error``/``failed_operations`` wrapper ``script()``
662
+ returns internally. Skipped validations are reported on the console
663
+ instead, not written into the file.
664
+
665
+ Run ``dpmcore fix-script`` on the output afterwards: a fixed set of
666
+ validations carry a known EBA source-data error that this command
667
+ faithfully reproduces (it's not a generation bug), and EBA doesn't
668
+ amend published releases, so it stays wrong indefinitely until
669
+ patched.
644
670
  """
645
671
  import json
646
672
  from pathlib import Path
@@ -721,7 +747,7 @@ def export_script(
721
747
 
722
748
  out_path = out_dir / f"{code}-{version}.json"
723
749
  out_path.write_text(
724
- json.dumps(result, indent=2, default=str),
750
+ json.dumps(result["enriched_ast"], indent=2, default=str),
725
751
  encoding="utf-8",
726
752
  )
727
753
  n_ops, n_skipped, n_dep = _script_result_counts(result)
@@ -732,6 +758,7 @@ def export_script(
732
758
  f"{n_skipped} skipped, "
733
759
  f"{n_dep} dependency modules)"
734
760
  )
761
+ _report_skipped_operations(console, result)
735
762
  succeeded.append((code, version))
736
763
 
737
764
  _print_sweep_summary(
@@ -766,7 +793,8 @@ def export_script(
766
793
  )
767
794
  out_path.parent.mkdir(parents=True, exist_ok=True)
768
795
  out_path.write_text(
769
- json.dumps(result, indent=2, default=str), encoding="utf-8"
796
+ json.dumps(result["enriched_ast"], indent=2, default=str),
797
+ encoding="utf-8",
770
798
  )
771
799
 
772
800
  n_ops, n_skipped, n_dep = _script_result_counts(result)
@@ -867,7 +895,7 @@ def _print_sweep_summary(
867
895
  if total_skipped:
868
896
  console.print(
869
897
  f"[yellow]{total_skipped} validations skipped[/yellow] for "
870
- "semantic errors — see 'failed_operations' in each script "
898
+ "semantic errors — see the per-target detail printed above "
871
899
  "for the reason per validation."
872
900
  )
873
901
 
@@ -877,10 +905,11 @@ def _report_skipped_operations(
877
905
  ) -> None:
878
906
  """Print why each skipped validation was left out of the script.
879
907
 
880
- ``failed_operations`` is already written to the output JSON, but a
881
- console line that only counts what made it gives no hint that
882
- anything was dropped (#355). Long lists are truncated — the file
883
- holds all of them.
908
+ A console line that only counts what made it gives no hint that
909
+ anything was dropped (#355) — the output file no longer carries
910
+ ``failed_operations`` for ``export-script`` (only the ``enriched_ast``
911
+ content, matching mdpm's own script shape), so this is the only place
912
+ skipped validations are reported for that command.
884
913
 
885
914
  Reasons are escaped before printing: a message naming an item, e.g.
886
915
  ``[eba_AS:x2]``, reads as rich markup and would otherwise be
@@ -898,10 +927,110 @@ def _report_skipped_operations(
898
927
  for code, reason in list(failed_ops.items())[:limit]:
899
928
  console.print(f" [yellow]{escape(code)}[/yellow]: {escape(reason)}")
900
929
  if len(failed_ops) > limit:
930
+ console.print(f" ... and {len(failed_ops) - limit} more skipped")
931
+
932
+
933
+ @main.command("fix-script")
934
+ @click.option(
935
+ "--input-path",
936
+ required=True,
937
+ type=click.Path(exists=True),
938
+ help="A single script .json file, or a directory of them with --bulk.",
939
+ )
940
+ @click.option(
941
+ "--bulk",
942
+ is_flag=True,
943
+ default=False,
944
+ help="Treat --input-path as a directory and fix every .json file "
945
+ "in it, instead of a single file.",
946
+ )
947
+ def fix_script(input_path: str, bulk: bool) -> None:
948
+ """Patch known EBA source-data errors into already-generated scripts.
949
+
950
+ Rewrites each file in place. Ported from mdpm's
951
+ ``mdm-fix-json-values.py``: see
952
+ ``dpmcore.services.script_fixups`` for the fixed validations and
953
+ why they're wrong at the data level, not a generation bug — EBA
954
+ doesn't amend past releases, so run this after every
955
+ ``export-script`` generation, not just once.
956
+ """
957
+ try:
958
+ from rich.console import Console
959
+ except ImportError:
960
+ click.echo(
961
+ "Install 'rich' for pretty output: pip install dpmcore[cli]",
962
+ err=True,
963
+ )
964
+ sys.exit(1)
965
+
966
+ console = Console()
967
+ json_files = _resolve_fix_script_targets(console, input_path, bulk)
968
+
969
+ total_fixed = 0
970
+ for json_file in json_files:
971
+ total_fixed += _fix_script_file(console, json_file)
972
+
973
+ if not total_fixed:
974
+ console.print("[yellow]No known data-quality issues found.[/yellow]")
975
+
976
+
977
+ def _resolve_fix_script_targets(
978
+ console: Any, input_path: str, bulk: bool
979
+ ) -> list[Any]:
980
+ """Resolve ``fix-script``'s ``--input-path``/``--bulk`` to files."""
981
+ from pathlib import Path
982
+
983
+ path = Path(input_path)
984
+ if bulk:
985
+ if not path.is_dir():
986
+ console.print(
987
+ f"[red]--bulk requires --input-path to be a "
988
+ f"directory:[/red] {input_path}"
989
+ )
990
+ sys.exit(1)
991
+ json_files = sorted(path.glob("*.json"))
992
+ if not json_files:
993
+ console.print(f"[red]No .json files found in:[/red] {input_path}")
994
+ sys.exit(1)
995
+ return json_files
996
+
997
+ if not path.is_file() or path.suffix != ".json":
901
998
  console.print(
902
- f" ... and {len(failed_ops) - limit} more — see "
903
- "'failed_operations' in the output file."
999
+ f"[red]--input-path is not a .json file:[/red] {input_path}"
904
1000
  )
1001
+ sys.exit(1)
1002
+ return [path]
1003
+
1004
+
1005
+ def _fix_script_file(console: Any, json_file: Any) -> int:
1006
+ """Apply known fixups to one script file, rewriting it in place.
1007
+
1008
+ Returns the number of validations changed (0 leaves the file
1009
+ untouched, so an already-fixed file is never rewritten).
1010
+ """
1011
+ import json
1012
+
1013
+ from dpmcore.services.script_fixups import fix_module_operations
1014
+
1015
+ data = json.loads(json_file.read_text(encoding="utf-8"))
1016
+ changed_codes: list[str] = []
1017
+ for module_data in data.values():
1018
+ operations = module_data.get("operations")
1019
+ if operations:
1020
+ changed_codes.extend(fix_module_operations(operations))
1021
+ if not changed_codes:
1022
+ return 0
1023
+
1024
+ json_file.write_text(
1025
+ json.dumps(data, indent=2, ensure_ascii=False, default=str),
1026
+ encoding="utf-8",
1027
+ )
1028
+ console.print(
1029
+ f"[green]Fixed[/green] {json_file} "
1030
+ f"({len(changed_codes)} validations: "
1031
+ f"{', '.join(sorted(set(changed_codes)))})"
1032
+ )
1033
+ return len(changed_codes)
905
1034
 
906
1035
 
907
1036
  def _script_result_counts(result: dict[str, Any]) -> tuple[int, int, int]:
@@ -226,7 +226,7 @@ class ModuleDependencies(ASTTemplate, ABC):
226
226
  raise errors.SemanticError("6-3", precondition=node.variable_id)
227
227
 
228
228
  if not TableVersionQuery.check_table_exists(
229
- self.session, node.variable_id, self.release_id
229
+ self.session, str(node.variable_id), self.release_id
230
230
  ):
231
231
  raise errors.SemanticError("1-3", variable=node.variable_id)
232
232
 
@@ -964,7 +964,7 @@ class PreconditionItem(AST):
964
964
  """
965
965
 
966
966
  def __init__(
967
- self, variable_id: str, variable_code: str | None = None
967
+ self, variable_id: int, variable_code: str | None = None
968
968
  ) -> None:
969
969
  super().__init__()
970
970
  self.variable_id = variable_id