flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,772 @@
|
|
|
1
|
+
"""Apply a YAML delta to a ``tests/fixtures/*.json`` test fixture.
|
|
2
|
+
|
|
3
|
+
Stage 2c addendum to the Spine-DB fixture maintenance toolkit. Sits
|
|
4
|
+
alongside :mod:`flextool.update_flextool.test_fixtures` (round-trip
|
|
5
|
+
migrator) and :mod:`flextool.update_flextool.generate_canonical`
|
|
6
|
+
(subset-and-export filter). Provides a third, agent-friendly path:
|
|
7
|
+
small, declarative additions described in YAML rather than driven
|
|
8
|
+
through the SpineDB editor GUI.
|
|
9
|
+
|
|
10
|
+
Scope
|
|
11
|
+
-----
|
|
12
|
+
Append-only. Adds new entities, alternatives, parameter values, and
|
|
13
|
+
scenarios. Editing existing entries is intentionally out of scope —
|
|
14
|
+
that workflow remains the SpineDB editor or a Python migration in
|
|
15
|
+
``db_migration.py``. Complex parameter value types (time-series,
|
|
16
|
+
maps, arrays) are also out of scope; the YAML accepts scalar values
|
|
17
|
+
only (int / float / str / bool / None). Anything more structured
|
|
18
|
+
should be edited via the SpineDB editor.
|
|
19
|
+
|
|
20
|
+
YAML schema
|
|
21
|
+
-----------
|
|
22
|
+
::
|
|
23
|
+
|
|
24
|
+
target: tests/fixtures/tests.json # repo-relative path
|
|
25
|
+
|
|
26
|
+
new_entities:
|
|
27
|
+
- {class: node, name: my_node, description: "(Optional)"}
|
|
28
|
+
- {class: connection__node__node, entities: [my_conn, src, dst]}
|
|
29
|
+
|
|
30
|
+
new_alternatives:
|
|
31
|
+
- {name: my_feature_init, description: "..."}
|
|
32
|
+
|
|
33
|
+
new_parameter_values:
|
|
34
|
+
- {class: node, entity: [my_node],
|
|
35
|
+
alternative: my_feature_init,
|
|
36
|
+
parameter: inflow, value: 42.0}
|
|
37
|
+
|
|
38
|
+
new_scenarios:
|
|
39
|
+
- {name: my_feature_demo,
|
|
40
|
+
alternatives: [Base, my_feature_init]}
|
|
41
|
+
|
|
42
|
+
regenerate_canonical: true # optional
|
|
43
|
+
|
|
44
|
+
Workflow
|
|
45
|
+
--------
|
|
46
|
+
1. Load source ``tests/fixtures/*.json`` into a temp SQLite via
|
|
47
|
+
:func:`tests.db_utils.json_to_db`.
|
|
48
|
+
2. Validate the delta against ``flextool/schemas/spinedb_schema.json``
|
|
49
|
+
(entity classes exist, parameters exist) AND against the source
|
|
50
|
+
contents (append-only — names must not collide with existing
|
|
51
|
+
entities / alternatives / scenarios / (entity, parameter,
|
|
52
|
+
alternative) value tuples).
|
|
53
|
+
3. Apply the delta via :mod:`spinedb_api`.
|
|
54
|
+
4. Export the SQLite back over the source JSON via
|
|
55
|
+
:func:`tests.db_utils.db_to_json`.
|
|
56
|
+
5. Optionally regenerate any canonical_databases recipe whose
|
|
57
|
+
``source`` field references the modified fixture.
|
|
58
|
+
|
|
59
|
+
CLI
|
|
60
|
+
---
|
|
61
|
+
::
|
|
62
|
+
|
|
63
|
+
python -m flextool.update_flextool.extend_tests_fixture <yaml>
|
|
64
|
+
python -m flextool.update_flextool.extend_tests_fixture <yaml> \\
|
|
65
|
+
--target tests/fixtures/tests.json
|
|
66
|
+
python -m flextool.update_flextool.extend_tests_fixture --validate <yaml>
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
from __future__ import annotations
|
|
70
|
+
|
|
71
|
+
import argparse
|
|
72
|
+
import difflib
|
|
73
|
+
import json
|
|
74
|
+
import shutil
|
|
75
|
+
import sys
|
|
76
|
+
import tempfile
|
|
77
|
+
from pathlib import Path
|
|
78
|
+
from typing import Any
|
|
79
|
+
|
|
80
|
+
import yaml
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# Repo-root resolution mirrors generate_canonical.py / test_fixtures.py —
|
|
84
|
+
# this module is a source-checkout maintenance tool, never invoked from
|
|
85
|
+
# an installed wheel.
|
|
86
|
+
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
87
|
+
_SCHEMA_PATH = (
|
|
88
|
+
Path(__file__).resolve().parents[1] / "schemas" / "spinedb_schema.json"
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# ----------------------------------------------------------------------------
|
|
93
|
+
# Schema indexing
|
|
94
|
+
# ----------------------------------------------------------------------------
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _load_schema() -> dict[str, Any]:
|
|
98
|
+
"""Return the parsed ``spinedb_schema.json``.
|
|
99
|
+
|
|
100
|
+
Loaded lazily by :func:`apply_delta` / :func:`validate_delta` rather
|
|
101
|
+
than at import time so the module can be imported in environments
|
|
102
|
+
where the source checkout layout differs (e.g. for type-checking).
|
|
103
|
+
"""
|
|
104
|
+
if not _SCHEMA_PATH.is_file():
|
|
105
|
+
raise RuntimeError(f"Schema file missing: {_SCHEMA_PATH}")
|
|
106
|
+
with open(_SCHEMA_PATH) as f:
|
|
107
|
+
return json.load(f)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _index_schema(schema: dict[str, Any]) -> dict[str, Any]:
|
|
111
|
+
"""Build the lookup tables needed by :func:`validate_delta`.
|
|
112
|
+
|
|
113
|
+
Returns a dict with:
|
|
114
|
+
|
|
115
|
+
* ``entity_classes`` : ``{name: dimension_name_list}`` — empty
|
|
116
|
+
tuple for 0-dim ("primary") classes, populated for relationship
|
|
117
|
+
classes.
|
|
118
|
+
* ``parameters`` : ``{(class_name, param_name): None}`` — set
|
|
119
|
+
of valid (class, parameter) pairs.
|
|
120
|
+
|
|
121
|
+
Indexed once per :func:`apply_delta` call. The schema lists are
|
|
122
|
+
short enough (~30 entity_classes, ~230 parameter_definitions) that
|
|
123
|
+
a single linear pass is fine.
|
|
124
|
+
"""
|
|
125
|
+
entity_classes: dict[str, tuple[str, ...]] = {}
|
|
126
|
+
for row in schema.get("entity_classes", []):
|
|
127
|
+
# row layout: [name, dimension_name_list, description, ...]
|
|
128
|
+
name = row[0]
|
|
129
|
+
dims = tuple(row[1]) if row[1] else ()
|
|
130
|
+
entity_classes[name] = dims
|
|
131
|
+
|
|
132
|
+
parameters: dict[tuple[str, str], None] = {}
|
|
133
|
+
for row in schema.get("parameter_definitions", []):
|
|
134
|
+
# row layout: [entity_class_name, name, default_value, ...]
|
|
135
|
+
parameters[(row[0], row[1])] = None
|
|
136
|
+
|
|
137
|
+
return {"entity_classes": entity_classes, "parameters": parameters}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# ----------------------------------------------------------------------------
|
|
141
|
+
# Source-state indexing
|
|
142
|
+
# ----------------------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _index_source(source_json: Path) -> dict[str, Any]:
|
|
146
|
+
"""Build the lookup tables for append-only enforcement.
|
|
147
|
+
|
|
148
|
+
Reads the source JSON directly (no SQLite round-trip needed for
|
|
149
|
+
name lookups). Returns:
|
|
150
|
+
|
|
151
|
+
* ``entities`` : ``set[(class, name)]`` for 0-dim entities;
|
|
152
|
+
``set[(class, tuple(element_names))]`` for multi-dim entities.
|
|
153
|
+
Both forms are keyed by class, so a name collision in a *different*
|
|
154
|
+
class is correctly allowed.
|
|
155
|
+
* ``alternatives`` : ``set[name]``
|
|
156
|
+
* ``scenarios`` : ``set[name]``
|
|
157
|
+
* ``parameter_values`` : ``set[(class, entity_name, parameter,
|
|
158
|
+
alternative)]`` — the uniqueness key for SpineDB parameter values.
|
|
159
|
+
|
|
160
|
+
Used only for validation diagnostics; the actual append happens
|
|
161
|
+
against a fresh SQLite via spinedb_api which enforces the same
|
|
162
|
+
uniqueness constraints at the DB level.
|
|
163
|
+
"""
|
|
164
|
+
with open(source_json) as f:
|
|
165
|
+
data = json.load(f)
|
|
166
|
+
|
|
167
|
+
entities: set[tuple[str, Any]] = set()
|
|
168
|
+
for row in data.get("entities", []):
|
|
169
|
+
# row layout: [class_name, name_or_element_list, description]
|
|
170
|
+
cls_name = row[0]
|
|
171
|
+
ident = row[1]
|
|
172
|
+
if isinstance(ident, list):
|
|
173
|
+
entities.add((cls_name, tuple(ident)))
|
|
174
|
+
else:
|
|
175
|
+
entities.add((cls_name, ident))
|
|
176
|
+
|
|
177
|
+
alternatives = {row[0] for row in data.get("alternatives", [])}
|
|
178
|
+
scenarios = {row[0] for row in data.get("scenarios", [])}
|
|
179
|
+
|
|
180
|
+
parameter_values: set[tuple[str, str, str, str]] = set()
|
|
181
|
+
for row in data.get("parameter_values", []):
|
|
182
|
+
# row layout: [class, entity_name, parameter, value, alternative].
|
|
183
|
+
# entity_name is a list for multi-dim entities; flatten it via
|
|
184
|
+
# _multi_dim_name so the lookup key matches the form we build
|
|
185
|
+
# for delta rows (``_multi_dim_name(ent_elements)``).
|
|
186
|
+
ent_name = (
|
|
187
|
+
_multi_dim_name(row[1]) if isinstance(row[1], list) else row[1]
|
|
188
|
+
)
|
|
189
|
+
parameter_values.add((row[0], ent_name, row[2], row[4]))
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
"entities": entities,
|
|
193
|
+
"alternatives": alternatives,
|
|
194
|
+
"scenarios": scenarios,
|
|
195
|
+
"parameter_values": parameter_values,
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
# ----------------------------------------------------------------------------
|
|
200
|
+
# Validation
|
|
201
|
+
# ----------------------------------------------------------------------------
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
_SCALAR_TYPES: tuple[type, ...] = (int, float, str, bool, type(None))
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _suggest(name: str, candidates: list[str], cutoff: float = 0.6) -> str:
|
|
208
|
+
"""Format a ``did you mean`` clause for an unknown identifier.
|
|
209
|
+
|
|
210
|
+
Returns an empty string when no close match exists, otherwise
|
|
211
|
+
" (did you mean 'X'?)" with the single best suggestion. Uses
|
|
212
|
+
:func:`difflib.get_close_matches` so the threshold matches what
|
|
213
|
+
Python's own ``KeyError`` style suggestions use.
|
|
214
|
+
"""
|
|
215
|
+
hits = difflib.get_close_matches(name, candidates, n=1, cutoff=cutoff)
|
|
216
|
+
return f" (did you mean {hits[0]!r}?)" if hits else ""
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _multi_dim_name(elements: list[str]) -> str:
|
|
220
|
+
"""Construct SpineDB's canonical multi-dim entity name.
|
|
221
|
+
|
|
222
|
+
Spine uses double-underscore concatenation of element names. We
|
|
223
|
+
surface this in error messages so users can grep their fixture for
|
|
224
|
+
pre-existing collisions.
|
|
225
|
+
"""
|
|
226
|
+
return "__".join(elements)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def validate_delta(delta: dict, schema: dict, source_index: dict | None = None) -> list[str]:
|
|
230
|
+
"""Return validation errors for a parsed YAML delta.
|
|
231
|
+
|
|
232
|
+
Empty list means the delta is safe to apply. Each string is a
|
|
233
|
+
standalone, readable error message — callers print them one per
|
|
234
|
+
line.
|
|
235
|
+
|
|
236
|
+
Two layers of checks:
|
|
237
|
+
|
|
238
|
+
* Schema-level (always) — entity_classes exist; parameter names
|
|
239
|
+
exist for the named entity class; values are scalar.
|
|
240
|
+
* Source-level (when ``source_index`` is provided) — append-only:
|
|
241
|
+
no name collisions with existing entities, alternatives,
|
|
242
|
+
scenarios, or (class, entity, parameter, alternative) value
|
|
243
|
+
tuples.
|
|
244
|
+
|
|
245
|
+
The two layers are split because the YAML can be schema-valid but
|
|
246
|
+
source-invalid; callers (e.g. the ``--validate`` CLI mode) may want
|
|
247
|
+
schema-only validation when the target fixture isn't accessible.
|
|
248
|
+
"""
|
|
249
|
+
errors: list[str] = []
|
|
250
|
+
idx = _index_schema(schema)
|
|
251
|
+
entity_classes: dict[str, tuple[str, ...]] = idx["entity_classes"]
|
|
252
|
+
parameters: dict[tuple[str, str], None] = idx["parameters"]
|
|
253
|
+
|
|
254
|
+
# Pre-resolved name lists for suggestion strings. Build once so
|
|
255
|
+
# the per-row hot loop doesn't re-iterate the schema.
|
|
256
|
+
class_names = list(entity_classes)
|
|
257
|
+
|
|
258
|
+
# Track names declared earlier in *this* delta so the second of two
|
|
259
|
+
# identically-named entries in the same YAML is rejected even when
|
|
260
|
+
# the source lookup passes (otherwise a typo would slip through and
|
|
261
|
+
# land as a NothingToCommit at apply time).
|
|
262
|
+
declared_entities: set[tuple[str, Any]] = set()
|
|
263
|
+
declared_alternatives: set[str] = set()
|
|
264
|
+
declared_scenarios: set[str] = set()
|
|
265
|
+
declared_parameter_values: set[tuple[str, str, str, str]] = set()
|
|
266
|
+
|
|
267
|
+
src_entities = source_index["entities"] if source_index else set()
|
|
268
|
+
src_alternatives = source_index["alternatives"] if source_index else set()
|
|
269
|
+
src_scenarios = source_index["scenarios"] if source_index else set()
|
|
270
|
+
src_param_values = source_index["parameter_values"] if source_index else set()
|
|
271
|
+
|
|
272
|
+
# --- new_entities ----------------------------------------------------
|
|
273
|
+
for i, row in enumerate(delta.get("new_entities") or []):
|
|
274
|
+
prefix = f"new_entities[{i}]"
|
|
275
|
+
if not isinstance(row, dict):
|
|
276
|
+
errors.append(f"{prefix}: expected a mapping, got {type(row).__name__}")
|
|
277
|
+
continue
|
|
278
|
+
cls = row.get("class")
|
|
279
|
+
if not cls:
|
|
280
|
+
errors.append(f"{prefix}: missing required field 'class'")
|
|
281
|
+
continue
|
|
282
|
+
if cls not in entity_classes:
|
|
283
|
+
errors.append(
|
|
284
|
+
f"{prefix}: entity_class {cls!r} not in schema"
|
|
285
|
+
+ _suggest(cls, class_names)
|
|
286
|
+
)
|
|
287
|
+
continue
|
|
288
|
+
dims = entity_classes[cls]
|
|
289
|
+
if dims:
|
|
290
|
+
# multi-dim: 'entities' is required, 'name' is forbidden
|
|
291
|
+
if "name" in row:
|
|
292
|
+
errors.append(
|
|
293
|
+
f"{prefix}: multi-dim class {cls!r} takes 'entities: [...]', not 'name'"
|
|
294
|
+
)
|
|
295
|
+
continue
|
|
296
|
+
elems = row.get("entities")
|
|
297
|
+
if not isinstance(elems, list) or not all(isinstance(e, str) for e in elems):
|
|
298
|
+
errors.append(
|
|
299
|
+
f"{prefix}: multi-dim class {cls!r} requires 'entities: [...]' as a list of strings"
|
|
300
|
+
)
|
|
301
|
+
continue
|
|
302
|
+
if len(elems) != len(dims):
|
|
303
|
+
errors.append(
|
|
304
|
+
f"{prefix}: class {cls!r} has {len(dims)} dimensions {list(dims)}, "
|
|
305
|
+
f"got {len(elems)} element(s) {elems!r}"
|
|
306
|
+
)
|
|
307
|
+
continue
|
|
308
|
+
key = (cls, tuple(elems))
|
|
309
|
+
display = _multi_dim_name(elems)
|
|
310
|
+
else:
|
|
311
|
+
if "entities" in row:
|
|
312
|
+
errors.append(
|
|
313
|
+
f"{prefix}: 0-dim class {cls!r} takes 'name', not 'entities'"
|
|
314
|
+
)
|
|
315
|
+
continue
|
|
316
|
+
name = row.get("name")
|
|
317
|
+
if not isinstance(name, str):
|
|
318
|
+
errors.append(f"{prefix}: 0-dim class {cls!r} requires a string 'name'")
|
|
319
|
+
continue
|
|
320
|
+
key = (cls, name)
|
|
321
|
+
display = name
|
|
322
|
+
if key in src_entities:
|
|
323
|
+
errors.append(
|
|
324
|
+
f"{prefix}: entity {display!r} of class {cls!r} already exists in target "
|
|
325
|
+
"(append-only — edits go through the SpineDB editor)"
|
|
326
|
+
)
|
|
327
|
+
continue
|
|
328
|
+
if key in declared_entities:
|
|
329
|
+
errors.append(
|
|
330
|
+
f"{prefix}: entity {display!r} of class {cls!r} declared more than once in this delta"
|
|
331
|
+
)
|
|
332
|
+
continue
|
|
333
|
+
declared_entities.add(key)
|
|
334
|
+
|
|
335
|
+
# --- new_alternatives ------------------------------------------------
|
|
336
|
+
for i, row in enumerate(delta.get("new_alternatives") or []):
|
|
337
|
+
prefix = f"new_alternatives[{i}]"
|
|
338
|
+
if not isinstance(row, dict):
|
|
339
|
+
errors.append(f"{prefix}: expected a mapping, got {type(row).__name__}")
|
|
340
|
+
continue
|
|
341
|
+
name = row.get("name")
|
|
342
|
+
if not isinstance(name, str):
|
|
343
|
+
errors.append(f"{prefix}: missing or non-string 'name'")
|
|
344
|
+
continue
|
|
345
|
+
if name in src_alternatives:
|
|
346
|
+
errors.append(
|
|
347
|
+
f"{prefix}: alternative {name!r} already exists in target "
|
|
348
|
+
"(append-only)"
|
|
349
|
+
)
|
|
350
|
+
continue
|
|
351
|
+
if name in declared_alternatives:
|
|
352
|
+
errors.append(f"{prefix}: alternative {name!r} declared more than once in this delta")
|
|
353
|
+
continue
|
|
354
|
+
declared_alternatives.add(name)
|
|
355
|
+
|
|
356
|
+
# --- new_parameter_values --------------------------------------------
|
|
357
|
+
# The set of entity *names* available after this delta is applied =
|
|
358
|
+
# source entities ∪ entities declared earlier in this YAML. For
|
|
359
|
+
# 0-dim classes we compare by name; for multi-dim, by element tuple
|
|
360
|
+
# (the SpineDB convention).
|
|
361
|
+
available_entities: set[tuple[str, Any]] = set(src_entities) | declared_entities
|
|
362
|
+
available_alternatives: set[str] = set(src_alternatives) | declared_alternatives
|
|
363
|
+
|
|
364
|
+
for i, row in enumerate(delta.get("new_parameter_values") or []):
|
|
365
|
+
prefix = f"new_parameter_values[{i}]"
|
|
366
|
+
if not isinstance(row, dict):
|
|
367
|
+
errors.append(f"{prefix}: expected a mapping, got {type(row).__name__}")
|
|
368
|
+
continue
|
|
369
|
+
cls = row.get("class")
|
|
370
|
+
if not cls or cls not in entity_classes:
|
|
371
|
+
errors.append(
|
|
372
|
+
f"{prefix}: entity_class {cls!r} not in schema"
|
|
373
|
+
+ _suggest(cls or "", class_names)
|
|
374
|
+
)
|
|
375
|
+
continue
|
|
376
|
+
param = row.get("parameter")
|
|
377
|
+
if not isinstance(param, str):
|
|
378
|
+
errors.append(f"{prefix}: missing or non-string 'parameter'")
|
|
379
|
+
continue
|
|
380
|
+
if (cls, param) not in parameters:
|
|
381
|
+
class_params = [p for c, p in parameters if c == cls]
|
|
382
|
+
errors.append(
|
|
383
|
+
f"{prefix}: parameter {param!r} not in schema for class {cls!r}"
|
|
384
|
+
+ _suggest(param, class_params)
|
|
385
|
+
)
|
|
386
|
+
continue
|
|
387
|
+
# 'entity' is always a list of element names — for 0-dim, a
|
|
388
|
+
# single-element list. This keeps the YAML uniform whether the
|
|
389
|
+
# class is 0-dim or multi-dim.
|
|
390
|
+
ent = row.get("entity")
|
|
391
|
+
if not isinstance(ent, list) or not all(isinstance(e, str) for e in ent):
|
|
392
|
+
errors.append(f"{prefix}: 'entity' must be a list of strings")
|
|
393
|
+
continue
|
|
394
|
+
dims = entity_classes[cls]
|
|
395
|
+
if dims and len(ent) != len(dims):
|
|
396
|
+
errors.append(
|
|
397
|
+
f"{prefix}: class {cls!r} has {len(dims)} dimensions, got {len(ent)} element(s)"
|
|
398
|
+
)
|
|
399
|
+
continue
|
|
400
|
+
if not dims and len(ent) != 1:
|
|
401
|
+
errors.append(
|
|
402
|
+
f"{prefix}: 0-dim class {cls!r} requires exactly one element in 'entity'"
|
|
403
|
+
)
|
|
404
|
+
continue
|
|
405
|
+
entity_key: tuple[str, Any] = (cls, tuple(ent) if dims else ent[0])
|
|
406
|
+
if entity_key not in available_entities:
|
|
407
|
+
display = _multi_dim_name(ent) if dims else ent[0]
|
|
408
|
+
errors.append(
|
|
409
|
+
f"{prefix}: entity {display!r} of class {cls!r} not found in target "
|
|
410
|
+
"(and not declared earlier in this delta)"
|
|
411
|
+
)
|
|
412
|
+
continue
|
|
413
|
+
# Default alternative to "Base" — that's the SpineDB convention
|
|
414
|
+
# and the name used by the FlexTool test fixtures.
|
|
415
|
+
alt = row.get("alternative", "Base")
|
|
416
|
+
if not isinstance(alt, str):
|
|
417
|
+
errors.append(f"{prefix}: 'alternative' must be a string")
|
|
418
|
+
continue
|
|
419
|
+
if alt not in available_alternatives:
|
|
420
|
+
errors.append(
|
|
421
|
+
f"{prefix}: alternative {alt!r} not found in target "
|
|
422
|
+
"(and not declared earlier in this delta)"
|
|
423
|
+
+ _suggest(alt, sorted(available_alternatives))
|
|
424
|
+
)
|
|
425
|
+
continue
|
|
426
|
+
value = row.get("value", ...)
|
|
427
|
+
if value is ...:
|
|
428
|
+
errors.append(f"{prefix}: missing required field 'value'")
|
|
429
|
+
continue
|
|
430
|
+
if isinstance(value, (dict, list)):
|
|
431
|
+
errors.append(
|
|
432
|
+
f"{prefix}: structured 'value' ({type(value).__name__}) is out of scope. "
|
|
433
|
+
"This script supports scalar values only (int, float, str, bool, None). "
|
|
434
|
+
"For time-series, maps, or arrays use the SpineDB editor."
|
|
435
|
+
)
|
|
436
|
+
continue
|
|
437
|
+
if not isinstance(value, _SCALAR_TYPES):
|
|
438
|
+
errors.append(
|
|
439
|
+
f"{prefix}: 'value' must be int/float/str/bool/None, got {type(value).__name__}"
|
|
440
|
+
)
|
|
441
|
+
continue
|
|
442
|
+
# SpineDB's parameter_value key. Use the element-list-joined
|
|
443
|
+
# name for multi-dim entities (the form db_to_json round-trips).
|
|
444
|
+
entity_name = _multi_dim_name(ent) if dims else ent[0]
|
|
445
|
+
pv_key = (cls, entity_name, param, alt)
|
|
446
|
+
if pv_key in src_param_values:
|
|
447
|
+
errors.append(
|
|
448
|
+
f"{prefix}: parameter_value ({cls}, {entity_name}, {param}, {alt}) "
|
|
449
|
+
"already exists in target (append-only — edits go through the SpineDB editor)"
|
|
450
|
+
)
|
|
451
|
+
continue
|
|
452
|
+
if pv_key in declared_parameter_values:
|
|
453
|
+
errors.append(
|
|
454
|
+
f"{prefix}: parameter_value ({cls}, {entity_name}, {param}, {alt}) "
|
|
455
|
+
"declared more than once in this delta"
|
|
456
|
+
)
|
|
457
|
+
continue
|
|
458
|
+
declared_parameter_values.add(pv_key)
|
|
459
|
+
|
|
460
|
+
# --- new_scenarios ---------------------------------------------------
|
|
461
|
+
for i, row in enumerate(delta.get("new_scenarios") or []):
|
|
462
|
+
prefix = f"new_scenarios[{i}]"
|
|
463
|
+
if not isinstance(row, dict):
|
|
464
|
+
errors.append(f"{prefix}: expected a mapping, got {type(row).__name__}")
|
|
465
|
+
continue
|
|
466
|
+
name = row.get("name")
|
|
467
|
+
if not isinstance(name, str):
|
|
468
|
+
errors.append(f"{prefix}: missing or non-string 'name'")
|
|
469
|
+
continue
|
|
470
|
+
if name in src_scenarios:
|
|
471
|
+
errors.append(
|
|
472
|
+
f"{prefix}: scenario {name!r} already exists in target (append-only)"
|
|
473
|
+
)
|
|
474
|
+
continue
|
|
475
|
+
if name in declared_scenarios:
|
|
476
|
+
errors.append(f"{prefix}: scenario {name!r} declared more than once in this delta")
|
|
477
|
+
continue
|
|
478
|
+
alts = row.get("alternatives")
|
|
479
|
+
if not isinstance(alts, list) or not alts or not all(isinstance(a, str) for a in alts):
|
|
480
|
+
errors.append(f"{prefix}: 'alternatives' must be a non-empty list of strings")
|
|
481
|
+
continue
|
|
482
|
+
for a in alts:
|
|
483
|
+
if a not in available_alternatives:
|
|
484
|
+
errors.append(
|
|
485
|
+
f"{prefix}: alternative {a!r} not found in target "
|
|
486
|
+
"(and not declared earlier in this delta)"
|
|
487
|
+
+ _suggest(a, sorted(available_alternatives))
|
|
488
|
+
)
|
|
489
|
+
break
|
|
490
|
+
else:
|
|
491
|
+
declared_scenarios.add(name)
|
|
492
|
+
continue
|
|
493
|
+
# Reached only when the for/else 'break' fired; do nothing more.
|
|
494
|
+
|
|
495
|
+
return errors
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
# ----------------------------------------------------------------------------
|
|
499
|
+
# Application
|
|
500
|
+
# ----------------------------------------------------------------------------
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _apply_delta_to_db(sqlite_url: str, delta: dict, entity_classes: dict[str, tuple[str, ...]]) -> None:
|
|
504
|
+
"""Apply the delta to an open SpineDB.
|
|
505
|
+
|
|
506
|
+
Splits apart from :func:`apply_delta` so the spinedb_api dependency
|
|
507
|
+
stays out of the validation hot path. All four sections share one
|
|
508
|
+
``DatabaseMapping`` context and one ``commit_session`` so a partial
|
|
509
|
+
failure rolls back cleanly.
|
|
510
|
+
"""
|
|
511
|
+
# Lazy import so module import doesn't pay the spinedb_api startup
|
|
512
|
+
# cost when only validation is requested.
|
|
513
|
+
from spinedb_api import DatabaseMapping, to_database
|
|
514
|
+
|
|
515
|
+
with DatabaseMapping(sqlite_url, create=False, upgrade=False) as db:
|
|
516
|
+
# --- entities ----------------------------------------------------
|
|
517
|
+
for row in delta.get("new_entities") or []:
|
|
518
|
+
cls = row["class"]
|
|
519
|
+
dims = entity_classes[cls]
|
|
520
|
+
if dims:
|
|
521
|
+
db.add_entity(
|
|
522
|
+
entity_class_name=cls,
|
|
523
|
+
element_name_list=tuple(row["entities"]),
|
|
524
|
+
description=row.get("description"),
|
|
525
|
+
)
|
|
526
|
+
else:
|
|
527
|
+
db.add_entity(
|
|
528
|
+
entity_class_name=cls,
|
|
529
|
+
name=row["name"],
|
|
530
|
+
description=row.get("description"),
|
|
531
|
+
)
|
|
532
|
+
# --- alternatives ------------------------------------------------
|
|
533
|
+
for row in delta.get("new_alternatives") or []:
|
|
534
|
+
db.add_alternative(
|
|
535
|
+
name=row["name"],
|
|
536
|
+
description=row.get("description", ""),
|
|
537
|
+
)
|
|
538
|
+
# --- parameter values --------------------------------------------
|
|
539
|
+
for row in delta.get("new_parameter_values") or []:
|
|
540
|
+
cls = row["class"]
|
|
541
|
+
dims = entity_classes[cls]
|
|
542
|
+
ent_elements = row["entity"]
|
|
543
|
+
value, type_ = to_database(row["value"])
|
|
544
|
+
db.add_parameter_value(
|
|
545
|
+
entity_class_name=cls,
|
|
546
|
+
# SpineDB needs the *byname* tuple — for 0-dim that's a
|
|
547
|
+
# one-element tuple, for multi-dim the element list.
|
|
548
|
+
entity_byname=tuple(ent_elements),
|
|
549
|
+
parameter_definition_name=row["parameter"],
|
|
550
|
+
alternative_name=row.get("alternative", "Base"),
|
|
551
|
+
value=value,
|
|
552
|
+
type=type_,
|
|
553
|
+
)
|
|
554
|
+
# --- scenarios + ranked scenario_alternatives --------------------
|
|
555
|
+
for row in delta.get("new_scenarios") or []:
|
|
556
|
+
name = row["name"]
|
|
557
|
+
db.add_scenario(name=name, description=row.get("description", ""))
|
|
558
|
+
# SpineDB stores scenario_alternatives as a linked list
|
|
559
|
+
# serialised by rank. We assign rank 1..N in declaration
|
|
560
|
+
# order — this matches what generate_canonical / the
|
|
561
|
+
# SpineDB editor write.
|
|
562
|
+
for rank, alt in enumerate(row["alternatives"], start=1):
|
|
563
|
+
db.add_scenario_alternative(
|
|
564
|
+
scenario_name=name,
|
|
565
|
+
alternative_name=alt,
|
|
566
|
+
rank=rank,
|
|
567
|
+
)
|
|
568
|
+
db.commit_session("extend_tests_fixture YAML delta")
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
# ----------------------------------------------------------------------------
|
|
572
|
+
# Recipe regeneration
|
|
573
|
+
# ----------------------------------------------------------------------------
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _regenerate_canonical_for(target_rel: str) -> list[str]:
|
|
577
|
+
"""Regenerate every canonical recipe whose ``source`` matches the target.
|
|
578
|
+
|
|
579
|
+
``target_rel`` is the repo-relative path of the modified fixture
|
|
580
|
+
(e.g. ``tests/fixtures/tests.json``). Returns the list of recipe
|
|
581
|
+
names that were regenerated, in declaration order.
|
|
582
|
+
|
|
583
|
+
No-op when no recipe references the fixture — many test fixtures
|
|
584
|
+
have no downstream canonical output (they're test-only).
|
|
585
|
+
"""
|
|
586
|
+
# Lazy import to keep the validation path cheap and to avoid the
|
|
587
|
+
# circular structure if generate_canonical were ever to import this
|
|
588
|
+
# module (it currently doesn't, but the lazy guard is cheap).
|
|
589
|
+
from flextool.update_flextool.generate_canonical import (
|
|
590
|
+
_load_recipes,
|
|
591
|
+
generate_one,
|
|
592
|
+
)
|
|
593
|
+
|
|
594
|
+
recipes = _load_recipes()
|
|
595
|
+
regenerated: list[str] = []
|
|
596
|
+
for name, recipe in recipes.items():
|
|
597
|
+
if recipe.get("source") == target_rel:
|
|
598
|
+
generate_one(name)
|
|
599
|
+
regenerated.append(name)
|
|
600
|
+
return regenerated
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
# ----------------------------------------------------------------------------
|
|
604
|
+
# Top-level API
|
|
605
|
+
# ----------------------------------------------------------------------------
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _resolve_target(delta: dict, target_path: Path | None) -> Path:
|
|
609
|
+
"""Resolve and validate the target fixture path.
|
|
610
|
+
|
|
611
|
+
Honours, in order: explicit ``target_path`` argument, then the
|
|
612
|
+
``target`` field from the YAML. The result is resolved against
|
|
613
|
+
the repo root when relative. Fails loudly if neither is supplied
|
|
614
|
+
or the file is missing.
|
|
615
|
+
"""
|
|
616
|
+
if target_path is not None:
|
|
617
|
+
candidate = target_path
|
|
618
|
+
else:
|
|
619
|
+
target_rel = delta.get("target")
|
|
620
|
+
if not target_rel:
|
|
621
|
+
raise ValueError(
|
|
622
|
+
"YAML delta is missing the 'target' field and no --target was given"
|
|
623
|
+
)
|
|
624
|
+
candidate = Path(target_rel)
|
|
625
|
+
if not candidate.is_absolute():
|
|
626
|
+
candidate = (_REPO_ROOT / candidate).resolve()
|
|
627
|
+
if not candidate.is_file():
|
|
628
|
+
raise FileNotFoundError(f"Target fixture not found: {candidate}")
|
|
629
|
+
return candidate
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def apply_delta(
|
|
633
|
+
yaml_path: Path,
|
|
634
|
+
target_path: Path | None = None,
|
|
635
|
+
regenerate_canonical: bool | None = None,
|
|
636
|
+
) -> int:
|
|
637
|
+
"""Apply a YAML delta to a ``tests/fixtures/*.json``.
|
|
638
|
+
|
|
639
|
+
Args:
|
|
640
|
+
yaml_path: Path to the YAML delta file.
|
|
641
|
+
target_path: If provided, overrides the YAML's ``target`` field.
|
|
642
|
+
regenerate_canonical: If not None, overrides the YAML's
|
|
643
|
+
``regenerate_canonical`` field.
|
|
644
|
+
|
|
645
|
+
Returns 0 on success, non-zero on validation failure (with
|
|
646
|
+
per-error diagnostics printed to stderr). All-or-nothing: the
|
|
647
|
+
SQLite is committed atomically, then re-exported over the source
|
|
648
|
+
JSON only if validation and apply both succeed.
|
|
649
|
+
"""
|
|
650
|
+
if not yaml_path.is_file():
|
|
651
|
+
print(f"YAML delta file not found: {yaml_path}", file=sys.stderr)
|
|
652
|
+
return 2
|
|
653
|
+
with open(yaml_path) as f:
|
|
654
|
+
delta = yaml.safe_load(f) or {}
|
|
655
|
+
|
|
656
|
+
target = _resolve_target(delta, target_path)
|
|
657
|
+
|
|
658
|
+
schema = _load_schema()
|
|
659
|
+
source_index = _index_source(target)
|
|
660
|
+
errors = validate_delta(delta, schema, source_index=source_index)
|
|
661
|
+
if errors:
|
|
662
|
+
print(f"Validation failed for {yaml_path} (target: {target}):", file=sys.stderr)
|
|
663
|
+
for e in errors:
|
|
664
|
+
print(f" - {e}", file=sys.stderr)
|
|
665
|
+
return 1
|
|
666
|
+
|
|
667
|
+
# Imported lazily — mirrors the pattern in generate_canonical.py.
|
|
668
|
+
from tests.db_utils import db_to_json, json_to_db
|
|
669
|
+
|
|
670
|
+
entity_classes = _index_schema(schema)["entity_classes"]
|
|
671
|
+
|
|
672
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
673
|
+
staging_sqlite = Path(tmp) / "staging.sqlite"
|
|
674
|
+
url = json_to_db(target, staging_sqlite)
|
|
675
|
+
_apply_delta_to_db(url, delta, entity_classes)
|
|
676
|
+
# Export over a staging path first, then move atomically over
|
|
677
|
+
# the source — protects against half-written JSON if db_to_json
|
|
678
|
+
# raises mid-write.
|
|
679
|
+
staging_json = Path(tmp) / "out.json"
|
|
680
|
+
db_to_json(staging_sqlite, staging_json)
|
|
681
|
+
shutil.copyfile(staging_json, target)
|
|
682
|
+
|
|
683
|
+
regen = regenerate_canonical if regenerate_canonical is not None else bool(
|
|
684
|
+
delta.get("regenerate_canonical")
|
|
685
|
+
)
|
|
686
|
+
if regen:
|
|
687
|
+
# Recipes are keyed by repo-relative paths. Recompute the
|
|
688
|
+
# relative form (the resolved Path may differ in case / symlink
|
|
689
|
+
# form on weird platforms) before lookup.
|
|
690
|
+
try:
|
|
691
|
+
target_rel = str(target.relative_to(_REPO_ROOT))
|
|
692
|
+
except ValueError:
|
|
693
|
+
print(
|
|
694
|
+
f"Cannot regenerate canonical: target {target} is outside repo root {_REPO_ROOT}",
|
|
695
|
+
file=sys.stderr,
|
|
696
|
+
)
|
|
697
|
+
return 3
|
|
698
|
+
regenerated = _regenerate_canonical_for(target_rel)
|
|
699
|
+
if regenerated:
|
|
700
|
+
print(f"Regenerated canonical: {regenerated}")
|
|
701
|
+
else:
|
|
702
|
+
print(
|
|
703
|
+
f"regenerate_canonical=true, but no recipe references {target_rel} — "
|
|
704
|
+
"nothing to regenerate"
|
|
705
|
+
)
|
|
706
|
+
|
|
707
|
+
print(f"Applied delta {yaml_path.name} -> {target}")
|
|
708
|
+
return 0
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
# ----------------------------------------------------------------------------
|
|
712
|
+
# CLI
|
|
713
|
+
# ----------------------------------------------------------------------------
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
def main(argv: list[str] | None = None) -> int:
|
|
717
|
+
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
718
|
+
parser.add_argument("yaml", type=Path, help="Path to the YAML delta file.")
|
|
719
|
+
parser.add_argument(
|
|
720
|
+
"--target",
|
|
721
|
+
type=Path,
|
|
722
|
+
default=None,
|
|
723
|
+
help="Override the YAML's 'target' field (repo-relative or absolute).",
|
|
724
|
+
)
|
|
725
|
+
parser.add_argument(
|
|
726
|
+
"--validate",
|
|
727
|
+
action="store_true",
|
|
728
|
+
help="Run validation only; do not write to the target fixture.",
|
|
729
|
+
)
|
|
730
|
+
parser.add_argument(
|
|
731
|
+
"--regenerate-canonical",
|
|
732
|
+
dest="regenerate_canonical",
|
|
733
|
+
action="store_true",
|
|
734
|
+
default=None,
|
|
735
|
+
help="Force regenerate-canonical (override YAML's regenerate_canonical field).",
|
|
736
|
+
)
|
|
737
|
+
args = parser.parse_args(argv)
|
|
738
|
+
|
|
739
|
+
if args.validate:
|
|
740
|
+
if not args.yaml.is_file():
|
|
741
|
+
print(f"YAML delta file not found: {args.yaml}", file=sys.stderr)
|
|
742
|
+
return 2
|
|
743
|
+
with open(args.yaml) as f:
|
|
744
|
+
delta = yaml.safe_load(f) or {}
|
|
745
|
+
schema = _load_schema()
|
|
746
|
+
source_index: dict | None = None
|
|
747
|
+
try:
|
|
748
|
+
target = _resolve_target(delta, args.target)
|
|
749
|
+
source_index = _index_source(target)
|
|
750
|
+
except (ValueError, FileNotFoundError) as e:
|
|
751
|
+
# Source not resolvable -> schema-only validation. Useful
|
|
752
|
+
# for editing a YAML without the repo checkout's target on
|
|
753
|
+
# disk (rare but supported).
|
|
754
|
+
print(f"NOTE: skipping source-level checks ({e})", file=sys.stderr)
|
|
755
|
+
errors = validate_delta(delta, schema, source_index=source_index)
|
|
756
|
+
if errors:
|
|
757
|
+
print(f"Validation failed for {args.yaml}:", file=sys.stderr)
|
|
758
|
+
for e in errors:
|
|
759
|
+
print(f" - {e}", file=sys.stderr)
|
|
760
|
+
return 1
|
|
761
|
+
print(f"OK: {args.yaml} validates against current schema")
|
|
762
|
+
return 0
|
|
763
|
+
|
|
764
|
+
return apply_delta(
|
|
765
|
+
yaml_path=args.yaml,
|
|
766
|
+
target_path=args.target,
|
|
767
|
+
regenerate_canonical=args.regenerate_canonical,
|
|
768
|
+
)
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
if __name__ == "__main__":
|
|
772
|
+
raise SystemExit(main())
|