flextool 4.1.1__tar.gz → 4.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (487) hide show
  1. {flextool-4.1.1 → flextool-4.2.2}/PKG-INFO +21 -4
  2. {flextool-4.1.1 → flextool-4.2.2}/README.md +15 -1
  3. flextool-4.2.2/flextool/__init__.py +100 -0
  4. flextool-4.2.2/flextool/_allocator.py +261 -0
  5. flextool-4.2.2/flextool/_thread_budget.py +303 -0
  6. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_cli.py +12 -0
  7. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_db_alt.py +3 -2
  8. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_loop.py +4 -0
  9. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_sizing.py +16 -16
  10. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_solve.py +51 -39
  11. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_solve_status.py +2 -2
  12. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_execute_flextool_workflow.py +29 -9
  13. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_open_results_db.py +2 -2
  14. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_read_self_describing_tabular_input.py +3 -2
  15. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_run_flextool.py +137 -135
  16. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_scenario_results.py +3 -2
  17. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_solve_mps.py +5 -0
  18. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_write_outputs.py +2 -1
  19. {flextool-4.1.1 → flextool-4.2.2}/flextool/decomposition/region_decomposition.py +7 -7
  20. {flextool-4.1.1 → flextool-4.2.2}/flextool/decomposition/region_filter.py +30 -23
  21. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine}/__init__.py +10 -10
  22. flextool-4.2.2/flextool/engine/_behaviour_changes.py +348 -0
  23. flextool-4.2.2/flextool/engine/_csv_reload/__init__.py +56 -0
  24. flextool-4.2.2/flextool/engine/_csv_reload/_field_seeds.py +267 -0
  25. flextool-4.2.2/flextool/engine/_csv_reload/_meta.py +33 -0
  26. flextool-4.2.2/flextool/engine/_csv_reload/_seed.py +58 -0
  27. flextool-4.2.2/flextool/engine/_csv_reload/_stamp.py +115 -0
  28. flextool-4.2.2/flextool/engine/_csv_reload/_topology.py +77 -0
  29. flextool-4.2.2/flextool/engine/_csv_reload/load.py +64 -0
  30. flextool-4.2.2/flextool/engine/_csv_reload/offline_outputs.py +185 -0
  31. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine}/autoscale/__init__.py +1 -1
  32. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine}/autoscale/_config.py +37 -60
  33. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine}/autoscale/_layer2.py +127 -80
  34. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine}/autoscale/_layer2_types.py +266 -135
  35. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine}/autoscale/_quantity_types.py +124 -94
  36. flextool-4.2.2/flextool/engine/input/__init__.py +1 -0
  37. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_axis_enums.py +97 -7
  38. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_db_loader.py +28 -14
  39. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_db_reader.py +1 -1
  40. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_inmemory_reader.py +49 -1
  41. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_input_source.py +30 -156
  42. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_param_shapes.py +1356 -61
  43. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_solve_config.py +42 -73
  44. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_solve_state.py +20 -10
  45. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_spinedb_reader.py +141 -13
  46. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_timeline.py +488 -153
  47. flextool-4.2.2/flextool/engine/input/_timeline_rules.py +160 -0
  48. flextool-4.2.2/flextool/engine/model/__init__.py +1 -0
  49. flextool-4.2.2/flextool/engine/model/_balance_terms.py +96 -0
  50. flextool-4.2.2/flextool/engine/model/_coarse_flows.py +521 -0
  51. flextool-4.2.2/flextool/engine/model/_commodity_ladder.py +648 -0
  52. flextool-4.2.2/flextool/engine/model/_common.py +420 -0
  53. flextool-4.2.2/flextool/engine/model/_cumulative_invest.py +935 -0
  54. flextool-4.2.2/flextool/engine/model/_dc_power_flow.py +332 -0
  55. flextool-4.2.2/flextool/engine/model/_delay.py +301 -0
  56. flextool-4.2.2/flextool/engine/model/_exact.py +155 -0
  57. flextool-4.2.2/flextool/engine/model/_group_slack.py +735 -0
  58. flextool-4.2.2/flextool/engine/model/_lead_time.py +161 -0
  59. flextool-4.2.2/flextool/engine/model/_non_anticipativity.py +377 -0
  60. flextool-4.2.2/flextool/engine/model/_online.py +1058 -0
  61. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/model}/_pdt_join.py +4 -4
  62. flextool-4.2.2/flextool/engine/model/_reserve.py +1504 -0
  63. flextool-4.2.2/flextool/engine/model/_self_discharge.py +479 -0
  64. flextool-4.2.2/flextool/engine/model/_unit_floors.py +519 -0
  65. flextool-4.2.2/flextool/engine/model/core.py +4925 -0
  66. flextool-4.2.2/flextool/engine/output/__init__.py +1 -0
  67. flextool-4.2.2/flextool/engine/output/_coarse_flows.py +183 -0
  68. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/output}/_dump_csvs.py +57 -277
  69. flextool-4.2.2/flextool/engine/output/_horizon_dump.py +274 -0
  70. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/output}/_parquet_bundle.py +14 -1
  71. flextool-4.2.2/flextool/engine/output/_prices.py +155 -0
  72. flextool-4.1.1/flextool/engine_polars/_output_writer.py → flextool-4.2.2/flextool/engine/output/_writer.py +152 -16
  73. flextool-4.2.2/flextool/engine/preprocess/__init__.py +1 -0
  74. flextool-4.2.2/flextool/engine/preprocess/_dump_stamp.py +109 -0
  75. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess}/_flex_data_provider.py +14 -5
  76. flextool-4.1.1/flextool/engine_polars/_native_input_writer.py → flextool-4.2.2/flextool/engine/preprocess/_input_writer.py +5 -9
  77. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess}/_invest_seeds.py +4 -4
  78. flextool-4.1.1/flextool/engine_polars/_emit_provider_io.py → flextool-4.2.2/flextool/engine/preprocess/_provider_io.py +1 -42
  79. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess}/_provider_keys.py +43 -7
  80. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess}/_provider_translators.py +67 -31
  81. flextool-4.2.2/flextool/engine/preprocess/_reserve_methods.py +80 -0
  82. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess}/_solve_context.py +122 -257
  83. flextool-4.2.2/flextool/engine/preprocess/emit/__init__.py +1 -0
  84. flextool-4.1.1/flextool/engine_polars/_emit_arc_unions.py → flextool-4.2.2/flextool/engine/preprocess/emit/_arc_unions.py +23 -14
  85. flextool-4.2.2/flextool/engine/preprocess/emit/_base_steps.py +74 -0
  86. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess/emit}/_blocks.py +109 -453
  87. flextool-4.1.1/flextool/engine_polars/_emit_calc_params.py → flextool-4.2.2/flextool/engine/preprocess/emit/_calc_params.py +25 -17
  88. flextool-4.1.1/flextool/engine_polars/_emit_chain_params.py → flextool-4.2.2/flextool/engine/preprocess/emit/_chain_params.py +74 -57
  89. flextool-4.1.1/flextool/engine_polars/_emit_co2_accumulators.py → flextool-4.2.2/flextool/engine/preprocess/emit/_co2_accumulators.py +177 -62
  90. flextool-4.1.1/flextool/engine_polars/_emit_dispatchers.py → flextool-4.2.2/flextool/engine/preprocess/emit/_dispatchers.py +3 -3
  91. flextool-4.1.1/flextool/engine_polars/_emit_energy_margin.py → flextool-4.2.2/flextool/engine/preprocess/emit/_energy_margin.py +47 -6
  92. flextool-4.1.1/flextool/engine_polars/_emit_energy_margin_adder.py → flextool-4.2.2/flextool/engine/preprocess/emit/_energy_margin_adder.py +22 -21
  93. flextool-4.1.1/flextool/engine_polars/_emit_entity_annual.py → flextool-4.2.2/flextool/engine/preprocess/emit/_entity_annual.py +34 -13
  94. flextool-4.1.1/flextool/engine_polars/_emit_inflow_scaling.py → flextool-4.2.2/flextool/engine/preprocess/emit/_inflow_scaling.py +244 -145
  95. flextool-4.2.2/flextool/engine/preprocess/emit/_inflow_scaling_checks.py +275 -0
  96. flextool-4.1.1/flextool/engine_polars/_emit_leaf_sets.py → flextool-4.2.2/flextool/engine/preprocess/emit/_leaf_sets.py +35 -14
  97. flextool-4.1.1/flextool/engine_polars/_emit_lp_scaling.py → flextool-4.2.2/flextool/engine/preprocess/emit/_lp_scaling.py +7 -7
  98. flextool-4.1.1/flextool/engine_polars/_emit_mid_sets.py → flextool-4.2.2/flextool/engine/preprocess/emit/_mid_sets.py +16 -23
  99. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess/emit}/_pdt_lookup.py +231 -6
  100. flextool-4.1.1/flextool/engine_polars/_emit_pdt_params.py → flextool-4.2.2/flextool/engine/preprocess/emit/_pdt_params.py +34 -6
  101. flextool-4.1.1/flextool/engine_polars/_emit_per_solve.py → flextool-4.2.2/flextool/engine/preprocess/emit/_per_solve.py +160 -118
  102. flextool-4.1.1/flextool/engine_polars/_emit_period_calc.py → flextool-4.2.2/flextool/engine/preprocess/emit/_period_calc.py +48 -23
  103. flextool-4.1.1/flextool/engine_polars/_emit_period_params.py → flextool-4.2.2/flextool/engine/preprocess/emit/_period_params.py +125 -25
  104. flextool-4.1.1/flextool/engine_polars/_emit_reserve.py → flextool-4.2.2/flextool/engine/preprocess/emit/_reserve_sets.py +75 -12
  105. flextool-4.1.1/flextool/engine_polars/_emit_solve_time.py → flextool-4.2.2/flextool/engine/preprocess/emit/_solve_time.py +22 -26
  106. flextool-4.1.1/flextool/engine_polars/_emit_solve_writers.py → flextool-4.2.2/flextool/engine/preprocess/emit/_solve_writers.py +248 -199
  107. flextool-4.1.1/flextool/engine_polars/input.py → flextool-4.2.2/flextool/engine/preprocess/flex_data.py +2512 -1874
  108. flextool-4.2.2/flextool/engine/preprocess/params/__init__.py +1 -0
  109. flextool-4.1.1/flextool/engine_polars/_derived_arithmetic.py → flextool-4.2.2/flextool/engine/preprocess/params/_arithmetic.py +193 -78
  110. flextool-4.2.2/flextool/engine/preprocess/params/_block.py +346 -0
  111. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess/params}/_block_layout.py +70 -447
  112. flextool-4.1.1/flextool/engine_polars/_derived_branch.py → flextool-4.2.2/flextool/engine/preprocess/params/_branch.py +414 -102
  113. flextool-4.2.2/flextool/engine/preprocess/params/_coarse.py +458 -0
  114. flextool-4.1.1/flextool/engine_polars/_derived_params.py → flextool-4.2.2/flextool/engine/preprocess/params/_derived.py +2000 -2675
  115. flextool-4.1.1/flextool/engine_polars/_direct_params.py → flextool-4.2.2/flextool/engine/preprocess/params/_direct.py +471 -164
  116. flextool-4.1.1/flextool/engine_polars/_derived_existing.py → flextool-4.2.2/flextool/engine/preprocess/params/_existing.py +522 -346
  117. flextool-4.2.2/flextool/engine/preprocess/params/_mintime.py +330 -0
  118. flextool-4.1.1/flextool/engine_polars/_derived_npv.py → flextool-4.2.2/flextool/engine/preprocess/params/_npv.py +764 -129
  119. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess/params}/_per_solve_sets.py +188 -54
  120. flextool-4.1.1/flextool/engine_polars/_derived_profile.py → flextool-4.2.2/flextool/engine/preprocess/params/_profiles.py +115 -59
  121. flextool-4.1.1/flextool/engine_polars/_projection_params.py → flextool-4.2.2/flextool/engine/preprocess/params/_projection.py +192 -229
  122. flextool-4.2.2/flextool/engine/preprocess/params/_ramp_links.py +258 -0
  123. flextool-4.2.2/flextool/engine/preprocess/params/_retire_existing.py +235 -0
  124. flextool-4.2.2/flextool/engine/preprocess/params/_run_calendar.py +140 -0
  125. flextool-4.1.1/flextool/engine_polars/_derived_walks.py → flextool-4.2.2/flextool/engine/preprocess/params/_walks.py +36 -35
  126. flextool-4.2.2/flextool/engine/solve/__init__.py +1 -0
  127. flextool-4.1.1/flextool/engine_polars/_solve_acceptance.py → flextool-4.2.2/flextool/engine/solve/_acceptance.py +80 -17
  128. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/_benders.py +225 -209
  129. flextool-4.2.2/flextool/engine/solve/_carry_routing.py +400 -0
  130. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/_determinism.py +15 -8
  131. flextool-4.1.1/flextool/engine_polars/_solve_handoff.py → flextool-4.2.2/flextool/engine/solve/_handoff.py +67 -30
  132. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/_orchestration.py +906 -567
  133. flextool-4.2.2/flextool/engine/solve/_realization.py +381 -0
  134. flextool-4.2.2/flextool/engine/solve/_realized_overlap.py +279 -0
  135. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/_recursive_solve.py +4 -4
  136. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/_region_filter.py +166 -219
  137. flextool-4.1.1/flextool/engine_polars/_native_run_model.py → flextool-4.2.2/flextool/engine/solve/_run_model.py +489 -232
  138. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/_solver_base.py +1 -1
  139. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/_solver_dispatch.py +65 -40
  140. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/_stochastic.py +76 -40
  141. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/_subprocess_solve.py +64 -25
  142. flextool-4.2.2/flextool/engine/solve/_warm.py +523 -0
  143. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/solve}/chain.py +30 -41
  144. flextool-4.2.2/flextool/engine_polars/__init__.py +14 -0
  145. flextool-4.2.2/flextool/env_check.py +1176 -0
  146. {flextool-4.1.1 → flextool-4.2.2}/flextool/export_to_tabular/db_reader.py +16 -0
  147. {flextool-4.1.1 → flextool-4.2.2}/flextool/export_to_tabular/excel_writer.py +13 -2
  148. {flextool-4.1.1 → flextool-4.2.2}/flextool/export_to_tabular/export_settings.yaml +29 -8
  149. {flextool-4.1.1 → flextool-4.2.2}/flextool/export_to_tabular/export_to_excel.py +2 -1
  150. {flextool-4.1.1 → flextool-4.2.2}/flextool/export_to_tabular/sheet_config.py +25 -1
  151. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/calibrate_commands.py +6 -0
  152. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/calibrate_jobs.py +1 -0
  153. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/data_models.py +20 -34
  154. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/db_version_check.py +6 -5
  155. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/calibrate_dialog.py +6 -0
  156. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/plot_settings_picker.py +1 -1
  157. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/execution_manager.py +17 -27
  158. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/execution_window.py +11 -2
  159. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/input_sources.py +2 -2
  160. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/main_window.py +144 -171
  161. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/result_viewer.py +27 -81
  162. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/settings_io.py +52 -15
  163. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/solve_reader.py +3 -3
  164. {flextool-4.1.1 → flextool-4.2.2}/flextool/input_derivation/__init__.py +57 -17
  165. {flextool-4.1.1 → flextool-4.2.2}/flextool/input_derivation/_commodity_ladder.py +59 -48
  166. {flextool-4.1.1 → flextool-4.2.2}/flextool/input_derivation/_commodity_ladder_sets.py +1 -1
  167. {flextool-4.1.1 → flextool-4.2.2}/flextool/input_derivation/_dc_power_flow.py +44 -5
  168. {flextool-4.1.1 → flextool-4.2.2}/flextool/input_derivation/_method_constants.py +2 -2
  169. {flextool-4.1.1 → flextool-4.2.2}/flextool/input_derivation/_specs.py +133 -9
  170. {flextool-4.1.1 → flextool-4.2.2}/flextool/input_derivation/_validators.py +359 -1
  171. {flextool-4.1.1 → flextool-4.2.2}/flextool/model_builder/build_model.py +3 -2
  172. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/format_helpers.py +146 -72
  173. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/orchestrator.py +8 -4
  174. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/plan.py +14 -8
  175. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_inputs/read_matpower.py +3 -2
  176. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_inputs/read_self_describing_excel.py +24 -0
  177. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_inputs/write_old_flextool_to_db.py +5 -4
  178. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_inputs/write_self_describing_to_db.py +58 -3
  179. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_inputs/write_to_input_db.py +3 -2
  180. flextool-4.2.2/flextool/process_outputs/_fan_labels.py +62 -0
  181. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/_inmemory_helpers.py +2 -2
  182. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/_output_meta.py +22 -7
  183. flextool-4.2.2/flextool/process_outputs/_timeline_breaks.py +73 -0
  184. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/calc_capacity_flows.py +14 -23
  185. flextool-4.2.2/flextool/process_outputs/calc_costs.py +501 -0
  186. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/calc_slacks.py +10 -24
  187. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/calc_storage_vre.py +43 -41
  188. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/drop_levels.py +153 -25
  189. flextool-4.2.2/flextool/process_outputs/handoff_writers.py +886 -0
  190. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/out_ancillary.py +92 -54
  191. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/out_capacity.py +45 -34
  192. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/out_costs.py +83 -19
  193. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/out_flows.py +29 -5
  194. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/out_node.py +4 -4
  195. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/persist_realized_slice.py +75 -13
  196. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/read_highs_solution.py +581 -1011
  197. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/read_parameters.py +214 -217
  198. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/read_sets.py +253 -23
  199. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/read_variables.py +81 -224
  200. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/solve_order.py +13 -29
  201. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/write_outputs.py +132 -35
  202. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/write_spinedb.py +15 -5
  203. {flextool-4.1.1 → flextool-4.2.2}/flextool/representative_periods/netload_inputs.py +34 -7
  204. {flextool-4.1.1 → flextool-4.2.2}/flextool/representative_periods/netload_iterate.py +7 -6
  205. {flextool-4.1.1 → flextool-4.2.2}/flextool/representative_periods/preprocess.py +5 -4
  206. {flextool-4.1.1 → flextool-4.2.2}/flextool/representative_periods/scenario_stack.py +4 -4
  207. flextool-4.2.2/flextool/run_options.py +478 -0
  208. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/data_models.py +3 -0
  209. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/db_reader.py +7 -16
  210. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/dispatch_data.py +1 -1
  211. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/dispatch_plots.py +6 -6
  212. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/orchestrator.py +4 -5
  213. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/AXIS_CONTRACT.md +13 -1
  214. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/howto_aggregate_output.json +1046 -286
  215. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/howto_connections.json +1026 -306
  216. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/howto_demand.json +1068 -343
  217. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/howto_hydro_reservoir.json +1045 -325
  218. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +1048 -328
  219. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +1041 -289
  220. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +1026 -306
  221. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/howto_stochastics.json +1055 -283
  222. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/templates_examples.json +1095 -273
  223. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/canonical_databases/templates_time_settings_only.json +1042 -289
  224. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/default_plot_settings.yaml +11 -0
  225. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/default_plots.yaml +80 -6
  226. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/flextool_axis_contract.json +37 -36
  227. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/spinedb_results_schema.json +27 -3
  228. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/spinedb_schema.json +598 -120
  229. {flextool-4.1.1 → flextool-4.2.2}/flextool/solver_config/copt.opt.template +1 -1
  230. {flextool-4.1.1 → flextool-4.2.2}/flextool/solver_config/cplex.opt.template +1 -1
  231. {flextool-4.1.1 → flextool-4.2.2}/flextool/solver_config/gurobi.opt.template +1 -1
  232. {flextool-4.1.1 → flextool-4.2.2}/flextool/solver_config/xpress.opt.template +1 -1
  233. {flextool-4.1.1 → flextool-4.2.2}/flextool/spinedb_backend/__init__.py +1 -1
  234. {flextool-4.1.1 → flextool-4.2.2}/flextool/spinedb_backend/_axis_enums.py +61 -58
  235. {flextool-4.1.1 → flextool-4.2.2}/flextool/spinedb_backend/_backend.py +43 -78
  236. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/__init__.py +1 -1
  237. flextool-4.2.2/flextool/update_flextool/bundle_install.py +181 -0
  238. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/canonical_databases.py +64 -37
  239. flextool-4.2.2/flextool/update_flextool/db_handle.py +26 -0
  240. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/db_migration.py +1234 -42
  241. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/export_database.py +10 -2
  242. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/extend_tests_fixture.py +6 -4
  243. flextool-4.2.2/flextool/update_flextool/fixture_schema.py +307 -0
  244. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/generate_canonical.py +4 -4
  245. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/initialize_database.py +12 -2
  246. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/install_info.py +15 -1
  247. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/self_update.py +66 -26
  248. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/sync_master_json_template.py +9 -8
  249. flextool-4.2.2/flextool/update_flextool/test_fixtures.py +229 -0
  250. {flextool-4.1.1 → flextool-4.2.2}/flextool.egg-info/PKG-INFO +21 -4
  251. {flextool-4.1.1 → flextool-4.2.2}/flextool.egg-info/SOURCES.txt +146 -89
  252. {flextool-4.1.1 → flextool-4.2.2}/flextool.egg-info/requires.txt +5 -2
  253. {flextool-4.1.1 → flextool-4.2.2}/pyproject.toml +38 -10
  254. {flextool-4.1.1 → flextool-4.2.2}/tests/test_blocks.py +24 -218
  255. {flextool-4.1.1 → flextool-4.2.2}/tests/test_branch_weights.py +3 -3
  256. flextool-4.2.2/tests/test_child_run_argv.py +92 -0
  257. flextool-4.2.2/tests/test_ci_shard.py +66 -0
  258. {flextool-4.1.1 → flextool-4.2.2}/tests/test_commodity_ladder_rolling.py +62 -31
  259. {flextool-4.1.1 → flextool-4.2.2}/tests/test_cost_aggregation_semantics.py +128 -78
  260. {flextool-4.1.1 → flextool-4.2.2}/tests/test_cost_breakdown_by_entity.py +1 -1
  261. flextool-4.2.2/tests/test_csv_dump_roundtrip.py +276 -0
  262. {flextool-4.1.1 → flextool-4.2.2}/tests/test_dc_power_flow.py +13 -3
  263. flextool-4.2.2/tests/test_env_check.py +991 -0
  264. flextool-4.2.2/tests/test_env_knob_registry.py +188 -0
  265. {flextool-4.1.1 → flextool-4.2.2}/tests/test_export_to_tabular.py +4 -1
  266. flextool-4.2.2/tests/test_handoff_writers.py +434 -0
  267. {flextool-4.1.1 → flextool-4.2.2}/tests/test_lh2_three_region.py +42 -138
  268. {flextool-4.1.1 → flextool-4.2.2}/tests/test_min_load_rolling.py +9 -13
  269. {flextool-4.1.1 → flextool-4.2.2}/tests/test_non_anticipativity.py +22 -20
  270. {flextool-4.1.1 → flextool-4.2.2}/tests/test_output_column_metadata.py +1 -1
  271. flextool-4.2.2/tests/test_output_csv_names.py +67 -0
  272. flextool-4.2.2/tests/test_output_work_folder_isolation.py +321 -0
  273. {flextool-4.1.1 → flextool-4.2.2}/tests/test_read_highs_solution.py +182 -33
  274. {flextool-4.1.1 → flextool-4.2.2}/tests/test_recourse_cost_reconciliation.py +18 -26
  275. {flextool-4.1.1 → flextool-4.2.2}/tests/test_regional_filter.py +51 -3
  276. {flextool-4.1.1 → flextool-4.2.2}/tests/test_rolling_online_carry.py +70 -50
  277. {flextool-4.1.1 → flextool-4.2.2}/tests/test_rolling_online_mintime_carry.py +146 -28
  278. {flextool-4.1.1 → flextool-4.2.2}/tests/test_rolling_online_minuptime.py +21 -14
  279. flextool-4.2.2/tests/test_run_options.py +224 -0
  280. {flextool-4.1.1 → flextool-4.2.2}/tests/test_scenarios.py +167 -23
  281. {flextool-4.1.1 → flextool-4.2.2}/tests/test_solve_config.py +1 -1
  282. flextool-4.2.2/tests/test_stream_concat_temp_cleanup.py +50 -0
  283. flextool-4.2.2/tests/test_support_load.py +210 -0
  284. {flextool-4.1.1 → flextool-4.2.2}/tests/test_synthesize_invest_dual.py +25 -19
  285. flextool-4.2.2/tests/test_thread_budget.py +175 -0
  286. flextool-4.2.2/tests/test_timeline_breaks.py +208 -0
  287. {flextool-4.1.1 → flextool-4.2.2}/tests/test_timings_csv.py +11 -6
  288. {flextool-4.1.1 → flextool-4.2.2}/tests/test_unidirectional_connection.py +1 -2
  289. {flextool-4.1.1 → flextool-4.2.2}/tests/test_unit_online_dt_columns.py +5 -5
  290. {flextool-4.1.1 → flextool-4.2.2}/tests/test_v67_migration.py +1 -1
  291. {flextool-4.1.1 → flextool-4.2.2}/tests/test_workdir_empty_default.py +3 -3
  292. flextool-4.2.2/tests/test_xlsx_v77_round_trip.py +154 -0
  293. {flextool-4.1.1 → flextool-4.2.2}/tests/test_years_represented.py +1 -1
  294. flextool-4.1.1/flextool/__init__.py +0 -41
  295. flextool-4.1.1/flextool/common_utils/precision.py +0 -451
  296. flextool-4.1.1/flextool/engine_polars/_commodity_ladder.py +0 -660
  297. flextool-4.1.1/flextool/engine_polars/_cumulative_invest.py +0 -1220
  298. flextool-4.1.1/flextool/engine_polars/_dc_power_flow.py +0 -445
  299. flextool-4.1.1/flextool/engine_polars/_delay.py +0 -552
  300. flextool-4.1.1/flextool/engine_polars/_derived_block.py +0 -990
  301. flextool-4.1.1/flextool/engine_polars/_flex_data_accumulator.py +0 -389
  302. flextool-4.1.1/flextool/engine_polars/_group_slack.py +0 -1253
  303. flextool-4.1.1/flextool/engine_polars/_reserve.py +0 -1073
  304. flextool-4.1.1/flextool/engine_polars/_warm.py +0 -880
  305. flextool-4.1.1/flextool/engine_polars/model.py +0 -6293
  306. flextool-4.1.1/flextool/env_check.py +0 -388
  307. flextool-4.1.1/flextool/process_outputs/_branch_weight.py +0 -48
  308. flextool-4.1.1/flextool/process_outputs/calc_costs.py +0 -310
  309. flextool-4.1.1/flextool/process_outputs/handoff_writers.py +0 -1315
  310. flextool-4.1.1/flextool/update_flextool/test_fixtures.py +0 -187
  311. flextool-4.1.1/tests/test_csv_dump_roundtrip.py +0 -131
  312. flextool-4.1.1/tests/test_env_check.py +0 -156
  313. flextool-4.1.1/tests/test_handoff_writers.py +0 -503
  314. flextool-4.1.1/tests/test_precision.py +0 -226
  315. {flextool-4.1.1 → flextool-4.2.2}/LICENSE.txt +0 -0
  316. {flextool-4.1.1 → flextool-4.2.2}/flextool/_mem_sampler.py +0 -0
  317. {flextool-4.1.1 → flextool-4.2.2}/flextool/_resources.py +0 -0
  318. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/__init__.py +0 -0
  319. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/__main__.py +0 -0
  320. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_final_outputs.py +0 -0
  321. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_guard.py +0 -0
  322. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_readers.py +0 -0
  323. {flextool-4.1.1 → flextool-4.2.2}/flextool/calibrate/_report.py +0 -0
  324. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/__init__.py +0 -0
  325. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/_console.py +0 -0
  326. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/_timing.py +0 -0
  327. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_export_to_tabular.py +0 -0
  328. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_import_sensitivities.py +0 -0
  329. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_migrate_database.py +0 -0
  330. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_read_matpower.py +0 -0
  331. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_read_old_flextool.py +0 -0
  332. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_read_tabular_input.py +0 -0
  333. {flextool-4.1.1 → flextool-4.2.2}/flextool/cli/cmd_update_flextool.py +0 -0
  334. {flextool-4.1.1 → flextool-4.2.2}/flextool/common_utils/__init__.py +0 -0
  335. {flextool-4.1.1 → flextool-4.2.2}/flextool/common_utils/plot_mem_shape.py +0 -0
  336. {flextool-4.1.1 → flextool-4.2.2}/flextool/decomposition/__init__.py +0 -0
  337. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine}/autoscale/_report.py +0 -0
  338. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/input}/_constants.py +0 -0
  339. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess}/_stochastic_detect.py +0 -0
  340. {flextool-4.1.1/flextool/engine_polars → flextool-4.2.2/flextool/engine/preprocess/emit}/_vectorize.py +0 -0
  341. {flextool-4.1.1 → flextool-4.2.2}/flextool/export_to_tabular/__init__.py +0 -0
  342. {flextool-4.1.1 → flextool-4.2.2}/flextool/export_to_tabular/formatting.py +0 -0
  343. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/__init__.py +0 -0
  344. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/__main__.py +0 -0
  345. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/check_tree.py +0 -0
  346. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/cli_format.py +0 -0
  347. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/config_parser.py +0 -0
  348. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/db_editor_integration.py +0 -0
  349. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/__init__.py +0 -0
  350. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/add_dialog.py +0 -0
  351. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/file_picker.py +0 -0
  352. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/group_picker.py +0 -0
  353. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/migration_consent_dialog.py +0 -0
  354. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/migration_progress_dialog.py +0 -0
  355. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/plot_dialog.py +0 -0
  356. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/project_dialog.py +0 -0
  357. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/dialogs/update_dialog.py +0 -0
  358. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/downsampling.py +0 -0
  359. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/error_handling.py +0 -0
  360. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/hover_tooltip.py +0 -0
  361. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/network_graph.py +0 -0
  362. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/output_actions.py +0 -0
  363. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/output_log_window.py +0 -0
  364. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/platform_utils.py +0 -0
  365. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/plot_cache.py +0 -0
  366. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/plot_canvas.py +0 -0
  367. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/plot_config_reader.py +0 -0
  368. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/project_utils.py +0 -0
  369. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/scenario_key.py +0 -0
  370. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/scenario_lists.py +0 -0
  371. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/tree_reorder.py +0 -0
  372. {flextool-4.1.1 → flextool-4.2.2}/flextool/gui/ui_metrics.py +0 -0
  373. {flextool-4.1.1 → flextool-4.2.2}/flextool/input_derivation/_process_method.py +0 -0
  374. {flextool-4.1.1 → flextool-4.2.2}/flextool/lean_parquet.py +0 -0
  375. {flextool-4.1.1 → flextool-4.2.2}/flextool/model_builder/__init__.py +0 -0
  376. {flextool-4.1.1 → flextool-4.2.2}/flextool/model_builder/encoding.py +0 -0
  377. {flextool-4.1.1 → flextool-4.2.2}/flextool/model_builder/names.py +0 -0
  378. {flextool-4.1.1 → flextool-4.2.2}/flextool/model_builder/profiles.py +0 -0
  379. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/__init__.py +0 -0
  380. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/axis_helpers.py +0 -0
  381. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/color_template.py +0 -0
  382. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/config.py +0 -0
  383. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/legend_helpers.py +0 -0
  384. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/perf.py +0 -0
  385. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/plot_bars.py +0 -0
  386. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/plot_bars_detail.py +0 -0
  387. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/plot_lines.py +0 -0
  388. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/shared_manifest.py +0 -0
  389. {flextool-4.1.1 → flextool-4.2.2}/flextool/plot_outputs/subplot_helpers.py +0 -0
  390. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_inputs/__init__.py +0 -0
  391. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_inputs/import_old_excel_input.json +0 -0
  392. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_inputs/read_old_flextool.py +0 -0
  393. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_inputs/read_tabular_with_specification.py +0 -0
  394. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/__init__.py +0 -0
  395. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/_annualize.py +0 -0
  396. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/calc_connections.py +0 -0
  397. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/calc_group_flows.py +0 -0
  398. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/out_flowgroup.py +0 -0
  399. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/out_group.py +0 -0
  400. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/process_results.py +0 -0
  401. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/spinedb_replay.py +0 -0
  402. {flextool-4.1.1 → flextool-4.2.2}/flextool/process_outputs/union_realized_slice.py +0 -0
  403. {flextool-4.1.1 → flextool-4.2.2}/flextool/representative_periods/__init__.py +0 -0
  404. {flextool-4.1.1 → flextool-4.2.2}/flextool/representative_periods/clustering.py +0 -0
  405. {flextool-4.1.1 → flextool-4.2.2}/flextool/representative_periods/force_include.py +0 -0
  406. {flextool-4.1.1 → flextool-4.2.2}/flextool/representative_periods/netload.py +0 -0
  407. {flextool-4.1.1 → flextool-4.2.2}/flextool/representative_periods/weights.py +0 -0
  408. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/__init__.py +0 -0
  409. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/config_builder.py +0 -0
  410. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/constants.py +0 -0
  411. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/dispatch_mappings.py +0 -0
  412. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/input_entity_colors.py +0 -0
  413. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/plan_union.py +0 -0
  414. {flextool-4.1.1 → flextool-4.2.2}/flextool/scenario_comparison/plot_settings_seed.py +0 -0
  415. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/comparison_settings_template.json +0 -0
  416. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/flextool_axis_contract.schema.json +0 -0
  417. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/old_flextool_import_template.json +0 -0
  418. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/output_info_template.json +0 -0
  419. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/output_settings_template.json +0 -0
  420. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_constant_default.json +0 -0
  421. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_default_optional_output.json +0 -0
  422. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_default_value.json +0 -0
  423. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_drop_down.json +0 -0
  424. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_lifetime_method.json +0 -0
  425. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_optional_outputs.json +0 -0
  426. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_output_node_flows.json +0 -0
  427. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_results_master.json +0 -0
  428. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +0 -0
  429. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_rolling_window.json +0 -0
  430. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +0 -0
  431. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_v2.json +0 -0
  432. {flextool-4.1.1 → flextool-4.2.2}/flextool/schemas/pre_v26/flextool_template_v25.json +0 -0
  433. {flextool-4.1.1 → flextool-4.2.2}/flextool/solver_config/highs.opt.template +0 -0
  434. {flextool-4.1.1 → flextool-4.2.2}/flextool/update_flextool/ensure_settings_db.py +0 -0
  435. {flextool-4.1.1 → flextool-4.2.2}/flextool.egg-info/dependency_links.txt +0 -0
  436. {flextool-4.1.1 → flextool-4.2.2}/flextool.egg-info/entry_points.txt +0 -0
  437. {flextool-4.1.1 → flextool-4.2.2}/flextool.egg-info/top_level.txt +0 -0
  438. {flextool-4.1.1 → flextool-4.2.2}/setup.cfg +0 -0
  439. {flextool-4.1.1 → flextool-4.2.2}/tests/test_bar_layout.py +0 -0
  440. {flextool-4.1.1 → flextool-4.2.2}/tests/test_bar_sub_pixel_labels.py +0 -0
  441. {flextool-4.1.1 → flextool-4.2.2}/tests/test_color_template.py +0 -0
  442. {flextool-4.1.1 → flextool-4.2.2}/tests/test_comparison_excel.py +0 -0
  443. {flextool-4.1.1 → flextool-4.2.2}/tests/test_comparison_project_root.py +0 -0
  444. {flextool-4.1.1 → flextool-4.2.2}/tests/test_complete_period_share_reduction.py +0 -0
  445. {flextool-4.1.1 → flextool-4.2.2}/tests/test_connection_node_warnings.py +0 -0
  446. {flextool-4.1.1 → flextool-4.2.2}/tests/test_console_run_tool.py +0 -0
  447. {flextool-4.1.1 → flextool-4.2.2}/tests/test_dispatch_consumer_and_connection_signs.py +0 -0
  448. {flextool-4.1.1 → flextool-4.2.2}/tests/test_dispatch_node_debug_and_tree.py +0 -0
  449. {flextool-4.1.1 → flextool-4.2.2}/tests/test_dispatch_order_specials.py +0 -0
  450. {flextool-4.1.1 → flextool-4.2.2}/tests/test_dispatch_scenario_tag.py +0 -0
  451. {flextool-4.1.1 → flextool-4.2.2}/tests/test_ensure_settings_db.py +0 -0
  452. {flextool-4.1.1 → flextool-4.2.2}/tests/test_entity_unitsize_invariant.py +0 -0
  453. {flextool-4.1.1 → flextool-4.2.2}/tests/test_entity_universe_carrier.py +0 -0
  454. {flextool-4.1.1 → flextool-4.2.2}/tests/test_extend_tests_fixture.py +0 -0
  455. {flextool-4.1.1 → flextool-4.2.2}/tests/test_flowgroup_indicators.py +0 -0
  456. {flextool-4.1.1 → flextool-4.2.2}/tests/test_force_include.py +0 -0
  457. {flextool-4.1.1 → flextool-4.2.2}/tests/test_group_output_warnings.py +0 -0
  458. {flextool-4.1.1 → flextool-4.2.2}/tests/test_input_entity_colors.py +0 -0
  459. {flextool-4.1.1 → flextool-4.2.2}/tests/test_old_flextool_importer_schema_drift.py +0 -0
  460. {flextool-4.1.1 → flextool-4.2.2}/tests/test_open_results_db.py +0 -0
  461. {flextool-4.1.1 → flextool-4.2.2}/tests/test_output_metadata_renderers.py +0 -0
  462. {flextool-4.1.1 → flextool-4.2.2}/tests/test_parameter_group_coverage.py +0 -0
  463. {flextool-4.1.1 → flextool-4.2.2}/tests/test_plan_union.py +0 -0
  464. {flextool-4.1.1 → flextool-4.2.2}/tests/test_plot_draw_order.py +0 -0
  465. {flextool-4.1.1 → flextool-4.2.2}/tests/test_plot_pipeline.py +0 -0
  466. {flextool-4.1.1 → flextool-4.2.2}/tests/test_plot_plan_chunk_reconstruction.py +0 -0
  467. {flextool-4.1.1 → flextool-4.2.2}/tests/test_plot_plan_time_labels_roundtrip.py +0 -0
  468. {flextool-4.1.1 → flextool-4.2.2}/tests/test_plot_settings_seed.py +0 -0
  469. {flextool-4.1.1 → flextool-4.2.2}/tests/test_representative_periods.py +0 -0
  470. {flextool-4.1.1 → flextool-4.2.2}/tests/test_rp_alternative_name.py +0 -0
  471. {flextool-4.1.1 → flextool-4.2.2}/tests/test_rp_force_include_wiring.py +0 -0
  472. {flextool-4.1.1 → flextool-4.2.2}/tests/test_rp_solves_filter.py +0 -0
  473. {flextool-4.1.1 → flextool-4.2.2}/tests/test_scenario_colors.py +0 -0
  474. {flextool-4.1.1 → flextool-4.2.2}/tests/test_scenario_stack.py +0 -0
  475. {flextool-4.1.1 → flextool-4.2.2}/tests/test_self_describing_reader.py +0 -0
  476. {flextool-4.1.1 → flextool-4.2.2}/tests/test_shared_axis_manifest.py +0 -0
  477. {flextool-4.1.1 → flextool-4.2.2}/tests/test_simple_bar_entity_color.py +0 -0
  478. {flextool-4.1.1 → flextool-4.2.2}/tests/test_solve_reader.py +0 -0
  479. {flextool-4.1.1 → flextool-4.2.2}/tests/test_v52_migration.py +0 -0
  480. {flextool-4.1.1 → flextool-4.2.2}/tests/test_v63_migration.py +0 -0
  481. {flextool-4.1.1 → flextool-4.2.2}/tests/test_v64_migration.py +0 -0
  482. {flextool-4.1.1 → flextool-4.2.2}/tests/test_v65_migration.py +0 -0
  483. {flextool-4.1.1 → flextool-4.2.2}/tests/test_v66_migration.py +0 -0
  484. {flextool-4.1.1 → flextool-4.2.2}/tests/test_v68_migration.py +0 -0
  485. {flextool-4.1.1 → flextool-4.2.2}/tests/test_v69_migration.py +0 -0
  486. {flextool-4.1.1 → flextool-4.2.2}/tests/test_v70_migration.py +0 -0
  487. {flextool-4.1.1 → flextool-4.2.2}/tests/test_xlsx_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flextool
3
- Version: 4.1.1
3
+ Version: 4.2.2
4
4
  Summary: IRENA FlexTool is an energy and power systems model for understanding the role of variable power generation in future energy systems.
5
5
  Author: IRENA FlexTool contributors
6
6
  License: © International Renewable Energy Agency 2018-2022
@@ -34,12 +34,13 @@ Classifier: Programming Language :: Python :: 3
34
34
  Classifier: Programming Language :: Python :: 3.11
35
35
  Classifier: Programming Language :: Python :: 3.12
36
36
  Classifier: Programming Language :: Python :: 3.13
37
+ Classifier: Programming Language :: Python :: 3.14
37
38
  Classifier: Topic :: Scientific/Engineering
38
39
  Requires-Python: >=3.11
39
40
  Description-Content-Type: text/markdown
40
41
  License-File: LICENSE.txt
41
- Requires-Dist: polar-high>=3.8.0
42
- Requires-Dist: highspy<1.15,>=1.14.0
42
+ Requires-Dist: polar-high>=3.12.3
43
+ Requires-Dist: highspy<1.16,>=1.15.1
43
44
  Requires-Dist: polars<3,>=1.40
44
45
  Requires-Dist: pandas<4,>=2.2.3
45
46
  Requires-Dist: spinedb-api<0.37,>=0.36.5
@@ -58,6 +59,8 @@ Requires-Dist: spinetoolbox; extra == "toolbox"
58
59
  Provides-Extra: viewer
59
60
  Requires-Dist: tsdownsample>=0.1.3; extra == "viewer"
60
61
  Requires-Dist: networkx>=3.0; extra == "viewer"
62
+ Provides-Extra: compat
63
+ Requires-Dist: polars[rtcompat]<3,>=1.40; extra == "compat"
61
64
  Provides-Extra: engine-polars
62
65
  Provides-Extra: dev
63
66
  Requires-Dist: pytest>=8.0; extra == "dev"
@@ -127,7 +130,11 @@ against linopy and Pyomo on the same HiGHS solver.
127
130
 
128
131
  ### Check your Python (and possibly install it)
129
132
 
130
- FlexTool requires **Python 3.11+**.
133
+ FlexTool requires **Python 3.11+**, 64-bit Windows 10 or 11, Linux or macOS, and on
134
+ Intel/AMD computers a processor with SSE4.2 and POPCNT (x86-64-v2: Intel Core
135
+ i3/i5/i7 from 2009 on, AMD FX or Ryzen; not Core 2 or Phenom II). A processor with
136
+ AVX2 (most from 2013 on) runs the fastest build. Windows 7 and 8.1 cannot run FlexTool 4.
137
+ If a library reports "DLL load failed" on Windows, install the Microsoft Visual C++ Redistributable (x64, https://aka.ms/vs/17/release/vc_redist.x64.exe).
131
138
 
132
139
  Open a terminal (Windows: Command Prompt or PowerShell; macOS: Terminal; Linux: your shell)
133
140
  and check which Python you have:
@@ -173,6 +180,16 @@ and viewing input and result databases) and the network-graph result view. To
173
180
  install without Spine Toolbox (edit/view data in a spreadsheet only), you can
174
181
  use plain `pip install flextool`.
175
182
 
183
+ On a processor without AVX2 (for example a Celeron/Pentium laptop up to about 2021, an
184
+ Intel Core from before 2013 or an AMD FX), add the `compat` extra:
185
+ `pip install "flextool[toolbox,viewer,compat]"`. It installs the compatible build of the
186
+ `polars` data library, which is about 10-30% slower but runs on any x86-64-v2 processor.
187
+ When FlexTool finds such a processor without the compatible build, it says why and gives
188
+ the pip command: in a terminal it asks "Run it now? [y/N]" and installs only on yes;
189
+ without a terminal (a Spine Toolbox tool, CI) it stops with that message; the GUI asks at
190
+ startup, before it loads polars (in its terminal, or in a yes/no dialog from a shortcut), and `flextool-update` installs it. `python -m flextool.env_check`
191
+ reports what it found.
192
+
176
193
  Your prompt should now show `(.venv)`. Verify with `python --version` — it
177
194
  should report 3.11 or newer.
178
195
 
@@ -59,7 +59,11 @@ against linopy and Pyomo on the same HiGHS solver.
59
59
 
60
60
  ### Check your Python (and possibly install it)
61
61
 
62
- FlexTool requires **Python 3.11+**.
62
+ FlexTool requires **Python 3.11+**, 64-bit Windows 10 or 11, Linux or macOS, and on
63
+ Intel/AMD computers a processor with SSE4.2 and POPCNT (x86-64-v2: Intel Core
64
+ i3/i5/i7 from 2009 on, AMD FX or Ryzen; not Core 2 or Phenom II). A processor with
65
+ AVX2 (most from 2013 on) runs the fastest build. Windows 7 and 8.1 cannot run FlexTool 4.
66
+ If a library reports "DLL load failed" on Windows, install the Microsoft Visual C++ Redistributable (x64, https://aka.ms/vs/17/release/vc_redist.x64.exe).
63
67
 
64
68
  Open a terminal (Windows: Command Prompt or PowerShell; macOS: Terminal; Linux: your shell)
65
69
  and check which Python you have:
@@ -105,6 +109,16 @@ and viewing input and result databases) and the network-graph result view. To
105
109
  install without Spine Toolbox (edit/view data in a spreadsheet only), you can
106
110
  use plain `pip install flextool`.
107
111
 
112
+ On a processor without AVX2 (for example a Celeron/Pentium laptop up to about 2021, an
113
+ Intel Core from before 2013 or an AMD FX), add the `compat` extra:
114
+ `pip install "flextool[toolbox,viewer,compat]"`. It installs the compatible build of the
115
+ `polars` data library, which is about 10-30% slower but runs on any x86-64-v2 processor.
116
+ When FlexTool finds such a processor without the compatible build, it says why and gives
117
+ the pip command: in a terminal it asks "Run it now? [y/N]" and installs only on yes;
118
+ without a terminal (a Spine Toolbox tool, CI) it stops with that message; the GUI asks at
119
+ startup, before it loads polars (in its terminal, or in a yes/no dialog from a shortcut), and `flextool-update` installs it. `python -m flextool.env_check`
120
+ reports what it found.
121
+
108
122
  Your prompt should now show `(.venv)`. Verify with `python --version` — it
109
123
  should report 3.11 or newer.
110
124
 
@@ -0,0 +1,100 @@
1
+ """FlexTool energy system optimization model package."""
2
+ # On a processor without AVX2/FMA/BMI2/MOVBE, polars' default native runtime
3
+ # can crash as soon as polars is imported (below, transitively). Read the
4
+ # CPU flags first (stdlib only) and, when the compatible runtime of the
5
+ # installed polars version is needed, say why and give the pip command: in a
6
+ # terminal FlexTool asks before running it; without one (a GUI subprocess, a
7
+ # Toolbox tool, CI) the import stops with that message (an ImportError). The
8
+ # GUI launchers import this package first, so the GUI asks here too: in its
9
+ # terminal if it has one, otherwise in a small tkinter yes/no dialog, both
10
+ # before polars is imported; on no it exits with the message. On a CPU that
11
+ # runs the default runtime, with the compatible runtime installed next to it
12
+ # (the Toolbox bundle ships both), POLARS_PREFER_PKG selects the default one:
13
+ # polars loads the compatible runtime first whenever it is installed. See
14
+ # flextool.env_check.ensure_polars_runtime_before_import. env_check.py is
15
+ # loaded from its file under a private name, so ``python -m
16
+ # flextool.env_check`` still runs a fresh module (no runpy warning).
17
+ def _ensure_polars_runtime() -> str:
18
+ import importlib.util
19
+ import os
20
+ import sys
21
+
22
+ name = "flextool._env_check_at_import"
23
+ spec = importlib.util.spec_from_file_location(name, os.path.join(os.path.dirname(__file__), "env_check.py"))
24
+ module = importlib.util.module_from_spec(spec)
25
+ sys.modules[name] = module # dataclasses look their module up while it executes
26
+ try:
27
+ spec.loader.exec_module(module)
28
+ return module.ensure_polars_runtime_before_import()
29
+ finally:
30
+ sys.modules.pop(name, None)
31
+
32
+
33
+ # ``python -m flextool.env_check`` on a processor that needs the compatible
34
+ # polars build, which is not installed: the diagnostics run without polars in
35
+ # this process (the default runtime would crash it here, before they print);
36
+ # their probe imports polars in a child process. The rest of the package is
37
+ # not imported then.
38
+ if _ensure_polars_runtime() != "diagnostics":
39
+ # jemalloc decay config for polars, applied BEFORE the first ``import polars``
40
+ # (which happens transitively in the imports below). polars statically links
41
+ # jemalloc and reads ``_RJEM_MALLOC_CONF`` once, when its runtime library
42
+ # loads; polars 1.x's own values are ``dirty_decay_ms:500,muzzy_decay_ms:-1``,
43
+ # and muzzy -1 leaves freed pages resident (MADV_FREE) for good (polars 2.x
44
+ # writes ``muzzy_decay_ms:1000``, which returns them after ~2 s). Then each
45
+ # rolling solve's freed GB-scale coefficient frames stay resident and peak
46
+ # memory roughly doubles on large models. ``dirty_decay_ms:1000`` purges
47
+ # freed pages ~1 s after they go idle without madvise-ing on every transient
48
+ # free inside a hot operation; ``muzzy_decay_ms:0`` forces MADV_DONTNEED, so
49
+ # Private_Dirty actually drops. glibc knobs (MALLOC_ARENA_MAX, malloc_trim)
50
+ # do NOT touch polars memory: this is the lever that does. When polars was
51
+ # imported before flextool (a Toolbox console, a notebook), the same values
52
+ # are written into jemalloc at runtime; see ``flextool/_allocator.py``.
53
+ # The run's thread budget (``--threads`` / ``FLEXTOOL_THREADS`` / the available cores) sizes polars' thread
54
+ # pool, which polars also reads only at import: ``POLARS_MAX_THREADS`` is set here and put back as it was once
55
+ # polars is loaded (below), so the environment does not carry it to child processes. See
56
+ # ``flextool/_thread_budget.py``.
57
+ from flextool._thread_budget import apply_before_polars as _apply_thread_budget # noqa: E402
58
+ _polars_threads_before = _apply_thread_budget()
59
+ from flextool._allocator import configure_polars_allocator as _configure_polars_allocator # noqa: E402
60
+ _configure_polars_allocator()
61
+ # polars' default ``engine="auto"`` lets polars 2.x run some lazy queries on the
62
+ # streaming engine, whose joins, group_by and unique return rows in an order
63
+ # that changes from run to run, and whose partial sums are added in a varying
64
+ # order (last-bit differences in the discount factors, hence in the LP). The
65
+ # outputs, the LP and the parity tests are built and verified on the in-memory
66
+ # engine (what ``auto`` is in polars 1.x), so FlexTool selects it; a
67
+ # POLARS_ENGINE_AFFINITY from the environment wins. Calls that pass
68
+ # ``engine="streaming"`` explicitly sort or reduce order-independently.
69
+ import os as _os # noqa: E402 (after the allocator config above)
70
+ if "POLARS_ENGINE_AFFINITY" not in _os.environ:
71
+ import polars as _pl
72
+ _pl.Config.set_engine_affinity("in-memory")
73
+ import warnings as _warnings # noqa: E402 (after the allocator config above)
74
+ # Silence requests' RequestsDependencyWarning. The FlexTool venv ships a
75
+ # ``requests`` whose pinned urllib3/charset_normalizer version ranges lag the
76
+ # (working) installed versions, so ``import requests`` warns once at import
77
+ # time. It is harmless but leaks into Spine Toolbox's Tool console for any
78
+ # path that pulls in requests (e.g. an ``http://`` Spine DB-server URL).
79
+ # Match by message — set BEFORE requests is imported transitively below — so
80
+ # we needn't import requests here just to reference its warning class, and the
81
+ # filter stays robust across the venv's exact version numbers.
82
+ _warnings.filterwarnings(
83
+ "ignore",
84
+ message=r"urllib3 .*doesn't match a supported version",
85
+ )
86
+ try:
87
+ import polars as _pl # noqa: E402,F811 (the pool size is fixed here)
88
+ from flextool.process_outputs import write_outputs # noqa: E402 (after env/warning config above)
89
+ from flextool.update_flextool import migrate_database, initialize_database, update_flextool # noqa: E402
90
+ finally:
91
+ from flextool._thread_budget import restore_env as _restore_thread_env # noqa: E402
92
+ _restore_thread_env(_polars_threads_before)
93
+
94
+ __all__ = [
95
+ 'write_outputs',
96
+ 'migrate_database',
97
+ 'initialize_database',
98
+ 'update_flextool',
99
+ ]
100
+ name = "flextool"
@@ -0,0 +1,261 @@
1
+ """polars' jemalloc decay settings, whichever of polars and flextool was imported first.
2
+
3
+ polars links jemalloc statically. jemalloc reads ``_RJEM_MALLOC_CONF`` once, while ``import polars`` loads the
4
+ runtime library. polars 1.x's own values, which ``import polars`` writes into the variable, are
5
+ ``dirty_decay_ms:500,muzzy_decay_ms:-1``: freed pages pass to the "muzzy" (MADV_FREE) state and stay resident, so a
6
+ large model's peak memory roughly doubles. polars 2.x writes ``muzzy_decay_ms:1000``, which returns them after ~2 s.
7
+ FlexTool wants ``dirty_decay_ms:1000,muzzy_decay_ms:0``: a freed page is returned to the OS about a second after it
8
+ goes idle.
9
+
10
+ * polars not loaded yet: the environment variable is set (``setdefault``, so a value from the shell wins) and
11
+ jemalloc reads it when polars loads. ``import polars`` puts its own values in front, and the later ones win.
12
+ * polars already loaded (a Spine Toolbox console, a notebook, a script that imported polars first): FlexTool's
13
+ values are appended to the variable, for child processes, and the same two decay values are written at runtime
14
+ through jemalloc's ``mallctl``: the default for new arenas (``arenas.*``) and every initialised arena
15
+ (``arena.<i>.*``). The ``mallctl`` symbol is local to the polars library (not exported), so its address is
16
+ computed from the ELF symbol table: load base = address of the library's exported ``PyInit_*`` function minus its
17
+ symbol value. The first bytes of the function at the computed address are compared with the same bytes of the
18
+ file before the function is called; any mismatch, a non-ELF library (macOS) or a failing ``mallctl`` call prints
19
+ one warning on stderr with the remedy and changes nothing. A ``_RJEM_MALLOC_CONF`` set before polars was
20
+ imported has already been read by jemalloc and wins as a whole, as it would with flextool imported first. One
21
+ set after polars was imported (it then lacks polars' own values in front) has its decay keys written at runtime.
22
+
23
+ Windows polars uses mimalloc, and a library without jemalloc has nothing to configure: both are a silent no-op.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import os
28
+ import re
29
+ import sys
30
+
31
+ ENV_VAR = "_RJEM_MALLOC_CONF"
32
+ DEFAULT_CONF = "dirty_decay_ms:1000,muzzy_decay_ms:0"
33
+ DECAY_KEYS = ("dirty_decay_ms", "muzzy_decay_ms")
34
+ MALLCTL_SYMBOL = "_rjem_mallctl" # local (not exported) in the polars library
35
+ REMEDY = (
36
+ "FlexTool: polars was imported before flextool and its memory allocator settings could not be adjusted "
37
+ "({reason}); without them peak memory can double. Import flextool before polars, or start Python with "
38
+ f"{ENV_VAR}={DEFAULT_CONF}"
39
+ )
40
+ _EFAULT = 14 # mallctl on an arena index that is not initialised
41
+ _COMPARE_BYTES = 32 # bytes of the function compared between memory and file
42
+ _SHT_SYMTAB = 2
43
+ _STT_FUNC = 2
44
+
45
+ # ``import polars`` writes its own decay values in front of any value that was already set, before the runtime
46
+ # library loads; what follows them is the value set before polars was imported.
47
+ _POLARS_PREFIX = re.compile(r"dirty_decay_ms:-?\d+,muzzy_decay_ms:-?\d+(,thp:always,metadata_thp:always)?(?=,|$)")
48
+
49
+ _state = {"warned": False}
50
+
51
+
52
+ class AllocatorUnreachable(Exception):
53
+ """jemalloc's mallctl in the loaded polars library cannot be located or called safely."""
54
+
55
+
56
+ def configure_polars_allocator() -> None:
57
+ """Set the decay config for polars' jemalloc; called first thing in ``flextool/__init__.py``.
58
+
59
+ The runtime path does what the environment path would have done in a process that imported flextool first:
60
+ a ``_RJEM_MALLOC_CONF`` that was set before polars loaded wins as a whole (jemalloc has already read it), and
61
+ otherwise FlexTool's decay values are applied.
62
+ """
63
+ conf = os.environ.get(ENV_VAR)
64
+ if not polars_loaded():
65
+ os.environ.setdefault(ENV_VAR, DEFAULT_CONF)
66
+ return
67
+ if sys.platform == "win32":
68
+ return
69
+ ctl, opt, failure = None, None, None
70
+ try:
71
+ library = runtime_library()
72
+ if library is None or not has_jemalloc(library[0]):
73
+ return
74
+ ctl = Mallctl.from_library(*library)
75
+ opt = {key: ctl.read_ssize(f"opt.{key}") for key in DECAY_KEYS}
76
+ except Exception as exc: # decided below whether anything needed the call
77
+ ctl, failure = None, exc
78
+ targets, child_conf = plan_decay(conf, opt)
79
+ if child_conf is not None:
80
+ os.environ[ENV_VAR] = child_conf
81
+ if not targets:
82
+ return
83
+ try:
84
+ if ctl is None:
85
+ raise failure or AllocatorUnreachable("mallctl not located")
86
+ apply_decay(ctl, targets)
87
+ except Exception as exc: # any failure leaves the allocator as it was; the warning says what to do
88
+ warn_once(f"{type(exc).__name__}: {exc}")
89
+
90
+
91
+ def plan_decay(conf: str | None, opt: dict[str, int] | None) -> tuple[dict[str, int], str | None]:
92
+ """(decay values to write at runtime, new ``_RJEM_MALLOC_CONF`` for child processes or None).
93
+
94
+ ``opt`` holds the decay values jemalloc read at load (None when mallctl is unreachable). A leading
95
+ ``dirty_decay_ms:…,muzzy_decay_ms:…`` pair is the one ``import polars`` wrote when jemalloc read exactly those
96
+ values; what follows it was set before polars was imported and wins as a whole. Any other value was set after
97
+ polars was imported, so jemalloc has not seen it, and its decay keys are written at runtime.
98
+ """
99
+ if not conf:
100
+ return decay_targets(DEFAULT_CONF), DEFAULT_CONF
101
+ prefix = _POLARS_PREFIX.match(conf)
102
+ if prefix is not None and (opt is None or decay_targets(prefix.group(0)) == opt):
103
+ if conf[prefix.end():].strip(","):
104
+ return {}, None
105
+ return decay_targets(DEFAULT_CONF), f"{conf},{DEFAULT_CONF}"
106
+ return decay_targets(conf), None
107
+
108
+
109
+ def polars_loaded() -> bool:
110
+ return any(m == "polars" or m.startswith("_polars_runtime") for m in sys.modules)
111
+
112
+
113
+ def decay_targets(conf: str) -> dict[str, int]:
114
+ """The decay keys of a jemalloc option string; a later occurrence wins, as in jemalloc."""
115
+ out: dict[str, int] = {}
116
+ for item in conf.split(","):
117
+ key, sep, val = item.strip().partition(":")
118
+ if sep and key in DECAY_KEYS:
119
+ try:
120
+ out[key] = int(val)
121
+ except ValueError:
122
+ continue
123
+ return out
124
+
125
+
126
+ def runtime_library() -> tuple[str, str] | None:
127
+ """(path, exported init symbol) of the loaded polars extension library, or None when none is loaded."""
128
+ for name, mod in list(sys.modules.items()):
129
+ if not (name.startswith("_polars_runtime") or name == "polars.polars"):
130
+ continue
131
+ path = getattr(mod, "__file__", None) or ""
132
+ if path.endswith((".so", ".pyd", ".dylib")):
133
+ return path, "PyInit_" + name.rpartition(".")[2]
134
+ return None
135
+
136
+
137
+ def has_jemalloc(path: str) -> bool:
138
+ """jemalloc's message prefix survives stripping, so it tells a jemalloc build from one without."""
139
+ import mmap
140
+ with open(path, "rb") as f, mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
141
+ return mm.find(b"<jemalloc>") >= 0
142
+
143
+
144
+ def warn_once(reason: str) -> None:
145
+ if _state["warned"]:
146
+ return
147
+ _state["warned"] = True
148
+ print(REMEDY.format(reason=reason), file=sys.stderr, flush=True)
149
+
150
+
151
+ def elf_function(path: str, names: tuple[str, ...]) -> dict[str, tuple[int, bytes]]:
152
+ """{name: (symbol value, first file bytes of the function)} from the ELF64 ``.symtab`` of ``path``."""
153
+ import mmap
154
+ import struct
155
+
156
+ import numpy as np
157
+
158
+ with open(path, "rb") as f, mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
159
+ if mm[:4] != b"\x7fELF" or mm[4] != 2 or mm[5] != 1:
160
+ raise AllocatorUnreachable("not an ELF64 little-endian library")
161
+ shoff, = struct.unpack_from("<Q", mm, 0x28)
162
+ shentsize, shnum = struct.unpack_from("<HH", mm, 0x3A)
163
+ sections = [struct.unpack_from("<IIQQQQIIQQ", mm, shoff + i * shentsize) for i in range(shnum)]
164
+ symtab = next((s for s in sections if s[1] == _SHT_SYMTAB), None)
165
+ if symtab is None:
166
+ raise AllocatorUnreachable("the library has no symbol table")
167
+ sym_off, sym_size, str_index = symtab[4], symtab[5], symtab[6]
168
+ str_off, str_size = sections[str_index][4], sections[str_index][5]
169
+ str_end = str_off + str_size
170
+ name_offsets = {}
171
+ for name in names:
172
+ pos = mm.find(b"\0" + name.encode() + b"\0", str_off, str_end)
173
+ if pos < 0:
174
+ raise AllocatorUnreachable(f"symbol {name} not found")
175
+ name_offsets[name] = pos + 1 - str_off
176
+ dtype = np.dtype([("name", "<u4"), ("info", "u1"), ("other", "u1"), ("shndx", "<u2"),
177
+ ("value", "<u8"), ("size", "<u8")])
178
+ syms = np.frombuffer(mm, dtype=dtype, count=sym_size // dtype.itemsize, offset=sym_off)
179
+ hits = {name: syms[(syms["name"] == noff) & ((syms["info"] & 0xF) == _STT_FUNC) & (syms["size"] > 0)]
180
+ for name, noff in name_offsets.items()}
181
+ del syms # copies are kept; the mmap closes only without views into it
182
+ out = {}
183
+ for name, hit in hits.items():
184
+ if len(hit) != 1:
185
+ raise AllocatorUnreachable(f"symbol {name}: {len(hit)} function entries")
186
+ value, shndx = int(hit["value"][0]), int(hit["shndx"][0])
187
+ if not 0 < shndx < shnum:
188
+ raise AllocatorUnreachable(f"symbol {name} is not in a section")
189
+ sec_addr, sec_offset, sec_size = sections[shndx][3], sections[shndx][4], sections[shndx][5]
190
+ n = min(int(hit["size"][0]), _COMPARE_BYTES)
191
+ if not (sec_addr <= value and value + n <= sec_addr + sec_size):
192
+ raise AllocatorUnreachable(f"symbol {name} lies outside its section")
193
+ start = sec_offset + (value - sec_addr)
194
+ out[name] = (value, bytes(mm[start:start + n]))
195
+ return out
196
+
197
+
198
+ class Mallctl:
199
+ """jemalloc's ``int mallctl(const char*, void* oldp, size_t* oldlenp, void* newp, size_t newlen)``."""
200
+
201
+ def __init__(self, fn) -> None:
202
+ self._fn = fn
203
+
204
+ @classmethod
205
+ def from_library(cls, path: str, init_symbol: str) -> Mallctl:
206
+ import ctypes
207
+
208
+ mallctl_symbol = MALLCTL_SYMBOL
209
+ found = elf_function(path, (init_symbol, mallctl_symbol))
210
+ lib = ctypes.CDLL(path, mode=os.RTLD_NOLOAD | os.RTLD_LAZY)
211
+ init_addr = ctypes.cast(getattr(lib, init_symbol), ctypes.c_void_p).value
212
+ if not init_addr:
213
+ raise AllocatorUnreachable(f"{init_symbol} has no address")
214
+ base = init_addr - found[init_symbol][0]
215
+ for name, (value, file_bytes) in found.items():
216
+ if read_memory(base + value, len(file_bytes)) != file_bytes:
217
+ raise AllocatorUnreachable(f"code of {name} in memory differs from the library file")
218
+ proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p, ctypes.POINTER(ctypes.c_size_t),
219
+ ctypes.c_void_p, ctypes.c_size_t)
220
+ return cls(proto(base + found[mallctl_symbol][0]))
221
+
222
+ def read_ssize(self, name: str) -> int:
223
+ import ctypes
224
+ val, size = ctypes.c_ssize_t(0), ctypes.c_size_t(ctypes.sizeof(ctypes.c_ssize_t))
225
+ rc = self._fn(name.encode(), ctypes.byref(val), ctypes.byref(size), None, 0)
226
+ if rc != 0:
227
+ raise AllocatorUnreachable(f"mallctl read {name} returned {rc}")
228
+ return val.value
229
+
230
+ def read_unsigned(self, name: str) -> int:
231
+ import ctypes
232
+ val, size = ctypes.c_uint(0), ctypes.c_size_t(ctypes.sizeof(ctypes.c_uint))
233
+ rc = self._fn(name.encode(), ctypes.byref(val), ctypes.byref(size), None, 0)
234
+ if rc != 0:
235
+ raise AllocatorUnreachable(f"mallctl read {name} returned {rc}")
236
+ return val.value
237
+
238
+ def write_ssize(self, name: str, value: int) -> int:
239
+ import ctypes
240
+ val = ctypes.c_ssize_t(value)
241
+ return self._fn(name.encode(), None, None, ctypes.byref(val), ctypes.sizeof(val))
242
+
243
+
244
+ def read_memory(address: int, n: int) -> bytes:
245
+ import ctypes
246
+ return ctypes.string_at(address, n)
247
+
248
+
249
+ def apply_decay(ctl: Mallctl, targets: dict[str, int]) -> None:
250
+ """Write the decay values as the new-arena default and into every initialised arena."""
251
+ if all(ctl.read_ssize(f"arenas.{key}") == val for key, val in targets.items()):
252
+ return
253
+ narenas = ctl.read_unsigned("arenas.narenas")
254
+ for key, val in targets.items():
255
+ rc = ctl.write_ssize(f"arenas.{key}", val)
256
+ if rc != 0:
257
+ raise AllocatorUnreachable(f"mallctl write arenas.{key} returned {rc}")
258
+ for i in range(narenas):
259
+ rc = ctl.write_ssize(f"arena.{i}.{key}", val)
260
+ if rc not in (0, _EFAULT):
261
+ raise AllocatorUnreachable(f"mallctl write arena.{i}.{key} returned {rc}")