processforge 0.3.23__tar.gz → 0.3.26__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. {processforge-0.3.23/src/processforge.egg-info → processforge-0.3.26}/PKG-INFO +1 -1
  2. {processforge-0.3.23 → processforge-0.3.26}/flowsheets/hydraulic-chain.json +1 -1
  3. {processforge-0.3.23 → processforge-0.3.26}/pyproject.toml +1 -1
  4. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/__init__.py +1 -1
  5. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/apply.py +24 -60
  6. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/common.py +1 -1
  7. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/display.py +32 -4
  8. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/init.py +115 -69
  9. processforge-0.3.26/src/processforge/cli/persist.py +99 -0
  10. processforge-0.3.26/src/processforge/cli/plan.py +198 -0
  11. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/run.py +6 -43
  12. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/flowsheet.py +46 -12
  13. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/solver.py +104 -65
  14. {processforge-0.3.23 → processforge-0.3.26/src/processforge.egg-info}/PKG-INFO +1 -1
  15. {processforge-0.3.23 → processforge-0.3.26}/src/processforge.egg-info/SOURCES.txt +3 -0
  16. processforge-0.3.26/tests/test_display.py +36 -0
  17. processforge-0.3.26/tests/test_persist.py +73 -0
  18. processforge-0.3.23/src/processforge/cli/plan.py +0 -141
  19. {processforge-0.3.23 → processforge-0.3.26}/LICENSE +0 -0
  20. {processforge-0.3.23 → processforge-0.3.26}/MANIFEST.in +0 -0
  21. {processforge-0.3.23 → processforge-0.3.26}/README.md +0 -0
  22. {processforge-0.3.23 → processforge-0.3.26}/flowsheets/closed-loop-chain.json +0 -0
  23. {processforge-0.3.23 → processforge-0.3.26}/flowsheets/coupled/msre_coupled.json +0 -0
  24. {processforge-0.3.23 → processforge-0.3.26}/flowsheets/custom/crimson/MUWS_HB.json +0 -0
  25. {processforge-0.3.23 → processforge-0.3.26}/flowsheets/custom/simple_hyrdaulic_scheme.json +0 -0
  26. {processforge-0.3.23 → processforge-0.3.26}/flowsheets/festim/hydrogen_transport_1d.json +0 -0
  27. {processforge-0.3.23 → processforge-0.3.26}/flowsheets/festim/tds_tungsten.json +0 -0
  28. {processforge-0.3.23 → processforge-0.3.26}/flowsheets/openmc/msre_eigenvalue.json +0 -0
  29. {processforge-0.3.23 → processforge-0.3.26}/setup.cfg +0 -0
  30. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/__main__.py +0 -0
  31. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/_schema.py +0 -0
  32. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/analysis/__init__.py +0 -0
  33. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/analysis/dof.py +0 -0
  34. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/api/__init__.py +0 -0
  35. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/api/serve.py +0 -0
  36. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/__init__.py +0 -0
  37. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/diagram.py +0 -0
  38. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/export_fmu.py +0 -0
  39. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/export_modelica.py +0 -0
  40. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/runs.py +0 -0
  41. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/cli/validate.py +0 -0
  42. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/compose.py +0 -0
  43. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/coupling.py +0 -0
  44. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/__init__.py +0 -0
  45. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/backends/__init__.py +0 -0
  46. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/backends/base.py +0 -0
  47. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/backends/casadi_backend.py +0 -0
  48. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/backends/pyomo_backend.py +0 -0
  49. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/backends/scipy_backend.py +0 -0
  50. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/jacobian.py +0 -0
  51. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/mixin.py +0 -0
  52. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/stream_var.py +0 -0
  53. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/units/__init__.py +0 -0
  54. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/units/flash_eo.py +0 -0
  55. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/units/heater_eo.py +0 -0
  56. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/units/pipes_eo.py +0 -0
  57. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/units/pump_eo.py +0 -0
  58. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/units/strainer_eo.py +0 -0
  59. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/eo/units/valve_eo.py +0 -0
  60. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/flowsheet.py +0 -0
  61. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/fmu/__init__.py +0 -0
  62. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/fmu/_fmi_vars.py +0 -0
  63. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/fmu/builder.py +0 -0
  64. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/fmu/slave_template.py +0 -0
  65. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/lock.py +0 -0
  66. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/modelica/__init__.py +0 -0
  67. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/modelica/mo_writer.py +0 -0
  68. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/modelica/omc_runner.py +0 -0
  69. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/modelica/transpiler.py +0 -0
  70. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/modelica/unit_equations.py +0 -0
  71. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/output_collector.py +0 -0
  72. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/pcl/__init__.py +0 -0
  73. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/pcl/compiler.py +0 -0
  74. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/pcl/namespace.py +0 -0
  75. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/persistence/__init__.py +0 -0
  76. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/persistence/archive.py +0 -0
  77. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/persistence/artifact_store.py +0 -0
  78. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/provenance.py +0 -0
  79. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/__init__.py +0 -0
  80. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/base.py +0 -0
  81. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/base_jacobian_mixin.py +0 -0
  82. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/cantera_jacobian.py +0 -0
  83. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/cantera_provider.py +0 -0
  84. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/container_client.py +0 -0
  85. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/coolprop_provider.py +0 -0
  86. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/errors.py +0 -0
  87. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/festim_provider.py +0 -0
  88. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/jacobian_contributor.py +0 -0
  89. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/manager.py +0 -0
  90. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/modelica_jacobian.py +0 -0
  91. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/modelica_provider.py +0 -0
  92. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/openmc_provider.py +0 -0
  93. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/reference_state_registry.py +0 -0
  94. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/providers/registry.py +0 -0
  95. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/quantity.py +0 -0
  96. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/result.py +0 -0
  97. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/result_schema.py +0 -0
  98. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/schemas/__init__.py +0 -0
  99. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/schemas/festim/festim_model.py +0 -0
  100. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/schemas/flowsheet_schema.json +0 -0
  101. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/schemas/openmc/openmc_model.py +0 -0
  102. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/simulate.py +0 -0
  103. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/solver.py +0 -0
  104. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/state.py +0 -0
  105. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/thermo.py +0 -0
  106. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/types.py +0 -0
  107. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/__init__.py +0 -0
  108. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/base.py +0 -0
  109. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/cstr.py +0 -0
  110. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/flash.py +0 -0
  111. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/heater.py +0 -0
  112. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/pfr.py +0 -0
  113. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/pipes.py +0 -0
  114. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/pump.py +0 -0
  115. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/registry.py +0 -0
  116. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/solver_unit.py +0 -0
  117. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/strainer.py +0 -0
  118. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/tank.py +0 -0
  119. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/units/valve.py +0 -0
  120. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/utils/__init__.py +0 -0
  121. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/utils/flowsheet_diagram.py +0 -0
  122. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/utils/mermaid_diagram.py +0 -0
  123. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/utils/s3_upload.py +0 -0
  124. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/utils/topology.py +0 -0
  125. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/utils/unit_consistency.py +0 -0
  126. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/utils/validate_flowsheet.py +0 -0
  127. {processforge-0.3.23 → processforge-0.3.26}/src/processforge/utils/validation.py +0 -0
  128. {processforge-0.3.23 → processforge-0.3.26}/src/processforge.egg-info/dependency_links.txt +0 -0
  129. {processforge-0.3.23 → processforge-0.3.26}/src/processforge.egg-info/entry_points.txt +0 -0
  130. {processforge-0.3.23 → processforge-0.3.26}/src/processforge.egg-info/requires.txt +0 -0
  131. {processforge-0.3.23 → processforge-0.3.26}/src/processforge.egg-info/top_level.txt +0 -0
  132. {processforge-0.3.23 → processforge-0.3.26}/tests/test_cli.py +0 -0
  133. {processforge-0.3.23 → processforge-0.3.26}/tests/test_container_client.py +0 -0
  134. {processforge-0.3.23 → processforge-0.3.26}/tests/test_coupling.py +0 -0
  135. {processforge-0.3.23 → processforge-0.3.26}/tests/test_festim_provider.py +0 -0
  136. {processforge-0.3.23 → processforge-0.3.26}/tests/test_festim_schema.py +0 -0
  137. {processforge-0.3.23 → processforge-0.3.26}/tests/test_init.py +0 -0
  138. {processforge-0.3.23 → processforge-0.3.26}/tests/test_openmc_provider.py +0 -0
  139. {processforge-0.3.23 → processforge-0.3.26}/tests/test_provider_errors.py +0 -0
  140. {processforge-0.3.23 → processforge-0.3.26}/tests/test_providers.py +0 -0
  141. {processforge-0.3.23 → processforge-0.3.26}/tests/test_result_zarr.py +0 -0
  142. {processforge-0.3.23 → processforge-0.3.26}/tests/test_runs_command.py +0 -0
  143. {processforge-0.3.23 → processforge-0.3.26}/tests/test_s3_upload.py +0 -0
  144. {processforge-0.3.23 → processforge-0.3.26}/tests/test_state.py +0 -0
  145. {processforge-0.3.23 → processforge-0.3.26}/tests/test_types.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: processforge
3
- Version: 0.3.23
3
+ Version: 0.3.26
4
4
  Summary: A Python-based process simulation framework for chemical engineering applications.
5
5
  Author-email: Processforge Team <team@processforge.dev>
6
6
  License-Expression: BSD-3-Clause
@@ -105,7 +105,7 @@
105
105
  "simulation": {
106
106
  "mode": "steady",
107
107
  "t0": 0.0,
108
- "tf": 25.0,
108
+ "tf": 15.0,
109
109
  "dt": 1.0
110
110
  }
111
111
  }
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "processforge"
7
- version = "0.3.23"
7
+ version = "0.3.26"
8
8
  description = "A Python-based process simulation framework for chemical engineering applications."
9
9
  readme = "README.md"
10
10
  license = "BSD-3-Clause"
@@ -30,7 +30,7 @@ from .units.flash import Flash
30
30
  from .units.heater import Heater
31
31
  from .eo import EOFlowsheet, EOSolver
32
32
 
33
- __version__ = "0.3.23"
33
+ __version__ = "0.3.26"
34
34
 
35
35
  __all__ = [
36
36
  "Flowsheet",
@@ -6,46 +6,10 @@ import os
6
6
  import time
7
7
  from typing import Literal
8
8
 
9
- import hashlib
10
- import json
11
- import os
12
9
  import typer
13
- from datetime import datetime, timezone
14
10
  from loguru import logger
15
11
 
16
-
17
- def _persist_run(archive, fs, results, run_info, config, base_name, snapshot_id):
18
- """Build a standardized RunManifest and persist it to the archive."""
19
- run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "_" + os.urandom(3).hex()
20
- flowsheet_hash = hashlib.sha256(
21
- json.dumps(config, sort_keys=True, default=str).encode("utf-8")
22
- ).hexdigest()[:16]
23
- manifest = fs.collect_outputs(
24
- run_id, config.get("simulation", {}).get("mode", "steady"),
25
- base_name, provenance=run_info.model_dump(),
26
- )
27
- manifest.snapshot_id = snapshot_id
28
- stream_results = {k: v for k, v in results.items() if not hasattr(v, "fields")}
29
- archive.save_run(manifest, stream_results=stream_results)
30
-
31
- # Always persist a Zarr copy of the standardized outputs (fields + artifacts)
32
- # inside the archive, mirroring `pf run`.
33
- try:
34
- from ..result import relink_latest_results, save_results_zarr
35
-
36
- run_results_dir = os.path.join(archive.path, "results", run_id)
37
- save_results_zarr(
38
- results,
39
- os.path.join(run_results_dir, "results.zarr"),
40
- run_info,
41
- )
42
- relink_latest_results(archive.path, run_id)
43
- except Exception as e:
44
- logger.warning(f"Failed to write results.zarr: {type(e).__name__}: {e}")
45
-
46
- logger.info(f"Run saved : {run_id}")
47
- return run_id
48
-
12
+ from .persist import make_run_id, persist_run
49
13
  from ..eo import EOFlowsheet
50
14
  from ..provenance import build_run_info
51
15
  from .common import (
@@ -63,7 +27,7 @@ from .common import (
63
27
  write_divergence_report,
64
28
  )
65
29
 
66
- from .display import print_structural_diff
30
+ from .display import print_param_drift, print_structural_diff
67
31
 
68
32
 
69
33
  def apply(
@@ -131,16 +95,9 @@ def apply(
131
95
  if not drifted:
132
96
  logger.info("No drift detected. System is already at the desired state.")
133
97
  return
134
- stream_drifts = [d for d in drifted if d.startswith("streams.")]
135
- unit_drifts = [d for d in drifted if d.startswith("units.")]
136
- other_drifts = [d for d in drifted if not d.startswith(("streams.", "units."))]
137
98
  logger.warning("Drift detected:")
138
- if stream_drifts:
139
- logger.warning(f" Stream drift : {stream_drifts}")
140
- if unit_drifts:
141
- logger.warning(f" Unit drift : {unit_drifts}")
142
- if other_drifts:
143
- logger.warning(f" Config drift : {other_drifts}")
99
+ old_config = state.config if hasattr(state, "config") else state.get("config", {})
100
+ print_param_drift(drifted, old_config, config)
144
101
 
145
102
  # Build flowsheet; attach saved state for warm-start unless topology changed
146
103
  fs = EOFlowsheet(config, backend=backend)
@@ -165,14 +122,15 @@ def apply(
165
122
  label="converged state",
166
123
  )
167
124
  run_info = build_run_info(config, x0=fs.x0, var_names=fs.var_names)
168
- saved_run_id = _persist_run(sm, fs, results, run_info, config, base_name, snapshot_id)
125
+ run_id = make_run_id()
126
+ persist_run(sm, fs, run_id, results, run_info, config, base_name, snapshot_id)
169
127
  logger.info("=== Apply Summary ===")
170
128
  logger.info(" Status : CONVERGED")
171
129
  logger.info(f" Final ||F|| : {fs.solver_stats.get('final_norm', '?'):.3e}")
172
130
  logger.info(f" Iterations : {fs.solver_stats.get('iterations', '?')}")
173
131
  logger.info(f" Backend : {display_backend(config, fs.backend)}")
174
132
  logger.info(f" Snapshot ID : {snapshot_id}")
175
- logger.info(f" Run ID : {saved_run_id}")
133
+ logger.info(f" Run ID : {run_id}")
176
134
  logger.info(f" Elapsed (s) : {elapsed:.2f}")
177
135
  return
178
136
 
@@ -187,25 +145,31 @@ def apply(
187
145
  tmp_fs = _EO(config, backend=backend)
188
146
  manager = tmp_fs._build()
189
147
  try:
190
- x_hom, hom_converged, hom_stats = solve_with_homotopy(
191
- tmp_fs, manager, solver, state, drifted
192
- )
148
+ homotopy_result = solve_with_homotopy(tmp_fs, manager, solver, state, drifted)
149
+ if homotopy_result.converged:
150
+ # Assemble the real (converged) outputs from the homotopy
151
+ # solution while the providers are still alive. The `results`
152
+ # from the earlier failed direct solve must NOT be persisted.
153
+ tmp_fs.assemble_from_solution(
154
+ manager, homotopy_result.x_solution, True, homotopy_result.stats
155
+ )
193
156
  finally:
194
157
  teardown_providers(tmp_fs._provider_map)
195
158
 
196
- if hom_converged:
159
+ if homotopy_result.converged:
197
160
  logger.info(
198
- f"Homotopy converged: ||F||={hom_stats.get('final_norm', '?'):.3e}, "
199
- f"iterations={hom_stats.get('iterations', '?')}"
161
+ f"Homotopy converged: ||F||={homotopy_result.stats.get('final_norm', '?'):.3e}, "
162
+ f"iterations={homotopy_result.stats.get('iterations', '?')}"
200
163
  )
201
- save_snapshot(
202
- sm, config, x_hom, fs.var_names,
164
+ snapshot_id = save_snapshot(
165
+ sm, config, homotopy_result.x_solution, fs.var_names,
203
166
  metadata=current_metadata,
204
167
  parent_snapshot_id=state.snapshot_id if state is not None and not topology_changed else None,
205
168
  label="homotopy solution",
206
169
  )
207
170
  run_info = build_run_info(config, x0=fs.x0, var_names=fs.var_names)
208
- _persist_run(sm, fs, results, run_info, config, base_name, snapshot_id)
171
+ run_id = make_run_id()
172
+ persist_run(sm, tmp_fs, run_id, tmp_fs.results, run_info, config, base_name, snapshot_id)
209
173
  logger.info("Homotopy apply succeeded. New snapshot saved.")
210
174
  return
211
175
 
@@ -217,8 +181,8 @@ def apply(
217
181
  breakdown = log_residual_breakdown(fs)
218
182
  divergence = build_divergence_report(
219
183
  drifted_params=drifted,
220
- solver_stats=hom_stats,
221
- x_last=x_hom,
184
+ solver_stats=homotopy_result.stats,
185
+ x_last=homotopy_result.x_solution,
222
186
  var_names=fs.var_names,
223
187
  breakdown=breakdown,
224
188
  )
@@ -4,7 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  import datetime
6
6
  import hashlib
7
- import importlib
7
+ import importlib.util
8
8
  import json
9
9
  import os
10
10
  import urllib.error
@@ -4,6 +4,8 @@ from __future__ import annotations
4
4
 
5
5
  from typing import Any
6
6
 
7
+ import importlib.util
8
+
7
9
  from loguru import logger
8
10
 
9
11
 
@@ -53,7 +55,7 @@ def print_unit_mismatches(mismatches: list[Any]) -> None:
53
55
  logger.warning(f"Unit annotation — stream '{m.stream_name}'.{m.property_name}: {m.message}")
54
56
 
55
57
 
56
- def print_provider_health(config: dict) -> list[str]:
58
+ def print_provider_health(config: dict, strict: bool = False) -> list[str]:
57
59
  """Log per-provider reachability for flowsheets using Docker containers.
58
60
 
59
61
  For each provider declared in *config*:
@@ -63,10 +65,9 @@ def print_provider_health(config: dict) -> list[str]:
63
65
  * pip-installable providers are reported as importable or not.
64
66
 
65
67
  Returns a list of failure messages — one per unreachable containerized
66
- provider (empty means every provider is healthy/available).
68
+ provider (empty means every provider is healthy/available). When *strict*
69
+ is True, a missing pip-installable provider is also appended to failures.
67
70
  """
68
- import importlib
69
-
70
71
  from ..providers.registry import is_containerized, _PROVIDER_CATALOG
71
72
 
72
73
  providers = config.get("providers", {})
@@ -104,6 +105,10 @@ def print_provider_health(config: dict) -> list[str]:
104
105
  dep = catalog.get("optional_dep")
105
106
  hint = f"pip install 'processforge[{dep}]'" if dep else "built-in"
106
107
  logger.warning(f" [WARN] {name} [{ptype}] — not installed ({hint})")
108
+ if strict:
109
+ failures.append(
110
+ f"Provider '{name}' not installed ({hint})"
111
+ )
107
112
 
108
113
  return failures
109
114
 
@@ -122,3 +127,26 @@ def print_structural_diff(diff: dict) -> None:
122
127
  logger.info(f" - {name:<20} [{unit_type}] (removed)")
123
128
  if not any(diff.get(k) for k in ("added", "modified", "removed")):
124
129
  logger.info(" (no structural changes)")
130
+
131
+
132
+ def _resolve_path(cfg: dict, path: str) -> object:
133
+ """Resolve a dotted config path (e.g. ``simulation.tf``) to its value."""
134
+ cur: object = cfg
135
+ for part in path.split("."):
136
+ if isinstance(cur, dict) and part in cur:
137
+ cur = cur[part]
138
+ else:
139
+ return "<missing>"
140
+ return cur
141
+
142
+
143
+ def print_param_drift(drifted: list[str], old_config: dict, new_config: dict) -> None:
144
+ """Log parameter drift paths with their old → new values."""
145
+ logger.info("=== Parameter Drift vs. Saved State ===")
146
+ if not drifted:
147
+ logger.info(" (no parameter changes)")
148
+ return
149
+ for path in drifted:
150
+ old_val = _resolve_path(old_config, path)
151
+ new_val = _resolve_path(new_config, path)
152
+ logger.info(f" ~ {path}: {old_val!r} → {new_val!r}")
@@ -2,7 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- import importlib
5
+ import importlib.util
6
6
  import json
7
7
  import os
8
8
  import shutil
@@ -11,8 +11,18 @@ import subprocess
11
11
  import typer
12
12
  from loguru import logger
13
13
 
14
- from .common import extract_providers, is_local_provider_url
15
- from ..lock import flowsheet_env_dir, read_lock
14
+ from .common import (
15
+ extract_providers,
16
+ is_local_provider_url,
17
+ )
18
+ from ..lock import flowsheet_env_dir, read_lock, write_lock
19
+ from ..providers.registry import (
20
+ get_provider_docker_image,
21
+ get_provider_default_port,
22
+ is_containerized,
23
+ _PROVIDER_CATALOG,
24
+ )
25
+ from ..utils.validate_flowsheet import validate_flowsheet
16
26
 
17
27
 
18
28
  def _migrate_legacy_env(pf_dir: str) -> None:
@@ -21,7 +31,9 @@ def _migrate_legacy_env(pf_dir: str) -> None:
21
31
  Older processforge stored a single environment at ``.processforge/lock.json``
22
32
  and ``.processforge/docker-compose.yml``. This relocates those into the hashed
23
33
  env dir derived from the flowsheet recorded in the legacy lock, so an existing
24
- repo keeps its provider environment after upgrading.
34
+ repo keeps its provider environment after upgrading. The target always uses the
35
+ same hashed env dir that ``pf init``/``read_lock`` locate, so the migrated env
36
+ is never orphaned.
25
37
  """
26
38
  legacy_lock = os.path.join(pf_dir, "lock.json")
27
39
  legacy_compose = os.path.join(pf_dir, "docker-compose.yml")
@@ -35,15 +47,19 @@ def _migrate_legacy_env(pf_dir: str) -> None:
35
47
  except Exception:
36
48
  recorded = None
37
49
 
38
- if recorded and os.path.exists(recorded):
39
- env_dir = flowsheet_env_dir(pf_dir, recorded)
50
+ # Always target the hashed env dir keyed by the recorded flowsheet path so it
51
+ # matches what `pf init <flowsheet>` / `read_lock` expect on subsequent runs.
52
+ env_dir = flowsheet_env_dir(pf_dir, recorded or "legacy")
53
+ if recorded:
54
+ logger.info(
55
+ f"Migrating legacy .processforge/lock.json + docker-compose.yml into "
56
+ f"'{os.path.relpath(env_dir, pf_dir)}/'."
57
+ )
40
58
  else:
41
- base = os.path.splitext(os.path.basename(recorded or "legacy"))[0] or "legacy"
42
- env_dir = os.path.join(pf_dir, base)
43
59
  logger.warning(
44
60
  "Migrating legacy .processforge/lock.json + docker-compose.yml into "
45
- f"'{os.path.relpath(env_dir, pf_dir)}/' (recorded flowsheet not found on "
46
- "disk — re-run `pf init <flowsheet.json>` to restore the correct env dir)."
61
+ f"'{os.path.relpath(env_dir, pf_dir)}/' (no recorded flowsheet found "
62
+ "re-run `pf init <flowsheet.json>` to restore the correct env dir)."
47
63
  )
48
64
 
49
65
  if os.path.exists(env_dir):
@@ -73,15 +89,19 @@ def init(
73
89
  "--path",
74
90
  help="Root directory to initialise in (default: current directory)",
75
91
  ),
92
+ no_pull: bool = typer.Option(
93
+ False,
94
+ "--no-pull",
95
+ help="Generate docker-compose.yml but skip pulling provider images",
96
+ ),
97
+ force: bool = typer.Option(
98
+ False,
99
+ "--force",
100
+ "-f",
101
+ help="Re-initialise even if the environment already exists",
102
+ ),
76
103
  ) -> None:
77
104
  """Initialise the .processforge/ project directory."""
78
- from ..providers.registry import (
79
- is_containerized,
80
- get_provider_docker_image,
81
- get_provider_default_port,
82
- _PROVIDER_CATALOG,
83
- )
84
- from ..lock import write_lock
85
105
  from ..compose import generate_compose
86
106
 
87
107
  root = path or "."
@@ -96,17 +116,6 @@ def init(
96
116
  # dir so existing repos aren't silently broken by the new structure.
97
117
  _migrate_legacy_env(pf_dir)
98
118
 
99
- # Remove stale .pfstate snapshot directories from outputs/
100
- stale_count = 0
101
- for entry in os.listdir(outputs_dir):
102
- if entry.endswith(".pfstate"):
103
- stale = os.path.join(outputs_dir, entry)
104
- if os.path.isdir(stale):
105
- shutil.rmtree(stale)
106
- stale_count += 1
107
- if stale_count:
108
- logger.info(f"Removed {stale_count} stale snapshot(s) from {outputs_dir}/.")
109
-
110
119
  # Write config.json (always)
111
120
  config_path = os.path.join(pf_dir, "config.json")
112
121
  if not os.path.exists(config_path):
@@ -121,6 +130,26 @@ def init(
121
130
  else:
122
131
  logger.info(f"{config_path} already exists — skipped.")
123
132
 
133
+ # Honour a configured outputs_dir for the rest of init.
134
+ outputs_dir_name = "outputs"
135
+ try:
136
+ with open(config_path, encoding="utf-8") as f:
137
+ outputs_dir_name = json.load(f).get("outputs_dir", "outputs")
138
+ except Exception:
139
+ pass
140
+ outputs_dir = os.path.join(root, outputs_dir_name)
141
+
142
+ # Remove stale .pfstate snapshot directories from outputs/
143
+ stale_count = 0
144
+ for entry in os.listdir(outputs_dir):
145
+ if entry.endswith(".pfstate"):
146
+ stale = os.path.join(outputs_dir, entry)
147
+ if os.path.isdir(stale):
148
+ shutil.rmtree(stale)
149
+ stale_count += 1
150
+ if stale_count:
151
+ logger.info(f"Removed {stale_count} stale snapshot(s) from {outputs_dir}/.")
152
+
124
153
  # No flowsheet → scaffold only
125
154
  if not flowsheet:
126
155
  logger.info(".processforge/ initialised successfully.")
@@ -131,15 +160,21 @@ def init(
131
160
  flowsheet_path = flowsheet
132
161
  if not os.path.exists(flowsheet_path):
133
162
  logger.error(f"Flowsheet '{flowsheet_path}' not found.")
134
- raise SystemExit(1)
163
+ raise typer.Exit(code=1)
164
+
165
+ try:
166
+ validate_flowsheet(flowsheet_path)
167
+ except Exception as e:
168
+ logger.error(f"Failed to validate flowsheet '{flowsheet_path}': {e}")
169
+ raise typer.Exit(code=1)
135
170
 
136
171
  providers = extract_providers(flowsheet_path)
137
172
  logger.info(f"Reading providers from {flowsheet_path}...")
138
173
 
139
- # Categorize providers
140
- pip_providers: dict[str, dict] = {}
174
+ # Categorize providers, reusing the shared local/remote/pip classification.
141
175
  local_docker_providers: dict[str, dict] = {}
142
176
  remote_docker_providers: dict[str, dict] = {}
177
+ pip_providers: dict[str, dict] = {}
143
178
  for name, cfg in providers.items():
144
179
  ptype = cfg.get("type", "")
145
180
  if is_containerized(ptype):
@@ -151,7 +186,8 @@ def init(
151
186
  local_docker_providers[name] = {
152
187
  "type": ptype,
153
188
  "url": url,
154
- "docker_image": cfg.get("docker_image") or get_provider_docker_image(ptype),
189
+ "docker_image": cfg.get("docker_image")
190
+ or get_provider_docker_image(ptype),
155
191
  "port": port,
156
192
  }
157
193
  logger.info(f" {name}: type={ptype}, url={url} (Docker, local)")
@@ -159,7 +195,8 @@ def init(
159
195
  remote_docker_providers[name] = {
160
196
  "type": ptype,
161
197
  "url": url,
162
- "docker_image": cfg.get("docker_image") or get_provider_docker_image(ptype),
198
+ "docker_image": cfg.get("docker_image")
199
+ or get_provider_docker_image(ptype),
163
200
  }
164
201
  logger.info(
165
202
  f" {name}: type={ptype}, url={url} (Docker, remote — skipping compose)"
@@ -185,45 +222,48 @@ def init(
185
222
  # Providers with an explicit remote URL are assumed to be running elsewhere
186
223
  # (e.g. a cloud deployment of the ghcr.io image) and are not touched here.
187
224
  if local_docker_providers:
188
- compose_path = os.path.join(
189
- flowsheet_env_dir(pf_dir, flowsheet_path), "docker-compose.yml"
190
- )
191
- if os.path.exists(compose_path):
225
+ env_dir = flowsheet_env_dir(pf_dir, flowsheet_path)
226
+ compose_path = os.path.join(env_dir, "docker-compose.yml")
227
+ if os.path.exists(compose_path) and not force:
192
228
  logger.warning(
193
229
  f"Environment already initialized — reinitializing from {flowsheet_path}"
194
230
  )
195
231
 
196
- generate_compose(pf_dir, local_docker_providers, outputs_dir, flowsheet=flowsheet_path)
232
+ generate_compose(
233
+ pf_dir, local_docker_providers, outputs_dir, flowsheet=flowsheet_path
234
+ )
197
235
  logger.info(f"Generated {compose_path}")
198
236
 
199
- # Attempt docker compose pull — stream progress live
200
- try:
201
- logger.info("Pulling Docker images (this may take a while)...")
202
- process = subprocess.Popen(
203
- ["docker", "compose", "-f", compose_path, "pull"],
204
- stdout=subprocess.PIPE,
205
- stderr=subprocess.STDOUT,
206
- text=True,
207
- bufsize=1,
208
- )
209
- assert process.stdout is not None
210
- for line in process.stdout:
211
- line = line.rstrip()
212
- if line:
213
- logger.info(line)
214
- returncode = process.wait(timeout=600)
215
- if returncode == 0:
216
- logger.info("Pulled Docker images.")
217
- else:
237
+ if no_pull:
238
+ logger.info("Skipping Docker image pull (--no-pull).")
239
+ else:
240
+ # Attempt docker compose pull. Capture output and stream it via the
241
+ # logger; on timeout the child is reaped (subprocess.run kills it)
242
+ # so no orphaned/blocked process is left behind.
243
+ try:
244
+ logger.info("Pulling Docker images (this may take a while)...")
245
+ result = subprocess.run(
246
+ ["docker", "compose", "-f", compose_path, "pull"],
247
+ stdout=subprocess.PIPE,
248
+ stderr=subprocess.STDOUT,
249
+ text=True,
250
+ timeout=600,
251
+ )
252
+ for line in result.stdout.splitlines():
253
+ if line.strip():
254
+ logger.info(line)
255
+ if result.returncode == 0:
256
+ logger.info("Pulled Docker images.")
257
+ else:
258
+ logger.warning(
259
+ f"docker compose pull failed with exit code {result.returncode}."
260
+ )
261
+ except FileNotFoundError:
218
262
  logger.warning(
219
- f"docker compose pull failed with exit code {returncode}."
263
+ "Docker not found. Install Docker to use containerized providers."
220
264
  )
221
- except FileNotFoundError:
222
- logger.warning(
223
- "Docker not found. Install Docker to use containerized providers."
224
- )
225
- except subprocess.TimeoutExpired:
226
- logger.warning("docker compose pull timed out after 600s.")
265
+ except subprocess.TimeoutExpired:
266
+ logger.warning("docker compose pull timed out after 600s.")
227
267
 
228
268
  logger.info(
229
269
  "To start the containerized provider(s), run:\n"
@@ -242,15 +282,19 @@ def init(
242
282
  else:
243
283
  logger.info("No containerized providers — skipping Docker setup.")
244
284
 
245
- # Write lock file
285
+ # Write lock file, reusing the shared URL resolution for the recorded url.
246
286
  lock_providers: dict[str, dict] = {}
247
287
  for name, cfg in providers.items():
248
288
  ptype = cfg.get("type", "")
249
289
  if is_containerized(ptype):
290
+ url = cfg.get("url")
291
+ if not url:
292
+ port = get_provider_default_port(ptype) or 9000
293
+ url = f"http://localhost:{port}"
250
294
  lock_providers[name] = {
251
- "docker_image": cfg.get("docker_image") or get_provider_docker_image(ptype),
252
- "url": cfg.get("url")
253
- or f"http://localhost:{get_provider_default_port(ptype) or 9000}",
295
+ "docker_image": cfg.get("docker_image")
296
+ or get_provider_docker_image(ptype),
297
+ "url": url,
254
298
  }
255
299
  else:
256
300
  lock_providers[name] = {
@@ -261,5 +305,7 @@ def init(
261
305
  from .. import __version__ as pf_version
262
306
 
263
307
  write_lock(pf_dir, flowsheet_path, lock_providers, pf_version)
264
- logger.info(f"Wrote {os.path.join(flowsheet_env_dir(pf_dir, flowsheet_path), 'lock.json')}")
308
+ logger.info(
309
+ f"Wrote {os.path.join(flowsheet_env_dir(pf_dir, flowsheet_path), 'lock.json')}"
310
+ )
265
311
  logger.info(".processforge/ initialised successfully.")
@@ -0,0 +1,99 @@
1
+ """Shared run-persistence helpers for the ``pf run`` / ``pf apply`` CLIs.
2
+
3
+ Both commands build a :class:`~processforge.types.RunManifest`, save it to the
4
+ :class:`~processforge.persistence.archive.ProcessStateArchive`, and write a Zarr
5
+ copy of the standardized outputs. That logic was previously duplicated (with
6
+ slightly different bugs in each copy); it now lives here in one place.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import datetime
12
+ import hashlib
13
+ import json
14
+ import os
15
+ from typing import TYPE_CHECKING, Any
16
+
17
+ from loguru import logger
18
+
19
+ if TYPE_CHECKING:
20
+ from ..persistence.archive import ProcessStateArchive
21
+ from ..types import RunInfo
22
+
23
+ _RUN_ID_TS_FMT = "%Y%m%dT%H%M%SZ"
24
+
25
+
26
+ def make_run_id() -> str:
27
+ """Build a unique, sortable run id: ``<UTC timestamp>_<6 hex bytes>``."""
28
+ return (
29
+ datetime.datetime.now(datetime.timezone.utc).strftime(_RUN_ID_TS_FMT)
30
+ + "_"
31
+ + os.urandom(3).hex()
32
+ )
33
+
34
+
35
+ def flowsheet_hash(config: dict) -> str:
36
+ """Short (16-hex) sha256 of a flowsheet config for run/index correlation."""
37
+ return hashlib.sha256(
38
+ json.dumps(config, sort_keys=True, default=str).encode("utf-8")
39
+ ).hexdigest()[:16]
40
+
41
+
42
+ def persist_run(
43
+ archive: "ProcessStateArchive",
44
+ fs: Any,
45
+ run_id: str,
46
+ results: dict,
47
+ run_info: "RunInfo | dict",
48
+ config: dict,
49
+ base_name: str,
50
+ snapshot_id: str | None = None,
51
+ ) -> str:
52
+ """Persist a run's manifest + Zarr outputs into *archive*.
53
+
54
+ Args:
55
+ archive: The :class:`ProcessStateArchive` for this flowsheet.
56
+ fs: The flowsheet object that produced *results* (must expose
57
+ ``collect_outputs``). Either ``EOFlowsheet`` or ``Flowsheet``.
58
+ run_id: Unique run id (use :func:`make_run_id`).
59
+ results: The solve's stream/unit result dict (the *converged* one).
60
+ run_info: Provenance from
61
+ :func:`processforge.provenance.build_run_info`.
62
+ config: Validated flowsheet config (for the simulation mode).
63
+ base_name: Flowsheet basename used for archive paths.
64
+ snapshot_id: Optional snapshot id to stamp on the manifest (so a run
65
+ can be linked back to the state it was derived from).
66
+
67
+ Returns:
68
+ The run id that was persisted.
69
+ """
70
+ mode = config.get("simulation", {}).get("mode", "steady")
71
+ run_info_dump = (
72
+ run_info.model_dump() if hasattr(run_info, "model_dump") else run_info
73
+ )
74
+ manifest = fs.collect_outputs(
75
+ run_id, mode, base_name, provenance=run_info_dump
76
+ )
77
+ if snapshot_id is not None:
78
+ manifest.snapshot_id = snapshot_id
79
+
80
+ stream_results = {k: v for k, v in results.items() if not hasattr(v, "fields")}
81
+ archive.save_run(manifest, stream_results=stream_results)
82
+
83
+ # Always persist a Zarr copy of the standardized outputs (fields + artifacts)
84
+ # inside the archive, mirroring `pf run`.
85
+ try:
86
+ from ..result import relink_latest_results, save_results_zarr
87
+
88
+ run_results_dir = os.path.join(archive.path, "results", run_id)
89
+ save_results_zarr(
90
+ results,
91
+ os.path.join(run_results_dir, "results.zarr"),
92
+ run_info,
93
+ )
94
+ relink_latest_results(archive.path, run_id)
95
+ except Exception as e:
96
+ logger.warning(f"Failed to write results.zarr: {type(e).__name__}: {e}")
97
+
98
+ logger.info(f"Run saved : {run_id}")
99
+ return run_id