processforge 0.3.17__tar.gz → 0.3.19__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. {processforge-0.3.17 → processforge-0.3.19}/PKG-INFO +12 -6
  2. {processforge-0.3.17 → processforge-0.3.19}/README.md +12 -6
  3. {processforge-0.3.17 → processforge-0.3.19}/flowsheets/openmc/msre_eigenvalue.json +1 -1
  4. {processforge-0.3.17 → processforge-0.3.19}/pyproject.toml +1 -1
  5. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/__init__.py +1 -1
  6. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/api/serve.py +24 -0
  7. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/apply.py +14 -0
  8. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/run.py +15 -0
  9. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/persistence/artifact_store.py +51 -7
  10. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/container_client.py +1 -12
  11. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/errors.py +1 -0
  12. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/festim_provider.py +1 -0
  13. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/openmc_provider.py +2 -0
  14. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/result.py +47 -2
  15. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/types.py +4 -0
  16. processforge-0.3.19/src/processforge/utils/s3_upload.py +66 -0
  17. {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/PKG-INFO +12 -6
  18. {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/SOURCES.txt +1 -0
  19. {processforge-0.3.17 → processforge-0.3.19}/tests/test_cli.py +10 -5
  20. processforge-0.3.19/tests/test_container_client.py +93 -0
  21. {processforge-0.3.17 → processforge-0.3.19}/tests/test_festim_provider.py +11 -5
  22. processforge-0.3.19/tests/test_result_zarr.py +73 -0
  23. processforge-0.3.19/tests/test_s3_upload.py +64 -0
  24. processforge-0.3.17/src/processforge/utils/s3_upload.py +0 -69
  25. processforge-0.3.17/tests/test_container_client.py +0 -33
  26. processforge-0.3.17/tests/test_s3_upload.py +0 -81
  27. {processforge-0.3.17 → processforge-0.3.19}/LICENSE +0 -0
  28. {processforge-0.3.17 → processforge-0.3.19}/MANIFEST.in +0 -0
  29. {processforge-0.3.17 → processforge-0.3.19}/flowsheets/closed-loop-chain.json +0 -0
  30. {processforge-0.3.17 → processforge-0.3.19}/flowsheets/coupled/msre_coupled.json +0 -0
  31. {processforge-0.3.17 → processforge-0.3.19}/flowsheets/custom/crimson/MUWS_HB.json +0 -0
  32. {processforge-0.3.17 → processforge-0.3.19}/flowsheets/custom/simple_hyrdaulic_scheme.json +0 -0
  33. {processforge-0.3.17 → processforge-0.3.19}/flowsheets/festim/hydrogen_transport_1d.json +0 -0
  34. {processforge-0.3.17 → processforge-0.3.19}/flowsheets/festim/tds_tungsten.json +0 -0
  35. {processforge-0.3.17 → processforge-0.3.19}/flowsheets/hydraulic-chain.json +0 -0
  36. {processforge-0.3.17 → processforge-0.3.19}/setup.cfg +0 -0
  37. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/__main__.py +0 -0
  38. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/_schema.py +0 -0
  39. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/analysis/__init__.py +0 -0
  40. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/analysis/dof.py +0 -0
  41. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/api/__init__.py +0 -0
  42. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/__init__.py +0 -0
  43. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/common.py +0 -0
  44. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/diagram.py +0 -0
  45. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/display.py +0 -0
  46. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/export_fmu.py +0 -0
  47. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/export_modelica.py +0 -0
  48. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/init.py +0 -0
  49. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/plan.py +0 -0
  50. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/validate.py +0 -0
  51. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/compose.py +0 -0
  52. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/coupling.py +0 -0
  53. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/__init__.py +0 -0
  54. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/__init__.py +0 -0
  55. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/base.py +0 -0
  56. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/casadi_backend.py +0 -0
  57. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/pyomo_backend.py +0 -0
  58. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/scipy_backend.py +0 -0
  59. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/flowsheet.py +0 -0
  60. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/jacobian.py +0 -0
  61. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/mixin.py +0 -0
  62. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/solver.py +0 -0
  63. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/stream_var.py +0 -0
  64. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/__init__.py +0 -0
  65. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/flash_eo.py +0 -0
  66. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/heater_eo.py +0 -0
  67. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/pipes_eo.py +0 -0
  68. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/pump_eo.py +0 -0
  69. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/strainer_eo.py +0 -0
  70. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/valve_eo.py +0 -0
  71. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/flowsheet.py +0 -0
  72. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/fmu/__init__.py +0 -0
  73. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/fmu/_fmi_vars.py +0 -0
  74. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/fmu/builder.py +0 -0
  75. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/fmu/slave_template.py +0 -0
  76. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/lock.py +0 -0
  77. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/__init__.py +0 -0
  78. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/mo_writer.py +0 -0
  79. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/omc_runner.py +0 -0
  80. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/transpiler.py +0 -0
  81. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/unit_equations.py +0 -0
  82. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/output_collector.py +0 -0
  83. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/pcl/__init__.py +0 -0
  84. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/pcl/compiler.py +0 -0
  85. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/pcl/namespace.py +0 -0
  86. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/persistence/__init__.py +0 -0
  87. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/persistence/archive.py +0 -0
  88. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/provenance.py +0 -0
  89. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/__init__.py +0 -0
  90. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/base.py +0 -0
  91. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/base_jacobian_mixin.py +0 -0
  92. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/cantera_jacobian.py +0 -0
  93. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/cantera_provider.py +0 -0
  94. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/coolprop_provider.py +0 -0
  95. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/jacobian_contributor.py +0 -0
  96. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/manager.py +0 -0
  97. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/modelica_jacobian.py +0 -0
  98. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/modelica_provider.py +0 -0
  99. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/reference_state_registry.py +0 -0
  100. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/registry.py +0 -0
  101. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/quantity.py +0 -0
  102. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/result_schema.py +0 -0
  103. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/schemas/__init__.py +0 -0
  104. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/schemas/festim/festim_model.py +0 -0
  105. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/schemas/flowsheet_schema.json +0 -0
  106. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/schemas/openmc/openmc_model.py +0 -0
  107. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/simulate.py +0 -0
  108. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/solver.py +0 -0
  109. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/state.py +0 -0
  110. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/thermo.py +0 -0
  111. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/__init__.py +0 -0
  112. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/base.py +0 -0
  113. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/cstr.py +0 -0
  114. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/flash.py +0 -0
  115. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/heater.py +0 -0
  116. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/pfr.py +0 -0
  117. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/pipes.py +0 -0
  118. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/pump.py +0 -0
  119. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/registry.py +0 -0
  120. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/solver_unit.py +0 -0
  121. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/strainer.py +0 -0
  122. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/tank.py +0 -0
  123. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/valve.py +0 -0
  124. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/__init__.py +0 -0
  125. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/flowsheet_diagram.py +0 -0
  126. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/mermaid_diagram.py +0 -0
  127. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/topology.py +0 -0
  128. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/unit_consistency.py +0 -0
  129. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/validate_flowsheet.py +0 -0
  130. {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/validation.py +0 -0
  131. {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/dependency_links.txt +0 -0
  132. {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/entry_points.txt +0 -0
  133. {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/requires.txt +0 -0
  134. {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/top_level.txt +0 -0
  135. {processforge-0.3.17 → processforge-0.3.19}/tests/test_coupling.py +0 -0
  136. {processforge-0.3.17 → processforge-0.3.19}/tests/test_festim_schema.py +0 -0
  137. {processforge-0.3.17 → processforge-0.3.19}/tests/test_init.py +0 -0
  138. {processforge-0.3.17 → processforge-0.3.19}/tests/test_openmc_provider.py +0 -0
  139. {processforge-0.3.17 → processforge-0.3.19}/tests/test_provider_errors.py +0 -0
  140. {processforge-0.3.17 → processforge-0.3.19}/tests/test_providers.py +0 -0
  141. {processforge-0.3.17 → processforge-0.3.19}/tests/test_state.py +0 -0
  142. {processforge-0.3.17 → processforge-0.3.19}/tests/test_types.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: processforge
3
- Version: 0.3.17
3
+ Version: 0.3.19
4
4
  Summary: A Python-based process simulation framework for chemical engineering applications.
5
5
  Author-email: Processforge Team <team@processforge.dev>
6
6
  License-Expression: BSD-3-Clause
@@ -114,12 +114,18 @@ uv tool install "processforge[modelica]"
114
114
 
115
115
  4. **Look at the output**
116
116
 
117
- `pf apply` writes results under `outputs/`:
118
- - `*_results.zarr` — simulation results store (per-variable arrays, composition flattened)
119
- - `*_results.zarr.schema.json` — schema file describing streams, variables, dtypes, units, shapes, and run provenance
120
- - `*.pfstate/` — versioned snapshot store with a `latest` pointer
117
+ `pf apply` (and `pf run`) write a single unified archive under `outputs/`:
121
118
 
122
- For flowsheets with Tank units (dynamic), use `pf run` instead of `pf apply` to solve with the SM solver.
119
+ - `outputs/<base>.pfarchive/` the unified store for a flowsheet's solved state and outputs:
120
+ - `snapshots/` — Zarr store of converged state vectors (one group per successful `pf apply`), each with `x`/`x_delta` arrays and config/var-name/metadata attributes, plus a `latest` pointer (powers warm-start, drift detection, and homotopy).
121
+ - `runs/<run_id>.json` — the run manifest: every stream and unit engine output (values, units, dtypes, shapes) and run provenance (backend, version, flowsheet hash).
122
+ - `outputs/streams/<name>.json` — per-stream timeseries from the solve.
123
+ - `artifacts.json` — content-addressed registry of all output artifacts (local + remote URIs).
124
+ - `index.json` — `field_name → occurrences` index for fast cross-run lookups.
125
+ - `latest_run` — plain-text pointer to the most recent run.
126
+ - `outputs/<base>_divergence.json` — written only when both direct and homotopy solves fail on `pf apply`, capturing drifted params, solver stats, and top residual violators.
127
+
128
+ For dynamic flowsheets with Tank units, use `pf run` (not `pf apply`, which is steady-state EO only) to solve with the dynamic engine.
123
129
 
124
130
  ## Python API
125
131
 
@@ -52,12 +52,18 @@ uv tool install "processforge[modelica]"
52
52
 
53
53
  4. **Look at the output**
54
54
 
55
- `pf apply` writes results under `outputs/`:
56
- - `*_results.zarr` — simulation results store (per-variable arrays, composition flattened)
57
- - `*_results.zarr.schema.json` — schema file describing streams, variables, dtypes, units, shapes, and run provenance
58
- - `*.pfstate/` — versioned snapshot store with a `latest` pointer
59
-
60
- For flowsheets with Tank units (dynamic), use `pf run` instead of `pf apply` to solve with the SM solver.
55
+ `pf apply` (and `pf run`) write a single unified archive under `outputs/`:
56
+
57
+ - `outputs/<base>.pfarchive/`the unified store for a flowsheet's solved state and outputs:
58
+ - `snapshots/` — Zarr store of converged state vectors (one group per successful `pf apply`), each with `x`/`x_delta` arrays and config/var-name/metadata attributes, plus a `latest` pointer (powers warm-start, drift detection, and homotopy).
59
+ - `runs/<run_id>.json` — the run manifest: every stream and unit engine output (values, units, dtypes, shapes) and run provenance (backend, version, flowsheet hash).
60
+ - `outputs/streams/<name>.json` per-stream timeseries from the solve.
61
+ - `artifacts.json` — content-addressed registry of all output artifacts (local + remote URIs).
62
+ - `index.json` — `field_name → occurrences` index for fast cross-run lookups.
63
+ - `latest_run` — plain-text pointer to the most recent run.
64
+ - `outputs/<base>_divergence.json` — written only when both direct and homotopy solves fail on `pf apply`, capturing drifted params, solver stats, and top residual violators.
65
+
66
+ For dynamic flowsheets with Tank units, use `pf run` (not `pf apply`, which is steady-state EO only) to solve with the dynamic engine.
61
67
 
62
68
  ## Python API
63
69
 
@@ -7,7 +7,7 @@
7
7
  "providers": {
8
8
  "openmc": {
9
9
  "type": "openmc",
10
- "url": "http://localhost:9001",
10
+ "url": "https://processforge-openmc-production.up.railway.app",
11
11
  "output_dir": "openmc/msre_run",
12
12
  "cross_sections": "${OPENMC_DATA_ROOT}/cross_sections/cross_sections.xml"
13
13
  }
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "processforge"
7
- version = "0.3.17"
7
+ version = "0.3.19"
8
8
  description = "A Python-based process simulation framework for chemical engineering applications."
9
9
  readme = "README.md"
10
10
  license = "BSD-3-Clause"
@@ -29,7 +29,7 @@ from .units.flash import Flash
29
29
  from .units.heater import Heater
30
30
  from .eo import EOFlowsheet, EOSolver
31
31
 
32
- __version__ = "0.3.16"
32
+ __version__ = "0.3.19"
33
33
 
34
34
  __all__ = [
35
35
  "Flowsheet",
@@ -99,7 +99,31 @@ def run(body: dict):
99
99
  provider.teardown()
100
100
 
101
101
 
102
+ def _log_startup() -> None:
103
+ """Report where runs are kept and the S3 upload status at server start."""
104
+ root = os.environ.get("PROCESSFORGE_OUTPUT_DIR", "outputs")
105
+ logger.info(
106
+ f"Provider run outputs (scratch) are kept locally in: {root} "
107
+ f"(per-provider subdirs: outputs/openmc, outputs/festim)."
108
+ )
109
+ if os.environ.get("S3_BUCKET"):
110
+ bucket = os.environ["S3_BUCKET"]
111
+ logger.info(f"Remote artifact upload enabled → S3 bucket '{bucket}'.")
112
+ from processforge.utils.s3_upload import validate_s3
113
+
114
+ validate_s3() # fail fast on misconfigured credentials
115
+ else:
116
+ logger.warning(
117
+ "S3_BUCKET not set — remote upload disabled; runs are ephemeral "
118
+ "on this container."
119
+ )
120
+
121
+
122
+ _log_startup()
123
+
124
+
102
125
  if __name__ == "__main__":
103
126
  import uvicorn
104
127
 
105
128
  uvicorn.run(app, host="0.0.0.0", port=DEFAULT_PORT)
129
+
@@ -27,6 +27,20 @@ def _persist_run(archive, fs, results, run_info, config, base_name, snapshot_id)
27
27
  manifest.snapshot_id = snapshot_id
28
28
  stream_results = {k: v for k, v in results.items() if not hasattr(v, "fields")}
29
29
  archive.save_run(manifest, stream_results=stream_results)
30
+
31
+ # Always persist a Zarr copy of the standardized outputs (fields + artifacts)
32
+ # inside the archive, mirroring `pf run`.
33
+ try:
34
+ from ..result import save_results_zarr
35
+
36
+ save_results_zarr(
37
+ results,
38
+ os.path.join(archive.path, "results.zarr"),
39
+ run_info,
40
+ )
41
+ except Exception as e:
42
+ logger.warning(f"Failed to write results.zarr: {type(e).__name__}: {e}")
43
+
30
44
  logger.info(f"Run saved : {run_id}")
31
45
  return run_id
32
46
 
@@ -134,6 +134,21 @@ def run(
134
134
  }
135
135
  archive.save_run(manifest, stream_results=stream_results)
136
136
 
137
+ # Always persist a Zarr copy of the standardized outputs (fields + artifacts)
138
+ # inside the archive. Works for every provider (OpenMC, CoolProp, FESTIM,
139
+ # Cantera) and for both local and remote docker runs — remote artifacts
140
+ # reference their S3 remote_uris.
141
+ try:
142
+ from ..result import save_results_zarr
143
+
144
+ save_results_zarr(
145
+ results,
146
+ os.path.join(archive.path, "results.zarr"),
147
+ run_info,
148
+ )
149
+ except Exception as e:
150
+ logger.warning(f"Failed to write results.zarr: {type(e).__name__}: {e}")
151
+
137
152
  logger.info(f"Backend : {display_backend(config, getattr(fs, 'backend', 'dynamic'))}")
138
153
  logger.info(f"Run saved : {os.path.join(archive_path, 'runs', run_id + '.json')}")
139
154
 
@@ -1,7 +1,6 @@
1
1
  """Content-addressed artifact storage for simulation outputs.
2
2
 
3
- Replaces the old ``upload_directory_to_s3`` whole-directory dump. Providers
4
- declare their output files as :class:`~processforge.types.OutputArtifact`
3
+ Providers declare their output files as :class:`~processforge.types.OutputArtifact`
5
4
  records (with ``local_path``); the :class:`ArtifactStore` computes a content
6
5
  hash, uploads each to object storage under a deterministic, reproducible key,
7
6
  and fills in ``remote_uris``. The CLI and the container server both use this
@@ -120,13 +119,58 @@ class ArtifactStore:
120
119
  # Container-side convenience: persist all artifacts in engine outputs
121
120
  # ------------------------------------------------------------------
122
121
  def persist_outputs(self, outputs, *, run_id: str = "", flowsheet_hash: str = "", unit: str = ""):
123
- """Upload every artifact referenced by *outputs*.
122
+ """Upload every artifact referenced by *outputs* to object storage.
124
123
 
125
- Mutates the artifacts in place, filling ``remote_uris``. ``outputs``
126
- may be a single object or an iterable of them.
124
+ Uploads the explicitly-declared artifacts, then sweeps each output's
125
+ ``run_dir`` for any remaining files so *all* model outputs (not just the
126
+ ones a provider happened to declare) reach S3. Newly swept files are
127
+ appended to ``out.artifacts`` with ``remote_uris`` filled. Mutates the
128
+ artifacts in place. ``outputs`` may be a single object or an iterable.
127
129
  """
130
+ if not self.bucket:
131
+ logger.debug(
132
+ "Remote artifact upload disabled (S3_BUCKET not set); provider "
133
+ "run outputs remain in the container's local scratch only."
134
+ )
135
+ return
136
+
128
137
  items = outputs if isinstance(outputs, (list, tuple)) else [outputs]
138
+ uploaded = 0
139
+ local_only = 0
129
140
  for out in items:
141
+ seen_paths: set[str] = set()
130
142
  for art in getattr(out, "artifacts", []):
131
- if art.local_path and (self.bucket or os.environ.get("S3_BUCKET")):
132
- self.upload(art, run_id=run_id, flowsheet_hash=flowsheet_hash, unit=unit)
143
+ if not art.local_path:
144
+ continue
145
+ seen_paths.add(os.path.abspath(art.local_path))
146
+ self.upload(art, run_id=run_id, flowsheet_hash=flowsheet_hash, unit=unit)
147
+ if art.remote_uris:
148
+ uploaded += 1
149
+ else:
150
+ local_only += 1
151
+
152
+ run_dir = getattr(out, "run_dir", "") or ""
153
+ if run_dir and os.path.isdir(run_dir):
154
+ for root, _dirs, files in os.walk(run_dir):
155
+ for name in files:
156
+ path = os.path.abspath(os.path.join(root, name))
157
+ if path in seen_paths:
158
+ continue
159
+ seen_paths.add(path)
160
+ art = OutputArtifact(
161
+ name=name,
162
+ kind=_artifact_kind(name),
163
+ local_path=path,
164
+ source="local",
165
+ )
166
+ self.upload(art, run_id=run_id, flowsheet_hash=flowsheet_hash, unit=unit)
167
+ if art.remote_uris:
168
+ out.artifacts.append(art)
169
+ uploaded += 1
170
+ else:
171
+ local_only += 1
172
+
173
+ logger.info(
174
+ f"Artifact persistence: {uploaded} uploaded to S3, {local_only} local-only."
175
+ )
176
+
@@ -10,14 +10,13 @@ as a thin HTTP wrapper around that server.
10
10
  Request/response contract (matches ``processforge.api.serve``)
11
11
  ----------------------------------------------------------------
12
12
  * ``POST /run`` with a JSON body
13
- ``{"unit_config", "materials", "inlet", "output_dir", "provider_config"}``.
13
+ ``{"unit_config", "materials", "inlet", "provider_config"}``.
14
14
  * JSON response ``{"status", "sim_type", <scalars…>, "metadata"}`` deserialised
15
15
  into a :class:`~processforge.types.SimulationResult`.
16
16
  """
17
17
  from __future__ import annotations
18
18
 
19
19
  import json
20
- import os
21
20
  import urllib.error
22
21
  import urllib.request
23
22
  from typing import TYPE_CHECKING, Optional
@@ -54,7 +53,6 @@ class ContainerProviderClient(AbstractProvider):
54
53
  def __init__(self, provider_type: str):
55
54
  self._ptype = provider_type
56
55
  self._url: Optional[str] = None
57
- self._provider_output_dir: str = "outputs"
58
56
  self._provider_config: Optional["ProviderConfig"] = None
59
57
  self._materials: dict = {}
60
58
  self._initialized: bool = False
@@ -123,14 +121,6 @@ class ContainerProviderClient(AbstractProvider):
123
121
  """Store config, resolve URL, verify the Docker service is reachable."""
124
122
  self._url = self._resolve_url(provider_config)
125
123
 
126
- out_dir = os.path.expandvars(
127
- getattr(provider_config, "output_dir", "outputs") or "outputs"
128
- )
129
- if not os.path.isabs(out_dir):
130
- root = os.environ.get("PROCESSFORGE_OUTPUT_DIR", "outputs")
131
- out_dir = os.path.join(root, out_dir)
132
- self._provider_output_dir = out_dir
133
-
134
124
  # Material registry — serialised into /run and used by validate_material().
135
125
  self._materials = dict(flowsheet_config.materials)
136
126
  self._provider_config = provider_config
@@ -191,7 +181,6 @@ class ContainerProviderClient(AbstractProvider):
191
181
  "unit_config": self._serialize_unit_config(unit_config),
192
182
  "materials": self._serialize_materials(self._materials),
193
183
  "inlet": inlet,
194
- "output_dir": self._provider_output_dir,
195
184
  "provider_config": provider_config_dump,
196
185
  "run_id": run_id,
197
186
  "flowsheet_hash": flowsheet_hash,
@@ -236,6 +236,7 @@ def make_failed_output(
236
236
  sim_type=sim_type,
237
237
  unit=unit,
238
238
  error=err,
239
+ run_dir=str(getattr(run_dir, "resolve", lambda: run_dir)()),
239
240
  diagnostics={
240
241
  "run_dir": str(getattr(run_dir, "resolve", lambda: run_dir)()),
241
242
  "error": err.detail,
@@ -788,6 +788,7 @@ class FestimProvider(AbstractProvider):
788
788
  artifacts=artifacts,
789
789
  diagnostics=diagnostics,
790
790
  provenance=OutputProvenance(),
791
+ run_dir=str(run_dir),
791
792
  )
792
793
 
793
794
  def _resolve_run_dir(self) -> pathlib.Path:
@@ -832,6 +832,7 @@ class OpenMCProvider(AbstractProvider):
832
832
  fields=fields,
833
833
  artifacts=artifacts,
834
834
  diagnostics=diagnostics,
835
+ run_dir=str(run_dir),
835
836
  )
836
837
 
837
838
  return EngineOutput(
@@ -842,6 +843,7 @@ class OpenMCProvider(AbstractProvider):
842
843
  artifacts=artifacts,
843
844
  diagnostics=diagnostics,
844
845
  provenance=OutputProvenance(),
846
+ run_dir=str(run_dir),
845
847
  )
846
848
 
847
849
  def _resolve_run_dir(self) -> pathlib.Path:
@@ -1,3 +1,4 @@
1
+ import json
1
2
  import os
2
3
  import shutil
3
4
  from datetime import datetime, timezone
@@ -27,6 +28,8 @@ def _ensure_array(value):
27
28
 
28
29
  def _is_dynamic(results):
29
30
  for stream in results.values():
31
+ if not hasattr(stream, "get"):
32
+ continue
30
33
  time_series = stream.get("time")
31
34
  if isinstance(time_series, (list, tuple, np.ndarray)):
32
35
  return True
@@ -51,6 +54,34 @@ def _store_solver_unit(group, data):
51
54
  group.attrs[k] = v
52
55
 
53
56
 
57
+ def _store_engine_output(group, obj):
58
+ """Persist an ``EngineOutput``/``StreamOutput`` to a Zarr group.
59
+
60
+ Stores ``status``/``engine``/``sim_type`` as group attrs, each field as a
61
+ unit-bearing array, and a nested ``artifacts`` subgroup whose attrs hold the
62
+ serialized :class:`OutputArtifact` (local path for local runs, S3
63
+ ``remote_uris`` for remote docker providers).
64
+ """
65
+ group.attrs["status"] = getattr(obj, "status", "")
66
+ group.attrs["engine"] = getattr(obj, "engine", "")
67
+ group.attrs["sim_type"] = getattr(obj, "sim_type", "")
68
+
69
+ for f in getattr(obj, "fields", []):
70
+ val = _ensure_array(f.quantity.value)
71
+ arr = group.create_array(f.name, data=val)
72
+ arr.attrs["unit"] = f.quantity.unit or ""
73
+ if getattr(f.quantity, "std_dev", None) is not None:
74
+ arr.attrs["std_dev"] = f.quantity.std_dev
75
+ if getattr(f, "source", ""):
76
+ arr.attrs["source"] = f.source
77
+
78
+ arts = getattr(obj, "artifacts", [])
79
+ if arts:
80
+ ag = group.create_group("artifacts")
81
+ for art in arts:
82
+ ag.attrs[art.name] = json.dumps(art.model_dump())
83
+
84
+
54
85
  def _normalize_run_info(run_info: RunInfo | dict) -> dict:
55
86
  """Normalize dataclass or dict run_info input to a plain dict."""
56
87
  if isinstance(run_info, RunInfo):
@@ -212,8 +243,11 @@ def save_results_zarr(results, fname="results.zarr", run_info: RunInfo | dict |
212
243
 
213
244
  streams = {}
214
245
  solver_units = {}
246
+ engine_outputs = {}
215
247
  for name, data in results.items():
216
- if isinstance(data, dict) and "status" in data:
248
+ if hasattr(data, "fields"):
249
+ engine_outputs[name] = data
250
+ elif isinstance(data, dict) and "status" in data:
217
251
  solver_units[name] = data
218
252
  else:
219
253
  streams[name] = data
@@ -226,6 +260,10 @@ def save_results_zarr(results, fname="results.zarr", run_info: RunInfo | dict |
226
260
  group = root.create_group(name)
227
261
  _store_solver_unit(group, data)
228
262
 
263
+ for name, data in engine_outputs.items():
264
+ group = root.create_group(name)
265
+ _store_engine_output(group, data)
266
+
229
267
  if run_info is not None:
230
268
  _store_run_info(root, run_info)
231
269
 
@@ -270,8 +308,11 @@ def save_results_zarr_s3(results, s3_uri: str, run_info: RunInfo | dict | None =
270
308
 
271
309
  streams = {}
272
310
  solver_units = {}
311
+ engine_outputs = {}
273
312
  for name, data in results.items():
274
- if isinstance(data, dict) and "status" in data:
313
+ if hasattr(data, "fields"):
314
+ engine_outputs[name] = data
315
+ elif isinstance(data, dict) and "status" in data:
275
316
  solver_units[name] = data
276
317
  else:
277
318
  streams[name] = data
@@ -284,6 +325,10 @@ def save_results_zarr_s3(results, s3_uri: str, run_info: RunInfo | dict | None =
284
325
  group = root.create_group(name)
285
326
  _store_solver_unit(group, data)
286
327
 
328
+ for name, data in engine_outputs.items():
329
+ group = root.create_group(name)
330
+ _store_engine_output(group, data)
331
+
287
332
  if run_info is not None:
288
333
  _store_run_info(root, run_info)
289
334
 
@@ -477,6 +477,10 @@ class EngineOutput(BaseModel):
477
477
  # status == "failed"). source is always "provider" so a runtime engine error
478
478
  # is distinguishable from a flowsheet/setup validation error.
479
479
  error: Optional["ProviderRunError"] = None
480
+ # Local directory the run was executed in (provider scratch). Surfaced so
481
+ # the container server can sweep it for artifacts to upload to object
482
+ # storage, and so forensics can trace where outputs were produced.
483
+ run_dir: str = ""
480
484
 
481
485
  # -- convenience accessors -------------------------------------------
482
486
  def get_field(self, name: str) -> Optional[OutputField]:
@@ -0,0 +1,66 @@
1
+ """S3 storage-options helper.
2
+
3
+ Builds s3fs storage options (credentials, endpoint, region) from the
4
+ container environment for use by :class:`processforge.persistence.artifact_store.ArtifactStore`.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import os
10
+
11
+
12
+ def s3_storage_options() -> dict:
13
+ """Build s3fs storage options from environment variables.
14
+
15
+ Returns an empty dict when no credentials are configured so s3fs falls
16
+ back to its default credential chain (e.g. instance/role-based auth).
17
+ """
18
+ opts: dict = {}
19
+ key = os.environ.get("S3_ACCESS_KEY")
20
+ secret = os.environ.get("S3_SECRET_KEY")
21
+ if key:
22
+ opts["key"] = key
23
+ if secret:
24
+ opts["secret"] = secret
25
+ client_kwargs = {
26
+ k: v
27
+ for k, v in {
28
+ "endpoint_url": os.environ.get("S3_ENDPOINT_URL"),
29
+ "region_name": os.environ.get("S3_REGION_NAME", "ams3"),
30
+ }.items()
31
+ if v
32
+ }
33
+ if client_kwargs:
34
+ opts["client_kwargs"] = client_kwargs
35
+ return opts
36
+
37
+
38
+ def validate_s3() -> None:
39
+ """Fail fast if S3 upload is requested but storage is unreachable.
40
+
41
+ Only acts when ``S3_BUCKET`` is set. Raises ``RuntimeError`` naming any
42
+ missing credentials/env vars when the bucket cannot be reached, so a
43
+ misconfigured container deployment surfaces the problem at startup rather
44
+ than silently producing outputs with no ``remote_uris``.
45
+ """
46
+ bucket = os.environ.get("S3_BUCKET")
47
+ if not bucket:
48
+ return
49
+ try:
50
+ import s3fs
51
+
52
+ fs = s3fs.S3FileSystem(**s3_storage_options())
53
+ fs.ls(bucket)
54
+ except Exception as exc: # noqa: BLE001
55
+ missing = [
56
+ v
57
+ for v in ("S3_ACCESS_KEY", "S3_SECRET_KEY", "S3_ENDPOINT_URL")
58
+ if not os.environ.get(v)
59
+ ]
60
+ hint = f" Missing env vars: {missing}." if missing else ""
61
+ raise RuntimeError(
62
+ f"S3 upload requested (S3_BUCKET='{bucket}') but storage is "
63
+ f"unreachable: {exc}.{hint} Set S3_BUCKET + credentials, or unset "
64
+ f"S3_BUCKET to disable remote upload."
65
+ ) from exc
66
+
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: processforge
3
- Version: 0.3.17
3
+ Version: 0.3.19
4
4
  Summary: A Python-based process simulation framework for chemical engineering applications.
5
5
  Author-email: Processforge Team <team@processforge.dev>
6
6
  License-Expression: BSD-3-Clause
@@ -114,12 +114,18 @@ uv tool install "processforge[modelica]"
114
114
 
115
115
  4. **Look at the output**
116
116
 
117
- `pf apply` writes results under `outputs/`:
118
- - `*_results.zarr` — simulation results store (per-variable arrays, composition flattened)
119
- - `*_results.zarr.schema.json` — schema file describing streams, variables, dtypes, units, shapes, and run provenance
120
- - `*.pfstate/` — versioned snapshot store with a `latest` pointer
117
+ `pf apply` (and `pf run`) write a single unified archive under `outputs/`:
121
118
 
122
- For flowsheets with Tank units (dynamic), use `pf run` instead of `pf apply` to solve with the SM solver.
119
+ - `outputs/<base>.pfarchive/` the unified store for a flowsheet's solved state and outputs:
120
+ - `snapshots/` — Zarr store of converged state vectors (one group per successful `pf apply`), each with `x`/`x_delta` arrays and config/var-name/metadata attributes, plus a `latest` pointer (powers warm-start, drift detection, and homotopy).
121
+ - `runs/<run_id>.json` — the run manifest: every stream and unit engine output (values, units, dtypes, shapes) and run provenance (backend, version, flowsheet hash).
122
+ - `outputs/streams/<name>.json` — per-stream timeseries from the solve.
123
+ - `artifacts.json` — content-addressed registry of all output artifacts (local + remote URIs).
124
+ - `index.json` — `field_name → occurrences` index for fast cross-run lookups.
125
+ - `latest_run` — plain-text pointer to the most recent run.
126
+ - `outputs/<base>_divergence.json` — written only when both direct and homotopy solves fail on `pf apply`, capturing drifted params, solver stats, and top residual violators.
127
+
128
+ For dynamic flowsheets with Tank units, use `pf run` (not `pf apply`, which is steady-state EO only) to solve with the dynamic engine.
123
129
 
124
130
  ## Python API
125
131
 
@@ -131,6 +131,7 @@ tests/test_init.py
131
131
  tests/test_openmc_provider.py
132
132
  tests/test_provider_errors.py
133
133
  tests/test_providers.py
134
+ tests/test_result_zarr.py
134
135
  tests/test_s3_upload.py
135
136
  tests/test_state.py
136
137
  tests/test_types.py
@@ -388,7 +388,8 @@ class TestCmdRun:
388
388
  return_value=self._containerized_config(flowsheet)), \
389
389
  patch.object(run_mod, "check_providers"), \
390
390
  patch.object(run_mod, "EOFlowsheet", return_value=mock_fs), \
391
- patch.object(run_mod, "ProcessStateArchive", MagicMock()):
391
+ patch.object(run_mod, "ProcessStateArchive", MagicMock()), \
392
+ patch("processforge.result.save_results_zarr", lambda *a, **k: None):
392
393
  run(flowsheet=str(flowsheet), export_images=False)
393
394
 
394
395
  def test_coolprop_flowsheet_no_lifecycle(self, tmp_path):
@@ -414,7 +415,8 @@ class TestCmdRun:
414
415
  "_config_path": str(flowsheet)}), \
415
416
  patch.object(run_mod, "check_providers"), \
416
417
  patch.object(run_mod, "EOFlowsheet", return_value=mock_fs), \
417
- patch.object(run_mod, "ProcessStateArchive", MagicMock()):
418
+ patch.object(run_mod, "ProcessStateArchive", MagicMock()), \
419
+ patch("processforge.result.save_results_zarr", lambda *a, **k: None):
418
420
  run(flowsheet=str(flowsheet), export_images=False)
419
421
 
420
422
  def test_no_compose_no_lifecycle(self, tmp_path):
@@ -437,7 +439,8 @@ class TestCmdRun:
437
439
  return_value=self._containerized_config(flowsheet)), \
438
440
  patch.object(run_mod, "check_providers"), \
439
441
  patch.object(run_mod, "EOFlowsheet", return_value=mock_fs), \
440
- patch.object(run_mod, "ProcessStateArchive", MagicMock()):
442
+ patch.object(run_mod, "ProcessStateArchive", MagicMock()), \
443
+ patch("processforge.result.save_results_zarr", lambda *a, **k: None):
441
444
  mock_os.path.exists.return_value = False # no compose file
442
445
  run(flowsheet=str(flowsheet), export_images=False)
443
446
 
@@ -523,7 +526,8 @@ class TestCmdApply:
523
526
  patch.object(apply_mod, "check_providers"), \
524
527
  patch.object(apply_mod, "EOFlowsheet", return_value=mock_fs), \
525
528
  patch.object(apply_mod, "save_snapshot", return_value="snap-1"), \
526
- patch.object(apply_mod, "load_state_manager", return_value=(MagicMock(), None)):
529
+ patch.object(apply_mod, "load_state_manager", return_value=(MagicMock(), None)), \
530
+ patch("processforge.result.save_results_zarr", lambda *a, **k: None):
527
531
  apply(flowsheet=str(flowsheet))
528
532
 
529
533
  def test_coolprop_flowsheet_no_lifecycle(self, tmp_path):
@@ -552,7 +556,8 @@ class TestCmdApply:
552
556
  patch.object(apply_mod, "check_providers"), \
553
557
  patch.object(apply_mod, "EOFlowsheet", return_value=mock_fs), \
554
558
  patch.object(apply_mod, "save_snapshot", return_value="snap-1"), \
555
- patch.object(apply_mod, "load_state_manager", return_value=(MagicMock(), None)):
559
+ patch.object(apply_mod, "load_state_manager", return_value=(MagicMock(), None)), \
560
+ patch("processforge.result.save_results_zarr", lambda *a, **k: None):
556
561
  apply(flowsheet=str(flowsheet))
557
562
 
558
563