processforge 0.3.17__tar.gz → 0.3.19__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {processforge-0.3.17 → processforge-0.3.19}/PKG-INFO +12 -6
- {processforge-0.3.17 → processforge-0.3.19}/README.md +12 -6
- {processforge-0.3.17 → processforge-0.3.19}/flowsheets/openmc/msre_eigenvalue.json +1 -1
- {processforge-0.3.17 → processforge-0.3.19}/pyproject.toml +1 -1
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/__init__.py +1 -1
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/api/serve.py +24 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/apply.py +14 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/run.py +15 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/persistence/artifact_store.py +51 -7
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/container_client.py +1 -12
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/errors.py +1 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/festim_provider.py +1 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/openmc_provider.py +2 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/result.py +47 -2
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/types.py +4 -0
- processforge-0.3.19/src/processforge/utils/s3_upload.py +66 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/PKG-INFO +12 -6
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/SOURCES.txt +1 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_cli.py +10 -5
- processforge-0.3.19/tests/test_container_client.py +93 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_festim_provider.py +11 -5
- processforge-0.3.19/tests/test_result_zarr.py +73 -0
- processforge-0.3.19/tests/test_s3_upload.py +64 -0
- processforge-0.3.17/src/processforge/utils/s3_upload.py +0 -69
- processforge-0.3.17/tests/test_container_client.py +0 -33
- processforge-0.3.17/tests/test_s3_upload.py +0 -81
- {processforge-0.3.17 → processforge-0.3.19}/LICENSE +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/MANIFEST.in +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/flowsheets/closed-loop-chain.json +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/flowsheets/coupled/msre_coupled.json +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/flowsheets/custom/crimson/MUWS_HB.json +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/flowsheets/custom/simple_hyrdaulic_scheme.json +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/flowsheets/festim/hydrogen_transport_1d.json +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/flowsheets/festim/tds_tungsten.json +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/flowsheets/hydraulic-chain.json +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/setup.cfg +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/__main__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/_schema.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/analysis/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/analysis/dof.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/api/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/common.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/diagram.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/display.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/export_fmu.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/export_modelica.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/init.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/plan.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/cli/validate.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/compose.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/coupling.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/base.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/casadi_backend.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/pyomo_backend.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/backends/scipy_backend.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/flowsheet.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/jacobian.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/mixin.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/solver.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/stream_var.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/flash_eo.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/heater_eo.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/pipes_eo.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/pump_eo.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/strainer_eo.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/eo/units/valve_eo.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/flowsheet.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/fmu/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/fmu/_fmi_vars.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/fmu/builder.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/fmu/slave_template.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/lock.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/mo_writer.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/omc_runner.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/transpiler.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/modelica/unit_equations.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/output_collector.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/pcl/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/pcl/compiler.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/pcl/namespace.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/persistence/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/persistence/archive.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/provenance.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/base.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/base_jacobian_mixin.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/cantera_jacobian.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/cantera_provider.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/coolprop_provider.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/jacobian_contributor.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/manager.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/modelica_jacobian.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/modelica_provider.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/reference_state_registry.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/providers/registry.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/quantity.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/result_schema.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/schemas/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/schemas/festim/festim_model.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/schemas/flowsheet_schema.json +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/schemas/openmc/openmc_model.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/simulate.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/solver.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/state.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/thermo.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/base.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/cstr.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/flash.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/heater.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/pfr.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/pipes.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/pump.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/registry.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/solver_unit.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/strainer.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/tank.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/units/valve.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/__init__.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/flowsheet_diagram.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/mermaid_diagram.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/topology.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/unit_consistency.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/validate_flowsheet.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge/utils/validation.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/dependency_links.txt +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/entry_points.txt +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/requires.txt +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/src/processforge.egg-info/top_level.txt +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_coupling.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_festim_schema.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_init.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_openmc_provider.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_provider_errors.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_providers.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_state.py +0 -0
- {processforge-0.3.17 → processforge-0.3.19}/tests/test_types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: processforge
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.19
|
|
4
4
|
Summary: A Python-based process simulation framework for chemical engineering applications.
|
|
5
5
|
Author-email: Processforge Team <team@processforge.dev>
|
|
6
6
|
License-Expression: BSD-3-Clause
|
|
@@ -114,12 +114,18 @@ uv tool install "processforge[modelica]"
|
|
|
114
114
|
|
|
115
115
|
4. **Look at the output**
|
|
116
116
|
|
|
117
|
-
|
|
118
|
-
- `*_results.zarr` — simulation results store (per-variable arrays, composition flattened)
|
|
119
|
-
- `*_results.zarr.schema.json` — schema file describing streams, variables, dtypes, units, shapes, and run provenance
|
|
120
|
-
- `*.pfstate/` — versioned snapshot store with a `latest` pointer
|
|
117
|
+
`pf apply` (and `pf run`) write a single unified archive under `outputs/`:
|
|
121
118
|
|
|
122
|
-
|
|
119
|
+
- `outputs/<base>.pfarchive/` — the unified store for a flowsheet's solved state and outputs:
|
|
120
|
+
- `snapshots/` — Zarr store of converged state vectors (one group per successful `pf apply`), each with `x`/`x_delta` arrays and config/var-name/metadata attributes, plus a `latest` pointer (powers warm-start, drift detection, and homotopy).
|
|
121
|
+
- `runs/<run_id>.json` — the run manifest: every stream and unit engine output (values, units, dtypes, shapes) and run provenance (backend, version, flowsheet hash).
|
|
122
|
+
- `outputs/streams/<name>.json` — per-stream timeseries from the solve.
|
|
123
|
+
- `artifacts.json` — content-addressed registry of all output artifacts (local + remote URIs).
|
|
124
|
+
- `index.json` — `field_name → occurrences` index for fast cross-run lookups.
|
|
125
|
+
- `latest_run` — plain-text pointer to the most recent run.
|
|
126
|
+
- `outputs/<base>_divergence.json` — written only when both direct and homotopy solves fail on `pf apply`, capturing drifted params, solver stats, and top residual violators.
|
|
127
|
+
|
|
128
|
+
For dynamic flowsheets with Tank units, use `pf run` (not `pf apply`, which is steady-state EO only) to solve with the dynamic engine.
|
|
123
129
|
|
|
124
130
|
## Python API
|
|
125
131
|
|
|
@@ -52,12 +52,18 @@ uv tool install "processforge[modelica]"
|
|
|
52
52
|
|
|
53
53
|
4. **Look at the output**
|
|
54
54
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
55
|
+
`pf apply` (and `pf run`) write a single unified archive under `outputs/`:
|
|
56
|
+
|
|
57
|
+
- `outputs/<base>.pfarchive/` — the unified store for a flowsheet's solved state and outputs:
|
|
58
|
+
- `snapshots/` — Zarr store of converged state vectors (one group per successful `pf apply`), each with `x`/`x_delta` arrays and config/var-name/metadata attributes, plus a `latest` pointer (powers warm-start, drift detection, and homotopy).
|
|
59
|
+
- `runs/<run_id>.json` — the run manifest: every stream and unit engine output (values, units, dtypes, shapes) and run provenance (backend, version, flowsheet hash).
|
|
60
|
+
- `outputs/streams/<name>.json` — per-stream timeseries from the solve.
|
|
61
|
+
- `artifacts.json` — content-addressed registry of all output artifacts (local + remote URIs).
|
|
62
|
+
- `index.json` — `field_name → occurrences` index for fast cross-run lookups.
|
|
63
|
+
- `latest_run` — plain-text pointer to the most recent run.
|
|
64
|
+
- `outputs/<base>_divergence.json` — written only when both direct and homotopy solves fail on `pf apply`, capturing drifted params, solver stats, and top residual violators.
|
|
65
|
+
|
|
66
|
+
For dynamic flowsheets with Tank units, use `pf run` (not `pf apply`, which is steady-state EO only) to solve with the dynamic engine.
|
|
61
67
|
|
|
62
68
|
## Python API
|
|
63
69
|
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
"providers": {
|
|
8
8
|
"openmc": {
|
|
9
9
|
"type": "openmc",
|
|
10
|
-
"url": "
|
|
10
|
+
"url": "https://processforge-openmc-production.up.railway.app",
|
|
11
11
|
"output_dir": "openmc/msre_run",
|
|
12
12
|
"cross_sections": "${OPENMC_DATA_ROOT}/cross_sections/cross_sections.xml"
|
|
13
13
|
}
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "processforge"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.19"
|
|
8
8
|
description = "A Python-based process simulation framework for chemical engineering applications."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "BSD-3-Clause"
|
|
@@ -99,7 +99,31 @@ def run(body: dict):
|
|
|
99
99
|
provider.teardown()
|
|
100
100
|
|
|
101
101
|
|
|
102
|
+
def _log_startup() -> None:
|
|
103
|
+
"""Report where runs are kept and the S3 upload status at server start."""
|
|
104
|
+
root = os.environ.get("PROCESSFORGE_OUTPUT_DIR", "outputs")
|
|
105
|
+
logger.info(
|
|
106
|
+
f"Provider run outputs (scratch) are kept locally in: {root} "
|
|
107
|
+
f"(per-provider subdirs: outputs/openmc, outputs/festim)."
|
|
108
|
+
)
|
|
109
|
+
if os.environ.get("S3_BUCKET"):
|
|
110
|
+
bucket = os.environ["S3_BUCKET"]
|
|
111
|
+
logger.info(f"Remote artifact upload enabled → S3 bucket '{bucket}'.")
|
|
112
|
+
from processforge.utils.s3_upload import validate_s3
|
|
113
|
+
|
|
114
|
+
validate_s3() # fail fast on misconfigured credentials
|
|
115
|
+
else:
|
|
116
|
+
logger.warning(
|
|
117
|
+
"S3_BUCKET not set — remote upload disabled; runs are ephemeral "
|
|
118
|
+
"on this container."
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
_log_startup()
|
|
123
|
+
|
|
124
|
+
|
|
102
125
|
if __name__ == "__main__":
|
|
103
126
|
import uvicorn
|
|
104
127
|
|
|
105
128
|
uvicorn.run(app, host="0.0.0.0", port=DEFAULT_PORT)
|
|
129
|
+
|
|
@@ -27,6 +27,20 @@ def _persist_run(archive, fs, results, run_info, config, base_name, snapshot_id)
|
|
|
27
27
|
manifest.snapshot_id = snapshot_id
|
|
28
28
|
stream_results = {k: v for k, v in results.items() if not hasattr(v, "fields")}
|
|
29
29
|
archive.save_run(manifest, stream_results=stream_results)
|
|
30
|
+
|
|
31
|
+
# Always persist a Zarr copy of the standardized outputs (fields + artifacts)
|
|
32
|
+
# inside the archive, mirroring `pf run`.
|
|
33
|
+
try:
|
|
34
|
+
from ..result import save_results_zarr
|
|
35
|
+
|
|
36
|
+
save_results_zarr(
|
|
37
|
+
results,
|
|
38
|
+
os.path.join(archive.path, "results.zarr"),
|
|
39
|
+
run_info,
|
|
40
|
+
)
|
|
41
|
+
except Exception as e:
|
|
42
|
+
logger.warning(f"Failed to write results.zarr: {type(e).__name__}: {e}")
|
|
43
|
+
|
|
30
44
|
logger.info(f"Run saved : {run_id}")
|
|
31
45
|
return run_id
|
|
32
46
|
|
|
@@ -134,6 +134,21 @@ def run(
|
|
|
134
134
|
}
|
|
135
135
|
archive.save_run(manifest, stream_results=stream_results)
|
|
136
136
|
|
|
137
|
+
# Always persist a Zarr copy of the standardized outputs (fields + artifacts)
|
|
138
|
+
# inside the archive. Works for every provider (OpenMC, CoolProp, FESTIM,
|
|
139
|
+
# Cantera) and for both local and remote docker runs — remote artifacts
|
|
140
|
+
# reference their S3 remote_uris.
|
|
141
|
+
try:
|
|
142
|
+
from ..result import save_results_zarr
|
|
143
|
+
|
|
144
|
+
save_results_zarr(
|
|
145
|
+
results,
|
|
146
|
+
os.path.join(archive.path, "results.zarr"),
|
|
147
|
+
run_info,
|
|
148
|
+
)
|
|
149
|
+
except Exception as e:
|
|
150
|
+
logger.warning(f"Failed to write results.zarr: {type(e).__name__}: {e}")
|
|
151
|
+
|
|
137
152
|
logger.info(f"Backend : {display_backend(config, getattr(fs, 'backend', 'dynamic'))}")
|
|
138
153
|
logger.info(f"Run saved : {os.path.join(archive_path, 'runs', run_id + '.json')}")
|
|
139
154
|
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"""Content-addressed artifact storage for simulation outputs.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
declare their output files as :class:`~processforge.types.OutputArtifact`
|
|
3
|
+
Providers declare their output files as :class:`~processforge.types.OutputArtifact`
|
|
5
4
|
records (with ``local_path``); the :class:`ArtifactStore` computes a content
|
|
6
5
|
hash, uploads each to object storage under a deterministic, reproducible key,
|
|
7
6
|
and fills in ``remote_uris``. The CLI and the container server both use this
|
|
@@ -120,13 +119,58 @@ class ArtifactStore:
|
|
|
120
119
|
# Container-side convenience: persist all artifacts in engine outputs
|
|
121
120
|
# ------------------------------------------------------------------
|
|
122
121
|
def persist_outputs(self, outputs, *, run_id: str = "", flowsheet_hash: str = "", unit: str = ""):
|
|
123
|
-
"""Upload every artifact referenced by *outputs
|
|
122
|
+
"""Upload every artifact referenced by *outputs* to object storage.
|
|
124
123
|
|
|
125
|
-
|
|
126
|
-
|
|
124
|
+
Uploads the explicitly-declared artifacts, then sweeps each output's
|
|
125
|
+
``run_dir`` for any remaining files so *all* model outputs (not just the
|
|
126
|
+
ones a provider happened to declare) reach S3. Newly swept files are
|
|
127
|
+
appended to ``out.artifacts`` with ``remote_uris`` filled. Mutates the
|
|
128
|
+
artifacts in place. ``outputs`` may be a single object or an iterable.
|
|
127
129
|
"""
|
|
130
|
+
if not self.bucket:
|
|
131
|
+
logger.debug(
|
|
132
|
+
"Remote artifact upload disabled (S3_BUCKET not set); provider "
|
|
133
|
+
"run outputs remain in the container's local scratch only."
|
|
134
|
+
)
|
|
135
|
+
return
|
|
136
|
+
|
|
128
137
|
items = outputs if isinstance(outputs, (list, tuple)) else [outputs]
|
|
138
|
+
uploaded = 0
|
|
139
|
+
local_only = 0
|
|
129
140
|
for out in items:
|
|
141
|
+
seen_paths: set[str] = set()
|
|
130
142
|
for art in getattr(out, "artifacts", []):
|
|
131
|
-
if art.local_path
|
|
132
|
-
|
|
143
|
+
if not art.local_path:
|
|
144
|
+
continue
|
|
145
|
+
seen_paths.add(os.path.abspath(art.local_path))
|
|
146
|
+
self.upload(art, run_id=run_id, flowsheet_hash=flowsheet_hash, unit=unit)
|
|
147
|
+
if art.remote_uris:
|
|
148
|
+
uploaded += 1
|
|
149
|
+
else:
|
|
150
|
+
local_only += 1
|
|
151
|
+
|
|
152
|
+
run_dir = getattr(out, "run_dir", "") or ""
|
|
153
|
+
if run_dir and os.path.isdir(run_dir):
|
|
154
|
+
for root, _dirs, files in os.walk(run_dir):
|
|
155
|
+
for name in files:
|
|
156
|
+
path = os.path.abspath(os.path.join(root, name))
|
|
157
|
+
if path in seen_paths:
|
|
158
|
+
continue
|
|
159
|
+
seen_paths.add(path)
|
|
160
|
+
art = OutputArtifact(
|
|
161
|
+
name=name,
|
|
162
|
+
kind=_artifact_kind(name),
|
|
163
|
+
local_path=path,
|
|
164
|
+
source="local",
|
|
165
|
+
)
|
|
166
|
+
self.upload(art, run_id=run_id, flowsheet_hash=flowsheet_hash, unit=unit)
|
|
167
|
+
if art.remote_uris:
|
|
168
|
+
out.artifacts.append(art)
|
|
169
|
+
uploaded += 1
|
|
170
|
+
else:
|
|
171
|
+
local_only += 1
|
|
172
|
+
|
|
173
|
+
logger.info(
|
|
174
|
+
f"Artifact persistence: {uploaded} uploaded to S3, {local_only} local-only."
|
|
175
|
+
)
|
|
176
|
+
|
|
@@ -10,14 +10,13 @@ as a thin HTTP wrapper around that server.
|
|
|
10
10
|
Request/response contract (matches ``processforge.api.serve``)
|
|
11
11
|
----------------------------------------------------------------
|
|
12
12
|
* ``POST /run`` with a JSON body
|
|
13
|
-
``{"unit_config", "materials", "inlet", "
|
|
13
|
+
``{"unit_config", "materials", "inlet", "provider_config"}``.
|
|
14
14
|
* JSON response ``{"status", "sim_type", <scalars…>, "metadata"}`` deserialised
|
|
15
15
|
into a :class:`~processforge.types.SimulationResult`.
|
|
16
16
|
"""
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
19
|
import json
|
|
20
|
-
import os
|
|
21
20
|
import urllib.error
|
|
22
21
|
import urllib.request
|
|
23
22
|
from typing import TYPE_CHECKING, Optional
|
|
@@ -54,7 +53,6 @@ class ContainerProviderClient(AbstractProvider):
|
|
|
54
53
|
def __init__(self, provider_type: str):
|
|
55
54
|
self._ptype = provider_type
|
|
56
55
|
self._url: Optional[str] = None
|
|
57
|
-
self._provider_output_dir: str = "outputs"
|
|
58
56
|
self._provider_config: Optional["ProviderConfig"] = None
|
|
59
57
|
self._materials: dict = {}
|
|
60
58
|
self._initialized: bool = False
|
|
@@ -123,14 +121,6 @@ class ContainerProviderClient(AbstractProvider):
|
|
|
123
121
|
"""Store config, resolve URL, verify the Docker service is reachable."""
|
|
124
122
|
self._url = self._resolve_url(provider_config)
|
|
125
123
|
|
|
126
|
-
out_dir = os.path.expandvars(
|
|
127
|
-
getattr(provider_config, "output_dir", "outputs") or "outputs"
|
|
128
|
-
)
|
|
129
|
-
if not os.path.isabs(out_dir):
|
|
130
|
-
root = os.environ.get("PROCESSFORGE_OUTPUT_DIR", "outputs")
|
|
131
|
-
out_dir = os.path.join(root, out_dir)
|
|
132
|
-
self._provider_output_dir = out_dir
|
|
133
|
-
|
|
134
124
|
# Material registry — serialised into /run and used by validate_material().
|
|
135
125
|
self._materials = dict(flowsheet_config.materials)
|
|
136
126
|
self._provider_config = provider_config
|
|
@@ -191,7 +181,6 @@ class ContainerProviderClient(AbstractProvider):
|
|
|
191
181
|
"unit_config": self._serialize_unit_config(unit_config),
|
|
192
182
|
"materials": self._serialize_materials(self._materials),
|
|
193
183
|
"inlet": inlet,
|
|
194
|
-
"output_dir": self._provider_output_dir,
|
|
195
184
|
"provider_config": provider_config_dump,
|
|
196
185
|
"run_id": run_id,
|
|
197
186
|
"flowsheet_hash": flowsheet_hash,
|
|
@@ -832,6 +832,7 @@ class OpenMCProvider(AbstractProvider):
|
|
|
832
832
|
fields=fields,
|
|
833
833
|
artifacts=artifacts,
|
|
834
834
|
diagnostics=diagnostics,
|
|
835
|
+
run_dir=str(run_dir),
|
|
835
836
|
)
|
|
836
837
|
|
|
837
838
|
return EngineOutput(
|
|
@@ -842,6 +843,7 @@ class OpenMCProvider(AbstractProvider):
|
|
|
842
843
|
artifacts=artifacts,
|
|
843
844
|
diagnostics=diagnostics,
|
|
844
845
|
provenance=OutputProvenance(),
|
|
846
|
+
run_dir=str(run_dir),
|
|
845
847
|
)
|
|
846
848
|
|
|
847
849
|
def _resolve_run_dir(self) -> pathlib.Path:
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import os
|
|
2
3
|
import shutil
|
|
3
4
|
from datetime import datetime, timezone
|
|
@@ -27,6 +28,8 @@ def _ensure_array(value):
|
|
|
27
28
|
|
|
28
29
|
def _is_dynamic(results):
|
|
29
30
|
for stream in results.values():
|
|
31
|
+
if not hasattr(stream, "get"):
|
|
32
|
+
continue
|
|
30
33
|
time_series = stream.get("time")
|
|
31
34
|
if isinstance(time_series, (list, tuple, np.ndarray)):
|
|
32
35
|
return True
|
|
@@ -51,6 +54,34 @@ def _store_solver_unit(group, data):
|
|
|
51
54
|
group.attrs[k] = v
|
|
52
55
|
|
|
53
56
|
|
|
57
|
+
def _store_engine_output(group, obj):
|
|
58
|
+
"""Persist an ``EngineOutput``/``StreamOutput`` to a Zarr group.
|
|
59
|
+
|
|
60
|
+
Stores ``status``/``engine``/``sim_type`` as group attrs, each field as a
|
|
61
|
+
unit-bearing array, and a nested ``artifacts`` subgroup whose attrs hold the
|
|
62
|
+
serialized :class:`OutputArtifact` (local path for local runs, S3
|
|
63
|
+
``remote_uris`` for remote docker providers).
|
|
64
|
+
"""
|
|
65
|
+
group.attrs["status"] = getattr(obj, "status", "")
|
|
66
|
+
group.attrs["engine"] = getattr(obj, "engine", "")
|
|
67
|
+
group.attrs["sim_type"] = getattr(obj, "sim_type", "")
|
|
68
|
+
|
|
69
|
+
for f in getattr(obj, "fields", []):
|
|
70
|
+
val = _ensure_array(f.quantity.value)
|
|
71
|
+
arr = group.create_array(f.name, data=val)
|
|
72
|
+
arr.attrs["unit"] = f.quantity.unit or ""
|
|
73
|
+
if getattr(f.quantity, "std_dev", None) is not None:
|
|
74
|
+
arr.attrs["std_dev"] = f.quantity.std_dev
|
|
75
|
+
if getattr(f, "source", ""):
|
|
76
|
+
arr.attrs["source"] = f.source
|
|
77
|
+
|
|
78
|
+
arts = getattr(obj, "artifacts", [])
|
|
79
|
+
if arts:
|
|
80
|
+
ag = group.create_group("artifacts")
|
|
81
|
+
for art in arts:
|
|
82
|
+
ag.attrs[art.name] = json.dumps(art.model_dump())
|
|
83
|
+
|
|
84
|
+
|
|
54
85
|
def _normalize_run_info(run_info: RunInfo | dict) -> dict:
|
|
55
86
|
"""Normalize dataclass or dict run_info input to a plain dict."""
|
|
56
87
|
if isinstance(run_info, RunInfo):
|
|
@@ -212,8 +243,11 @@ def save_results_zarr(results, fname="results.zarr", run_info: RunInfo | dict |
|
|
|
212
243
|
|
|
213
244
|
streams = {}
|
|
214
245
|
solver_units = {}
|
|
246
|
+
engine_outputs = {}
|
|
215
247
|
for name, data in results.items():
|
|
216
|
-
if
|
|
248
|
+
if hasattr(data, "fields"):
|
|
249
|
+
engine_outputs[name] = data
|
|
250
|
+
elif isinstance(data, dict) and "status" in data:
|
|
217
251
|
solver_units[name] = data
|
|
218
252
|
else:
|
|
219
253
|
streams[name] = data
|
|
@@ -226,6 +260,10 @@ def save_results_zarr(results, fname="results.zarr", run_info: RunInfo | dict |
|
|
|
226
260
|
group = root.create_group(name)
|
|
227
261
|
_store_solver_unit(group, data)
|
|
228
262
|
|
|
263
|
+
for name, data in engine_outputs.items():
|
|
264
|
+
group = root.create_group(name)
|
|
265
|
+
_store_engine_output(group, data)
|
|
266
|
+
|
|
229
267
|
if run_info is not None:
|
|
230
268
|
_store_run_info(root, run_info)
|
|
231
269
|
|
|
@@ -270,8 +308,11 @@ def save_results_zarr_s3(results, s3_uri: str, run_info: RunInfo | dict | None =
|
|
|
270
308
|
|
|
271
309
|
streams = {}
|
|
272
310
|
solver_units = {}
|
|
311
|
+
engine_outputs = {}
|
|
273
312
|
for name, data in results.items():
|
|
274
|
-
if
|
|
313
|
+
if hasattr(data, "fields"):
|
|
314
|
+
engine_outputs[name] = data
|
|
315
|
+
elif isinstance(data, dict) and "status" in data:
|
|
275
316
|
solver_units[name] = data
|
|
276
317
|
else:
|
|
277
318
|
streams[name] = data
|
|
@@ -284,6 +325,10 @@ def save_results_zarr_s3(results, s3_uri: str, run_info: RunInfo | dict | None =
|
|
|
284
325
|
group = root.create_group(name)
|
|
285
326
|
_store_solver_unit(group, data)
|
|
286
327
|
|
|
328
|
+
for name, data in engine_outputs.items():
|
|
329
|
+
group = root.create_group(name)
|
|
330
|
+
_store_engine_output(group, data)
|
|
331
|
+
|
|
287
332
|
if run_info is not None:
|
|
288
333
|
_store_run_info(root, run_info)
|
|
289
334
|
|
|
@@ -477,6 +477,10 @@ class EngineOutput(BaseModel):
|
|
|
477
477
|
# status == "failed"). source is always "provider" so a runtime engine error
|
|
478
478
|
# is distinguishable from a flowsheet/setup validation error.
|
|
479
479
|
error: Optional["ProviderRunError"] = None
|
|
480
|
+
# Local directory the run was executed in (provider scratch). Surfaced so
|
|
481
|
+
# the container server can sweep it for artifacts to upload to object
|
|
482
|
+
# storage, and so forensics can trace where outputs were produced.
|
|
483
|
+
run_dir: str = ""
|
|
480
484
|
|
|
481
485
|
# -- convenience accessors -------------------------------------------
|
|
482
486
|
def get_field(self, name: str) -> Optional[OutputField]:
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""S3 storage-options helper.
|
|
2
|
+
|
|
3
|
+
Builds s3fs storage options (credentials, endpoint, region) from the
|
|
4
|
+
container environment for use by :class:`processforge.persistence.artifact_store.ArtifactStore`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def s3_storage_options() -> dict:
|
|
13
|
+
"""Build s3fs storage options from environment variables.
|
|
14
|
+
|
|
15
|
+
Returns an empty dict when no credentials are configured so s3fs falls
|
|
16
|
+
back to its default credential chain (e.g. instance/role-based auth).
|
|
17
|
+
"""
|
|
18
|
+
opts: dict = {}
|
|
19
|
+
key = os.environ.get("S3_ACCESS_KEY")
|
|
20
|
+
secret = os.environ.get("S3_SECRET_KEY")
|
|
21
|
+
if key:
|
|
22
|
+
opts["key"] = key
|
|
23
|
+
if secret:
|
|
24
|
+
opts["secret"] = secret
|
|
25
|
+
client_kwargs = {
|
|
26
|
+
k: v
|
|
27
|
+
for k, v in {
|
|
28
|
+
"endpoint_url": os.environ.get("S3_ENDPOINT_URL"),
|
|
29
|
+
"region_name": os.environ.get("S3_REGION_NAME", "ams3"),
|
|
30
|
+
}.items()
|
|
31
|
+
if v
|
|
32
|
+
}
|
|
33
|
+
if client_kwargs:
|
|
34
|
+
opts["client_kwargs"] = client_kwargs
|
|
35
|
+
return opts
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def validate_s3() -> None:
|
|
39
|
+
"""Fail fast if S3 upload is requested but storage is unreachable.
|
|
40
|
+
|
|
41
|
+
Only acts when ``S3_BUCKET`` is set. Raises ``RuntimeError`` naming any
|
|
42
|
+
missing credentials/env vars when the bucket cannot be reached, so a
|
|
43
|
+
misconfigured container deployment surfaces the problem at startup rather
|
|
44
|
+
than silently producing outputs with no ``remote_uris``.
|
|
45
|
+
"""
|
|
46
|
+
bucket = os.environ.get("S3_BUCKET")
|
|
47
|
+
if not bucket:
|
|
48
|
+
return
|
|
49
|
+
try:
|
|
50
|
+
import s3fs
|
|
51
|
+
|
|
52
|
+
fs = s3fs.S3FileSystem(**s3_storage_options())
|
|
53
|
+
fs.ls(bucket)
|
|
54
|
+
except Exception as exc: # noqa: BLE001
|
|
55
|
+
missing = [
|
|
56
|
+
v
|
|
57
|
+
for v in ("S3_ACCESS_KEY", "S3_SECRET_KEY", "S3_ENDPOINT_URL")
|
|
58
|
+
if not os.environ.get(v)
|
|
59
|
+
]
|
|
60
|
+
hint = f" Missing env vars: {missing}." if missing else ""
|
|
61
|
+
raise RuntimeError(
|
|
62
|
+
f"S3 upload requested (S3_BUCKET='{bucket}') but storage is "
|
|
63
|
+
f"unreachable: {exc}.{hint} Set S3_BUCKET + credentials, or unset "
|
|
64
|
+
f"S3_BUCKET to disable remote upload."
|
|
65
|
+
) from exc
|
|
66
|
+
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: processforge
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.19
|
|
4
4
|
Summary: A Python-based process simulation framework for chemical engineering applications.
|
|
5
5
|
Author-email: Processforge Team <team@processforge.dev>
|
|
6
6
|
License-Expression: BSD-3-Clause
|
|
@@ -114,12 +114,18 @@ uv tool install "processforge[modelica]"
|
|
|
114
114
|
|
|
115
115
|
4. **Look at the output**
|
|
116
116
|
|
|
117
|
-
|
|
118
|
-
- `*_results.zarr` — simulation results store (per-variable arrays, composition flattened)
|
|
119
|
-
- `*_results.zarr.schema.json` — schema file describing streams, variables, dtypes, units, shapes, and run provenance
|
|
120
|
-
- `*.pfstate/` — versioned snapshot store with a `latest` pointer
|
|
117
|
+
`pf apply` (and `pf run`) write a single unified archive under `outputs/`:
|
|
121
118
|
|
|
122
|
-
|
|
119
|
+
- `outputs/<base>.pfarchive/` — the unified store for a flowsheet's solved state and outputs:
|
|
120
|
+
- `snapshots/` — Zarr store of converged state vectors (one group per successful `pf apply`), each with `x`/`x_delta` arrays and config/var-name/metadata attributes, plus a `latest` pointer (powers warm-start, drift detection, and homotopy).
|
|
121
|
+
- `runs/<run_id>.json` — the run manifest: every stream and unit engine output (values, units, dtypes, shapes) and run provenance (backend, version, flowsheet hash).
|
|
122
|
+
- `outputs/streams/<name>.json` — per-stream timeseries from the solve.
|
|
123
|
+
- `artifacts.json` — content-addressed registry of all output artifacts (local + remote URIs).
|
|
124
|
+
- `index.json` — `field_name → occurrences` index for fast cross-run lookups.
|
|
125
|
+
- `latest_run` — plain-text pointer to the most recent run.
|
|
126
|
+
- `outputs/<base>_divergence.json` — written only when both direct and homotopy solves fail on `pf apply`, capturing drifted params, solver stats, and top residual violators.
|
|
127
|
+
|
|
128
|
+
For dynamic flowsheets with Tank units, use `pf run` (not `pf apply`, which is steady-state EO only) to solve with the dynamic engine.
|
|
123
129
|
|
|
124
130
|
## Python API
|
|
125
131
|
|
|
@@ -388,7 +388,8 @@ class TestCmdRun:
|
|
|
388
388
|
return_value=self._containerized_config(flowsheet)), \
|
|
389
389
|
patch.object(run_mod, "check_providers"), \
|
|
390
390
|
patch.object(run_mod, "EOFlowsheet", return_value=mock_fs), \
|
|
391
|
-
patch.object(run_mod, "ProcessStateArchive", MagicMock())
|
|
391
|
+
patch.object(run_mod, "ProcessStateArchive", MagicMock()), \
|
|
392
|
+
patch("processforge.result.save_results_zarr", lambda *a, **k: None):
|
|
392
393
|
run(flowsheet=str(flowsheet), export_images=False)
|
|
393
394
|
|
|
394
395
|
def test_coolprop_flowsheet_no_lifecycle(self, tmp_path):
|
|
@@ -414,7 +415,8 @@ class TestCmdRun:
|
|
|
414
415
|
"_config_path": str(flowsheet)}), \
|
|
415
416
|
patch.object(run_mod, "check_providers"), \
|
|
416
417
|
patch.object(run_mod, "EOFlowsheet", return_value=mock_fs), \
|
|
417
|
-
patch.object(run_mod, "ProcessStateArchive", MagicMock())
|
|
418
|
+
patch.object(run_mod, "ProcessStateArchive", MagicMock()), \
|
|
419
|
+
patch("processforge.result.save_results_zarr", lambda *a, **k: None):
|
|
418
420
|
run(flowsheet=str(flowsheet), export_images=False)
|
|
419
421
|
|
|
420
422
|
def test_no_compose_no_lifecycle(self, tmp_path):
|
|
@@ -437,7 +439,8 @@ class TestCmdRun:
|
|
|
437
439
|
return_value=self._containerized_config(flowsheet)), \
|
|
438
440
|
patch.object(run_mod, "check_providers"), \
|
|
439
441
|
patch.object(run_mod, "EOFlowsheet", return_value=mock_fs), \
|
|
440
|
-
patch.object(run_mod, "ProcessStateArchive", MagicMock())
|
|
442
|
+
patch.object(run_mod, "ProcessStateArchive", MagicMock()), \
|
|
443
|
+
patch("processforge.result.save_results_zarr", lambda *a, **k: None):
|
|
441
444
|
mock_os.path.exists.return_value = False # no compose file
|
|
442
445
|
run(flowsheet=str(flowsheet), export_images=False)
|
|
443
446
|
|
|
@@ -523,7 +526,8 @@ class TestCmdApply:
|
|
|
523
526
|
patch.object(apply_mod, "check_providers"), \
|
|
524
527
|
patch.object(apply_mod, "EOFlowsheet", return_value=mock_fs), \
|
|
525
528
|
patch.object(apply_mod, "save_snapshot", return_value="snap-1"), \
|
|
526
|
-
patch.object(apply_mod, "load_state_manager", return_value=(MagicMock(), None))
|
|
529
|
+
patch.object(apply_mod, "load_state_manager", return_value=(MagicMock(), None)), \
|
|
530
|
+
patch("processforge.result.save_results_zarr", lambda *a, **k: None):
|
|
527
531
|
apply(flowsheet=str(flowsheet))
|
|
528
532
|
|
|
529
533
|
def test_coolprop_flowsheet_no_lifecycle(self, tmp_path):
|
|
@@ -552,7 +556,8 @@ class TestCmdApply:
|
|
|
552
556
|
patch.object(apply_mod, "check_providers"), \
|
|
553
557
|
patch.object(apply_mod, "EOFlowsheet", return_value=mock_fs), \
|
|
554
558
|
patch.object(apply_mod, "save_snapshot", return_value="snap-1"), \
|
|
555
|
-
patch.object(apply_mod, "load_state_manager", return_value=(MagicMock(), None))
|
|
559
|
+
patch.object(apply_mod, "load_state_manager", return_value=(MagicMock(), None)), \
|
|
560
|
+
patch("processforge.result.save_results_zarr", lambda *a, **k: None):
|
|
556
561
|
apply(flowsheet=str(flowsheet))
|
|
557
562
|
|
|
558
563
|
|