processforge 0.3.9__tar.gz → 0.3.10__tar.gz

{processforge-0.3.9/src/processforge.egg-info → processforge-0.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: processforge
-Version: 0.3.9
+Version: 0.3.10
 Summary: A Python-based process simulation framework for chemical engineering applications.
 Author-email: Processforge Team <team@processforge.dev>
 License-Expression: BSD-3-Clause
@@ -33,6 +33,7 @@ Requires-Dist: typer>=0.12
 Provides-Extra: api
 Requires-Dist: fastapi>=0.110; extra == "api"
 Requires-Dist: uvicorn>=0.29; extra == "api"
+Requires-Dist: s3fs>=2024.0; extra == "api"
 Provides-Extra: eo
 Requires-Dist: pyomo>=6.7; extra == "eo"
 Provides-Extra: eo-casadi

{processforge-0.3.9 → processforge-0.3.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "processforge"
-version = "0.3.9"
+version = "0.3.10"
 description = "A Python-based process simulation framework for chemical engineering applications."
 readme = "README.md"
 license = "BSD-3-Clause"
@@ -42,6 +42,7 @@ dependencies = [
 api = [
     "fastapi>=0.110",
     "uvicorn>=0.29",
+    "s3fs>=2024.0",
 ]
 eo = [
     "pyomo>=6.7",

{processforge-0.3.9 → processforge-0.3.10}/src/processforge/api/serve.py

@@ -15,6 +15,7 @@ import os
 import traceback
 
 from fastapi import FastAPI, HTTPException
+from loguru import logger
 
 app = FastAPI(title="Processforge Provider")
 
@@ -76,6 +77,7 @@ def run(body: dict):
     try:
         provider.initialize(provider_config, flowsheet_config)
         result = provider.run_simulation(unit_cfg, body.get("inlet", {}))
+        _maybe_upload_run_outputs(result)
         return result.as_dict() | {"metadata": result.metadata}
     except HTTPException:
         raise
@@ -88,6 +90,26 @@ def run(body: dict):
         provider.teardown()
 
 
+def _maybe_upload_run_outputs(result) -> None:
+    """Upload the provider's run_dir to S3 when configured.
+
+    No-op unless ``S3_BUCKET`` is set in the container environment and the
+    provider recorded a ``run_dir`` in its result metadata. Upload failures are
+    logged and never fail the run.
+    """
+    run_dir = result.metadata.get("run_dir")
+    if not run_dir:
+        return
+    try:
+        from processforge.utils.s3_upload import upload_directory_to_s3
+
+        s3_uri = upload_directory_to_s3(run_dir)
+        if s3_uri:
+            result.metadata["s3_uri"] = s3_uri
+    except Exception as exc:  # noqa: BLE001
+        logger.warning(f"S3 upload of {run_dir} failed: {exc}")
+
+
 if __name__ == "__main__":
     import uvicorn
 

{processforge-0.3.9 → processforge-0.3.10}/src/processforge/cli/run.py

@@ -100,6 +100,10 @@ def run(
     logger.info(f"  Backend      : {display_backend(config, getattr(fs, 'backend', 'dynamic'))}")
     logger.info(f"Results saved to {zarr_path}")
 
+    for name, res in results.items():
+        if isinstance(res, dict) and (s3_uri := res.get("metadata", {}).get("s3_uri")):
+            logger.info(f"  Run outputs (S3): {s3_uri}")
+
     if export_images:
         try:
             plot_results(results, fname=f"{base_name}_results.png")

{processforge-0.3.9 → processforge-0.3.10}/src/processforge/providers/container_client.py

@@ -1,13 +1,13 @@
 """CLI-side HTTP client for containerized providers (OpenMC, FESTIM, …).
 
 A provider that runs in a Docker container does its real work inside the
-container (in ``docker/provider_server.py``).  The CLI side must NOT import the
+container (in ``processforge.api.serve``).  The CLI side must NOT import the
 backend library (OpenMC, FESTIM, …) locally — instead it speaks HTTP to the
 container's ``provider_server.py`` instance.  :class:`ContainerProviderClient`
 implements the :class:`~processforge.providers.base.AbstractProvider` contract
 as a thin HTTP wrapper around that server.
 
-Request/response contract (matches ``docker/provider_server.py``)
+Request/response contract (matches ``processforge.api.serve``)
 ----------------------------------------------------------------
 * ``POST /run`` with a JSON body
   ``{"unit_config", "materials", "inlet", "output_dir", "provider_config"}``.

processforge-0.3.10/src/processforge/utils/s3_upload.py

@@ -0,0 +1,69 @@
+"""S3 upload helper for provider containers.
+
+Uploads a provider's run-output directory (``run_dir``) to S3 so ephemeral
+container-local artifacts (``statepoint.h5``, mesh tallies, ``*.h5m``) survive
+after the container stops. The container opts in by setting ``S3_BUCKET`` in its
+environment; credential env vars reuse the convention from
+:func:`processforge.result.save_results_zarr_s3`.
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+
+
+def s3_storage_options() -> dict:
+    """Build s3fs storage options from environment variables.
+
+    Returns an empty dict when no credentials are configured so s3fs falls
+    back to its default credential chain (e.g. instance/role-based auth).
+    """
+    opts: dict = {}
+    key = os.environ.get("S3_ACCESS_KEY")
+    secret = os.environ.get("S3_SECRET_KEY")
+    if key:
+        opts["key"] = key
+    if secret:
+        opts["secret"] = secret
+    client_kwargs = {
+        k: v
+        for k, v in {
+            "endpoint_url": os.environ.get("S3_ENDPOINT_URL"),
+            "region_name": os.environ.get("S3_REGION_NAME", "ams3"),
+        }.items()
+        if v
+    }
+    if client_kwargs:
+        opts["client_kwargs"] = client_kwargs
+    return opts
+
+
+def upload_directory_to_s3(run_dir: str) -> str | None:
+    """Upload *run_dir* recursively to S3.
+
+    Reads the destination from the container environment:
+
+    * ``S3_BUCKET``  — required; enables the upload. When unset, returns ``None``
+      and the caller keeps its local artifacts.
+    * ``S3_PREFIX``  — optional key prefix (e.g. ``openmc-runs``).
+
+    The object key is ``<S3_PREFIX>/<run_dir_basename>-<utc_stamp>/`` so
+    repeated runs do not clobber each other.
+
+    Returns the uploaded ``s3://`` URI, or ``None`` when disabled.
+    """
+    bucket = os.environ.get("S3_BUCKET")
+    if not bucket:
+        return None
+
+    prefix = os.environ.get("S3_PREFIX", "").strip("/")
+    tag = f"{os.path.basename(run_dir.rstrip('/'))}-{datetime.now(timezone.utc):%Y%m%dT%H%M%SZ}"
+    s3_dir = f"{prefix + '/' if prefix else ''}{tag}"
+
+    import s3fs
+
+    s3fs.S3FileSystem(**s3_storage_options()).put(
+        run_dir, f"s3://{bucket}/{s3_dir}", recursive=True
+    )
+    return f"s3://{bucket}/{s3_dir}"

{processforge-0.3.9 → processforge-0.3.10/src/processforge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: processforge
-Version: 0.3.9
+Version: 0.3.10
 Summary: A Python-based process simulation framework for chemical engineering applications.
 Author-email: Processforge Team <team@processforge.dev>
 License-Expression: BSD-3-Clause
@@ -33,6 +33,7 @@ Requires-Dist: typer>=0.12
 Provides-Extra: api
 Requires-Dist: fastapi>=0.110; extra == "api"
 Requires-Dist: uvicorn>=0.29; extra == "api"
+Requires-Dist: s3fs>=2024.0; extra == "api"
 Provides-Extra: eo
 Requires-Dist: pyomo>=6.7; extra == "eo"
 Provides-Extra: eo-casadi

{processforge-0.3.9 → processforge-0.3.10}/src/processforge.egg-info/SOURCES.txt

@@ -107,6 +107,7 @@ src/processforge/units/valve.py
 src/processforge/utils/__init__.py
 src/processforge/utils/flowsheet_diagram.py
 src/processforge/utils/mermaid_diagram.py
+src/processforge/utils/s3_upload.py
 src/processforge/utils/topology.py
 src/processforge/utils/unit_consistency.py
 src/processforge/utils/validate_flowsheet.py
@@ -115,5 +116,6 @@ tests/test_cli.py
 tests/test_festim_provider.py
 tests/test_init.py
 tests/test_providers.py
+tests/test_s3_upload.py
 tests/test_state.py
 tests/test_types.py

{processforge-0.3.9 → processforge-0.3.10}/src/processforge.egg-info/requires.txt

@@ -13,6 +13,7 @@ typer>=0.12
 [api]
 fastapi>=0.110
 uvicorn>=0.29
+s3fs>=2024.0
 
 [cantera]
 cantera>=3.0

processforge-0.3.10/tests/test_s3_upload.py

@@ -0,0 +1,81 @@
+"""Tests for the S3 run-output upload helper."""
+from __future__ import annotations
+
+import os
+import sys
+from datetime import datetime, timezone
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from processforge.utils import s3_upload
+
+
+@pytest.fixture
+def clean_env(monkeypatch):
+    for var in (
+        "S3_BUCKET",
+        "S3_PREFIX",
+        "S3_ACCESS_KEY",
+        "S3_SECRET_KEY",
+        "S3_ENDPOINT_URL",
+        "S3_REGION_NAME",
+    ):
+        monkeypatch.delenv(var, raising=False)
+    return monkeypatch
+
+
+def test_s3_storage_options_defaults_to_ams3(clean_env):
+    opts = s3_upload.s3_storage_options()
+    assert opts["client_kwargs"]["region_name"] == "ams3"
+    # No creds configured -> not present, s3fs uses default chain.
+    assert "key" not in opts
+    assert "secret" not in opts
+
+
+def test_s3_storage_options_includes_credentials(clean_env):
+    clean_env.setenv("S3_ACCESS_KEY", "ak")
+    clean_env.setenv("S3_SECRET_KEY", "sk")
+    clean_env.setenv("S3_ENDPOINT_URL", "https://s3.example.com")
+    opts = s3_upload.s3_storage_options()
+    assert opts["key"] == "ak"
+    assert opts["secret"] == "sk"
+    assert opts["client_kwargs"]["endpoint_url"] == "https://s3.example.com"
+
+
+def test_upload_directory_to_s3_disabled_without_bucket(clean_env):
+    assert s3_upload.upload_directory_to_s3("/tmp/run") is None
+
+
+def test_upload_directory_to_s3_uploads_recursively(clean_env):
+    clean_env.setenv("S3_BUCKET", "my-bucket")
+    clean_env.setenv("S3_PREFIX", "openmc-runs")
+
+    mock_fs = MagicMock()
+    mock_s3fs = MagicMock()
+    mock_s3fs.S3FileSystem.return_value = mock_fs
+
+    with patch.dict(sys.modules, {"s3fs": mock_s3fs}):
+        uri = s3_upload.upload_directory_to_s3("/tmp/processforge/openmc/msre_run")
+
+    assert uri.startswith("s3://my-bucket/openmc-runs/msre_run-")
+    assert uri.endswith(datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")[:8]) or "-" in uri
+    mock_fs.put.assert_called_once()
+    args, kwargs = mock_fs.put.call_args
+    assert args[0] == "/tmp/processforge/openmc/msre_run"
+    assert args[1].startswith("s3://my-bucket/openmc-runs/msre_run-")
+    assert kwargs.get("recursive") is True
+
+
+def test_upload_directory_to_s3_no_prefix(clean_env):
+    clean_env.setenv("S3_BUCKET", "b")
+
+    mock_fs = MagicMock()
+    mock_s3fs = MagicMock()
+    mock_s3fs.S3FileSystem.return_value = mock_fs
+
+    with patch.dict(sys.modules, {"s3fs": mock_s3fs}):
+        uri = s3_upload.upload_directory_to_s3("/data/festim")
+
+    assert uri == f"s3://b/festim-{datetime.now(timezone.utc):%Y%m%dT%H%M%SZ}"
+    assert mock_fs.put.call_args[1].get("recursive") is True