eosframes 1.1.0__tar.gz → 1.2.0__tar.gz

{eosframes-1.1.0 → eosframes-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eosframes
-Version: 1.1.0
+Version: 1.2.0
 Summary: Ersilia utilities for working with tabular output data
 License: MIT
 License-File: LICENSE
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Scientific/Engineering :: Chemistry
 Requires-Dist: click (>=8.0)
@@ -98,11 +99,41 @@ Run `eosframes --help` (or `eosframes <command> --help`) for inline help.
 
 See [`docs/cli.md`](docs/cli.md) for every flag, example, and refusal condition.
 
+`fit` and `transform` stream in bounded memory — `fit` walks one column at a
+time, `transform` one row-chunk at a time — so they handle files far larger
+than RAM (wide fingerprint frames, tens of GB). Tune `--chunksize` to trade
+memory for throughput.
+
+## Scripts
+
+[`scripts/build_scaler.sh`](scripts/build_scaler.sh) builds and packages a scaler
+for one model in a single step: it pulls the model's precalculations from the
+[isaura](https://github.com/ersilia-os/isaura) store over the Ersilia reference
+library (`data/ersilia_reference_library_v0.csv`), fits an `eosframes` scaler, and
+compresses the transformer into a versioned zip.
+
+```bash
+scripts/build_scaler.sh <model_id> <version>      # e.g. eos4e40 v1
+```
+
+The artifact is written to:
+
+```
+output/ersilia_reference_library_v0/<model_id>/<version>/scaler-<eosframes-major>.zip
+```
+
+(e.g. `output/ersilia_reference_library_v0/eos4e40/v1/scaler-1.zip`), containing a
+single `<model_id>_<version>_transformer.json`. The bucket defaults to
+`isaura-public`; override it with the `PROJECT_NAME` environment variable.
+
+**Prerequisites:** `isaura` and `eosframes` on `PATH`, and a running local isaura
+MinIO engine (`isaura engine --start`).
+
 ## Documentation
 
 - [`docs/cli.md`](docs/cli.md) — every CLI command, all flags, examples, and error patterns.
 - [`docs/nomenclature.md`](docs/nomenclature.md) — every recognised filename / directory pattern, the strict/lenient contract, and the two stack modes.
-- [`docs/scaling.md`](docs/scaling.md) — the type-aware robust scaler: column kinds, how each is picked, and quantization / imputation.
+- [`docs/scaling.md`](docs/scaling.md) — the type-aware robust scaler: column kinds, how each is picked, quantization / imputation, and the bounded-memory streaming fit/transform for large files.
 
 ## About the Ersilia Open Source Initiative
 

{eosframes-1.1.0 → eosframes-1.2.0}/README.md

@@ -65,11 +65,41 @@ Run `eosframes --help` (or `eosframes <command> --help`) for inline help.
 
 See [`docs/cli.md`](docs/cli.md) for every flag, example, and refusal condition.
 
+`fit` and `transform` stream in bounded memory — `fit` walks one column at a
+time, `transform` one row-chunk at a time — so they handle files far larger
+than RAM (wide fingerprint frames, tens of GB). Tune `--chunksize` to trade
+memory for throughput.
+
+## Scripts
+
+[`scripts/build_scaler.sh`](scripts/build_scaler.sh) builds and packages a scaler
+for one model in a single step: it pulls the model's precalculations from the
+[isaura](https://github.com/ersilia-os/isaura) store over the Ersilia reference
+library (`data/ersilia_reference_library_v0.csv`), fits an `eosframes` scaler, and
+compresses the transformer into a versioned zip.
+
+```bash
+scripts/build_scaler.sh <model_id> <version>      # e.g. eos4e40 v1
+```
+
+The artifact is written to:
+
+```
+output/ersilia_reference_library_v0/<model_id>/<version>/scaler-<eosframes-major>.zip
+```
+
+(e.g. `output/ersilia_reference_library_v0/eos4e40/v1/scaler-1.zip`), containing a
+single `<model_id>_<version>_transformer.json`. The bucket defaults to
+`isaura-public`; override it with the `PROJECT_NAME` environment variable.
+
+**Prerequisites:** `isaura` and `eosframes` on `PATH`, and a running local isaura
+MinIO engine (`isaura engine --start`).
+
 ## Documentation
 
 - [`docs/cli.md`](docs/cli.md) — every CLI command, all flags, examples, and error patterns.
 - [`docs/nomenclature.md`](docs/nomenclature.md) — every recognised filename / directory pattern, the strict/lenient contract, and the two stack modes.
-- [`docs/scaling.md`](docs/scaling.md) — the type-aware robust scaler: column kinds, how each is picked, and quantization / imputation.
+- [`docs/scaling.md`](docs/scaling.md) — the type-aware robust scaler: column kinds, how each is picked, quantization / imputation, and the bounded-memory streaming fit/transform for large files.
 
 ## About the Ersilia Open Source Initiative
 

{eosframes-1.1.0 → eosframes-1.2.0}/pyproject.toml

@@ -9,7 +9,7 @@ style = "pep440"
 
 [tool.poetry]
 name = "eosframes"
-version = "1.1.0"
+version = "1.2.0"
 description = "Ersilia utilities for working with tabular output data"
 authors = ["Ersilia Open Source Initiative <hello@ersilia.io>"]
 license = "MIT"

{eosframes-1.1.0 → eosframes-1.2.0}/src/eosframes/cli.py

@@ -186,9 +186,7 @@ def _render_sidecar(
         click.echo(resolved_out)
 
 
-def _fetch_or_clickfail(
-    fetch: Callable, *args, **kwargs
-):
+def _fetch_or_clickfail(fetch: Callable, *args, **kwargs):
     """Call a hub fetcher and convert ``EosframesError`` to ``ClickException``."""
     try:
         return fetch(*args, **kwargs)
@@ -664,7 +662,27 @@ def columns(input_file: str, output: str) -> None:
         "transform time."
     ),
 )
-def fit(input_file: str, scaler: str, output: str, quantize: bool, impute: bool) -> None:
+@click.option(
+    "--chunksize",
+    "chunksize",
+    type=int,
+    default=_scale._DEFAULT_CHUNKSIZE,
+    show_default=True,
+    help=(
+        "Rows per streamed chunk. The fit never holds the whole file — it "
+        "walks one column at a time (CSV inputs are first staged to a "
+        "temporary columnar H5 in chunks of this size). Lower it for very "
+        "wide frames, raise it for narrow ones."
+    ),
+)
+def fit(
+    input_file: str,
+    scaler: str,
+    output: str,
+    quantize: bool,
+    impute: bool,
+    chunksize: int,
+) -> None:
     """Fit a type-aware robust scaler on INPUT_FILE and save parameters to SCALER.
 
     Each numeric feature column is auto-classified (constant, binary, count,
@@ -700,6 +718,7 @@ def fit(input_file: str, scaler: str, output: str, quantize: bool, impute: bool)
             output_path=output,
             output_dtype=output_dtype,
             impute=impute,
+            chunksize=chunksize,
         )
     except EosframesError as e:
         raise _err(e) from e
@@ -744,7 +763,26 @@ def fit(input_file: str, scaler: str, output: str, quantize: bool, impute: bool)
         "no -128 sentinels)."
     ),
 )
-def transform(input_file: str, scaler: str, output: str, quantize: bool, impute: bool) -> None:
+@click.option(
+    "--chunksize",
+    "chunksize",
+    type=int,
+    default=_scale._DEFAULT_CHUNKSIZE,
+    show_default=True,
+    help=(
+        "Rows per streamed chunk. The input is read and the output written "
+        "one chunk at a time, so memory stays bounded regardless of file "
+        "size. Lower it for very wide frames, raise it for narrow ones."
+    ),
+)
+def transform(
+    input_file: str,
+    scaler: str,
+    output: str,
+    quantize: bool,
+    impute: bool,
+    chunksize: int,
+) -> None:
     """Apply a saved scaler to INPUT_FILE and write scaled data to OUTPUT.
 
     Loads the scaler parameters from SCALER and applies them to the numeric
@@ -757,7 +795,12 @@ def transform(input_file: str, scaler: str, output: str, quantize: bool, impute:
     output_dtype = "int8" if quantize else "float32"
     try:
         out = _scale.transform_file(
-            input_file, scaler, output, output_dtype=output_dtype, impute=impute
+            input_file,
+            scaler,
+            output,
+            output_dtype=output_dtype,
+            impute=impute,
+            chunksize=chunksize,
         )
     except EosframesError as e:
         raise _err(e) from e

{eosframes-1.1.0 → eosframes-1.2.0}/src/eosframes/ops.py

@@ -76,9 +76,7 @@ def _require_no_overwrite(path: str, *, kind: str = "file") -> None:
             f"Output folder '{path}' already exists. "
             "Remove it or choose a different name."
         )
-    raise EosframesError(
-        f"Output file '{path}' already exists. Remove it first."
-    )
+    raise EosframesError(f"Output file '{path}' already exists. Remove it first.")
 
 
 def _compute_summary_stats(df: pd.DataFrame) -> List[Dict]:

{eosframes-1.1.0 → eosframes-1.2.0}/src/eosframes/scale.py

@@ -173,8 +173,9 @@ hand-maintained schema number.
 
 import json
 import os
+import tempfile
 from datetime import datetime
-from typing import Callable, Dict, Optional, Tuple
+from typing import Callable, Dict, Iterator, Optional, Tuple
 
 import h5py
 import numpy as np
@@ -189,6 +190,7 @@ from .naming import (
     parse_name,
     parse_transformer_name,
 )
+from .utils import progress
 
 _META_COLS = {"key", "input"}
 
@@ -197,6 +199,31 @@ _METHOD_NAME = "robust_typed"
 _VALID_OUTPUT_DTYPES = ("float32", "int8")
 _DEFAULT_OUTPUT_DTYPE = "float32"
 
+# Rows per chunk for the streaming file-level fit/transform paths. Big
+# enough to amortise per-chunk overhead, small enough that one chunk of a
+# wide frame stays well under a GB (e.g. 50k rows × 3k cols × 4B ≈ 0.6 GB).
+_DEFAULT_CHUNKSIZE = 50_000
+
+# Row cadence for the CSV→H5 staging milestone log (independent of chunksize,
+# so the log reads the same whatever chunk size the caller picks).
+_STAGE_LOG_ROWS = 1_000_000
+
+# Columns pulled per HDF5 read during the fit. Reading the values dataset one
+# column at a time forces a re-read of every storage chunk the column touches,
+# so we read a batch of columns at once and then fit each column from the
+# in-memory block. 100 cols × n_rows × 4 B stays bounded (~0.5 GB at 1.3 M
+# rows) while cutting the number of HDF5 reads ~100×.
+_FIT_COLUMN_BATCH = 100
+
+# Target bytes per HDF5 storage chunk for the staged values dataset. Chunks are
+# tiled ``(row_tile, _FIT_COLUMN_BATCH)`` so the fit's column-batch reads align
+# to chunk boundaries — each chunk is read exactly once across the whole fit.
+_STAGE_CHUNK_BYTES = 4 * 1024 * 1024
+
+# Target number of progress log lines for the column-fit and row-chunk
+# transform loops when running off-TTY (one line per ~5% of the work).
+_PROGRESS_LOG_STEPS = 20
+
 _INT8_NAN_SENTINEL = -128
 _INT8_MAX_VAL = 127
 
@@ -309,9 +336,7 @@ _COUNT_SHIFTED_EXTENT_RATIO_MAX = 2.0
 _PIECEWISE_BLEND_FRACTION = 0.5
 
 
-def _tanh_tail(
-    mag: np.ndarray, body_extent: float, body_target: float
-) -> np.ndarray:
+def _tanh_tail(mag: np.ndarray, body_extent: float, body_target: float) -> np.ndarray:
     """Asymptotic tail for ``|x - center| > body_extent``.
 
     Returns ``y = body_target + (1 − body_target)·tanh(c·u)`` for
@@ -596,12 +621,10 @@ def _fit_continuous(series: pd.Series) -> dict:
     tail_asymmetry_right = right_span / max(left_span, _eps)
     tail_asymmetry_left = left_span / max(right_span, _eps)
     is_right_skew = (
-        bowley > _BOWLEY_THRESHOLD
-        and tail_asymmetry_right > _TAIL_ASYMMETRY_THRESHOLD
+        bowley > _BOWLEY_THRESHOLD and tail_asymmetry_right > _TAIL_ASYMMETRY_THRESHOLD
     )
     is_left_skew = (
-        bowley < -_BOWLEY_THRESHOLD
-        and tail_asymmetry_left > _TAIL_ASYMMETRY_THRESHOLD
+        bowley < -_BOWLEY_THRESHOLD and tail_asymmetry_left > _TAIL_ASYMMETRY_THRESHOLD
     )
 
     if is_right_skew:
@@ -756,6 +779,9 @@ def _fit_count(series: pd.Series) -> dict:
         # Flag near-degenerate sparse counts where most rows are 0 and
         # the scaler can only produce a handful of distinct values. Goes
         # into fit_notes — advisory only, never read by the transform.
+        # No per-column log here: sparse fingerprints (e.g. Morgan counts)
+        # flag hundreds of columns and would flood the log. fit() emits a
+        # single aggregated summary instead.
         scaled = np.clip(arr / high_anchor, 0.0, 1.0)
         n_distinct = int(np.unique(scaled).size)
         mode_fraction = float((arr == 0.0).sum()) / float(arr.size)
@@ -768,15 +794,6 @@ def _fit_count(series: pd.Series) -> dict:
                 "mode_fraction": mode_fraction,
                 "n_distinct_output": n_distinct,
             }
-            get_logger().warning(
-                "Count column '%s' is near-degenerate "
-                "(mode_fraction=%.2f, n_distinct_output=%d). "
-                "Output collapses to a handful of values — "
-                "consider dropping it or revisiting upstream featurization.",
-                series.name,
-                mode_fraction,
-                n_distinct,
-            )
         return entry
 
     # Count with a non-zero mode. Linear+clip on each side of the mode,
@@ -889,9 +906,7 @@ def _apply_constant(
     return _to_output_array(out, output_dtype)
 
 
-def _apply_binary(
-    series: pd.Series, transform: dict, output_dtype: str
-) -> np.ndarray:
+def _apply_binary(series: pd.Series, transform: dict, output_dtype: str) -> np.ndarray:
     low = float(transform["low"])
     high = float(transform["high"])
     arr = series.to_numpy(dtype=float)
@@ -1123,9 +1138,7 @@ def _apply_continuous_left(
         else:  # "finite"
             low = float(transform["low_anchor"])
             max_extent = high - low
-            out[tail_mask] = -_quadratic_tail(
-                mag, body_span, body_target, max_extent
-            )
+            out[tail_mask] = -_quadratic_tail(mag, body_span, body_target, max_extent)
 
     # Cubic Hermite blend across the body→middle junction (mirror of
     # the right-skew blend).
@@ -1193,7 +1206,9 @@ def _apply_continuous_centered(
     nan = np.isnan(arr)
     out = np.full(arr.shape, np.nan, dtype=float)
 
-    def _side(mask: np.ndarray, body_extent: float, max_extent: float, sign: float) -> None:
+    def _side(
+        mask: np.ndarray, body_extent: float, max_extent: float, sign: float
+    ) -> None:
         if not mask.any():
             return
         if body_extent <= 0:
@@ -1249,6 +1264,73 @@ def _dispatch_apply(
 # ---------------------------------------------------------------------------
 
 
+def _fit_series(series: pd.Series) -> dict:
+    """Classify one column and fit its type-specific transform.
+
+    Shared by the in-memory :func:`fit` and the streaming
+    :func:`fit_file`. All-NaN columns and columns that slip past
+    classification without a usable scale both fall back to
+    ``kind: "constant"``.
+    """
+    if series.dropna().empty:
+        # All-NaN column: fit as a constant. The dispatch at transform
+        # time maps non-NaN inputs to 0 and propagates NaN.
+        return _fit_constant(series)
+    type_ = _classify_type(series)
+    if type_ == "constant":
+        return _fit_constant(series)
+    if type_ == "binary":
+        return _fit_binary(series)
+    if type_ == "count":
+        return _fit_count(series)
+    try:
+        return _fit_continuous(series)
+    except EosframesError:
+        # Column slipped past _classify_type but has no usable scale.
+        return _fit_constant(series)
+
+
+def _log_fit_summary(columns: dict, logger) -> None:
+    """Emit the per-fit kind breakdown and near-degenerate advisory.
+
+    Shared by :func:`fit` and :func:`fit_file` so the streaming and
+    in-memory paths log identically.
+    """
+    kind_counts: dict = {}
+    for entry in columns.values():
+        kind = entry["transform"]["kind"]
+        kind_counts[kind] = kind_counts.get(kind, 0) + 1
+
+    kind_breakdown = ", ".join(
+        f"{kind}={count}" for kind, count in sorted(kind_counts.items())
+    )
+    n = len(columns)
+    logger.info("Fitted %d / %d numeric columns (%s)", n, n, kind_breakdown)
+
+    # Single aggregated advisory for near-degenerate count columns. The
+    # per-column flag lives in each entry's fit_notes; here we just
+    # summarise how many tripped it so sparse fingerprints don't flood
+    # the log with one warning per bit.
+    degenerate_cols = [
+        col
+        for col, entry in columns.items()
+        if entry.get("fit_notes", {}).get("degenerate")
+    ]
+    if degenerate_cols:
+        preview = ", ".join(str(c) for c in degenerate_cols[:5])
+        if len(degenerate_cols) > 5:
+            preview += ", …"
+        logger.warning(
+            "%d of %d count columns are near-degenerate (e.g. %s): output "
+            "collapses to a handful of values. Common for sparse "
+            "fingerprints; see each column's fit_notes for details. "
+            "Consider dropping them or revisiting upstream featurization.",
+            len(degenerate_cols),
+            n,
+            preview,
+        )
+
+
 def fit(df: pd.DataFrame) -> dict:
     """Fit a type-aware robust scaler on the numeric feature columns.
 
@@ -1294,44 +1376,10 @@ def fit(df: pd.DataFrame) -> dict:
         raise EosframesError("No numeric feature columns found to fit the scaler.")
 
     columns: dict = {}
-    kind_counts: dict = {}
-
-    for col in numeric_cols:
-        series = df[col]
-
-        if series.dropna().empty:
-            # All-NaN column: fit as a constant. The dispatch at
-            # transform time maps non-NaN inputs to 0 and propagates NaN.
-            entry = _fit_constant(series)
-        else:
-            type_ = _classify_type(series)
-            if type_ == "constant":
-                entry = _fit_constant(series)
-            elif type_ == "binary":
-                entry = _fit_binary(series)
-            elif type_ == "count":
-                entry = _fit_count(series)
-            else:
-                try:
-                    entry = _fit_continuous(series)
-                except EosframesError:
-                    # Column slipped past _classify_type but has no usable scale.
-                    entry = _fit_constant(series)
-
-        columns[col] = entry
-        kind = entry["transform"]["kind"]
-        kind_counts[kind] = kind_counts.get(kind, 0) + 1
-
-    kind_breakdown = ", ".join(
-        f"{kind}={count}" for kind, count in sorted(kind_counts.items())
-    )
-    logger.info(
-        "Fitted %d / %d numeric columns (%s)",
-        len(numeric_cols),
-        len(numeric_cols),
-        kind_breakdown,
-    )
+    for col in progress(numeric_cols, desc="Fitting columns"):
+        columns[col] = _fit_series(df[col])
 
+    _log_fit_summary(columns, logger)
     return {"method": _METHOD_NAME, "columns": columns}
 
 
@@ -1381,10 +1429,15 @@ def transform(
         On column mismatch, invalid ``output_dtype``, or unknown
         column type in *params*.
     """
+    _validate_transform(df, params, output_dtype)
+    return _apply_transform(df, params, output_dtype, impute, show_progress=True)
+
+
+def _validate_transform(df: pd.DataFrame, params: dict, output_dtype: str) -> None:
+    """Check output dtype and exact feature-column match before applying."""
     if output_dtype not in _VALID_OUTPUT_DTYPES:
         raise EosframesError(
-            f"Unknown output_dtype '{output_dtype}'. "
-            f"Supported: {_VALID_OUTPUT_DTYPES}"
+            f"Unknown output_dtype '{output_dtype}'. Supported: {_VALID_OUTPUT_DTYPES}"
         )
 
     expected_feature_cols = list(params["columns"].keys())
@@ -1396,9 +1449,29 @@ def transform(
             f"but transformer was fitted on {expected_feature_cols}."
         )
 
+
+def _apply_transform(
+    df: pd.DataFrame,
+    params: dict,
+    output_dtype: str,
+    impute: bool,
+    show_progress: bool,
+) -> pd.DataFrame:
+    """Apply the fitted per-column transforms to *df* (already validated).
+
+    *show_progress* draws the per-column bar — on for the single-shot
+    in-memory :func:`transform`, off for the streaming path where the bar
+    would otherwise redraw once per row-chunk.
+    """
     columns = params["columns"]
+    expected_feature_cols = list(columns.keys())
     result = df.copy()
-    for col in expected_feature_cols:
+    iterator = (
+        progress(expected_feature_cols, desc="Transforming columns")
+        if show_progress
+        else expected_feature_cols
+    )
+    for col in iterator:
         entry = columns[col]
         series = df[col]
         if impute:
@@ -1418,32 +1491,260 @@ def _values_dtype_for(output_dtype: str) -> np.dtype:
     return np.float32
 
 
-def _write_df(
-    df: pd.DataFrame, output_path: str, values_dtype: np.dtype = np.float32
-) -> None:
-    """Write a DataFrame to CSV or H5, bypassing the naming convention check.
+# ---------------------------------------------------------------------------
+# Streaming primitives — bounded-memory fit/transform for very wide or very
+# tall files. The whole matrix is never resident: fit walks one column at a
+# time (cheap on the columnar H5 layout), transform walks row-chunks.
+# ---------------------------------------------------------------------------
+
+
+def _h5_feature_names(h5_path: str) -> list:
+    with h5py.File(h5_path, "r") as f:
+        return [x.decode("utf-8") for x in f["features"][:]]
+
+
+def _h5_nrows(h5_path: str) -> int:
+    with h5py.File(h5_path, "r") as f:
+        return int(f["values"].shape[0])
+
+
+def _iter_h5_columns(
+    h5_path: str, batch: int = _FIT_COLUMN_BATCH
+) -> Iterator[Tuple[str, np.ndarray]]:
+    """Yield ``(feature_name, full_column_array)`` for every feature column.
+
+    Columns are pulled from disk a *batch* at a time — one ``values[:,
+    start:stop]`` hyperslab — then handed out one at a time from that
+    in-memory block. This keeps peak memory bounded (``batch`` columns, not
+    the whole matrix) while reading each underlying HDF5 storage chunk once,
+    instead of re-reading it for every column it contains.
+    """
+    with h5py.File(h5_path, "r") as f:
+        features = [x.decode("utf-8") for x in f["features"][:]]
+        values = f["values"]
+        n_features = len(features)
+        for start in range(0, n_features, batch):
+            block = values[:, start : start + batch]  # (n_rows, ≤batch)
+            for k in range(block.shape[1]):
+                yield features[start + k], block[:, k]
+
+
+def _stage_csv_to_h5(csv_path: str, h5_path: str, chunksize: int) -> Tuple[list, int]:
+    """Stream a CSV into a columnar temp H5 in one bounded-memory pass.
+
+    Reads the CSV in row-chunks and appends each chunk's numeric feature
+    columns to a resizable ``values`` dataset (``float32`` — matches the
+    pipeline's default output precision). Returns ``(feature_cols, n_rows)``.
+    The resulting H5 is then cheaply column-sliceable for the fit.
+    """
+    logger = get_logger()
+    feature_cols: Optional[list] = None
+    total = 0
+    next_log = _STAGE_LOG_ROWS
+    with h5py.File(h5_path, "w") as f:
+        values_ds = None
+        for chunk in pd.read_csv(csv_path, chunksize=chunksize):
+            if feature_cols is None:
+                feat = [c for c in chunk.columns if c not in _META_COLS]
+                feature_cols = [
+                    c for c in feat if pd.api.types.is_numeric_dtype(chunk[c])
+                ]
+                n_feat = len(feature_cols)
+                # Tile chunks to the fit's column-batch width so each storage
+                # chunk is read exactly once during the (batched) column fit.
+                if n_feat > 0:
+                    col_tile = min(n_feat, _FIT_COLUMN_BATCH)
+                    row_tile = max(1, _STAGE_CHUNK_BYTES // (col_tile * 4))
+                    chunk_shape: object = (row_tile, col_tile)
+                else:
+                    chunk_shape = True
+                values_ds = f.create_dataset(
+                    "values",
+                    shape=(0, n_feat),
+                    maxshape=(None, n_feat),
+                    dtype=np.float32,
+                    chunks=chunk_shape,
+                )
+                logger.info("Staging: detected %d numeric feature columns", n_feat)
+            arr = chunk[feature_cols].to_numpy(dtype=np.float32)
+            n = arr.shape[0]
+            values_ds.resize(total + n, axis=0)
+            values_ds[total : total + n, :] = arr
+            total += n
+            if total >= next_log:
+                logger.info("Staging: %s rows written to temp H5", f"{total:,}")
+                next_log += _STAGE_LOG_ROWS
+        dt = h5py.string_dtype(encoding="utf-8")
+        f.create_dataset("features", data=feature_cols or [], dtype=dt)
+    logger.info(
+        "Staging complete: %s rows × %d columns → %s",
+        f"{total:,}",
+        len(feature_cols or []),
+        os.path.basename(h5_path),
+    )
+    return feature_cols or [], total
+
+
+def _iter_row_chunks(path: str, chunksize: int) -> Iterator[pd.DataFrame]:
+    """Yield row-chunk DataFrames (``key``/``input`` + feature cols) from a file.
 
-    *values_dtype* controls the H5 ``values`` dataset dtype; CSV ignores it.
+    CSV is streamed natively by pandas; H5 is sliced ``[start:end, :]`` so
+    only ``chunksize`` rows are resident at a time. The yielded frames have
+    the same column layout the in-memory :func:`transform` expects.
     """
-    ext = os.path.splitext(output_path)[1].lower()
+    ext = os.path.splitext(path)[1].lower()
     if ext == ".csv":
-        df.to_csv(output_path, index=False)
+        yield from pd.read_csv(path, chunksize=chunksize)
     elif ext == ".h5":
-        feat_cols = [c for c in df.columns if c not in _META_COLS]
-        with h5py.File(output_path, "w") as f:
-            dt = h5py.string_dtype(encoding="utf-8")
-            if "key" in df.columns:
-                f.create_dataset("key", data=df["key"].astype(str).tolist(), dtype=dt)
-            if "input" in df.columns:
-                f.create_dataset(
-                    "input", data=df["input"].astype(str).tolist(), dtype=dt
-                )
-            f.create_dataset("features", data=feat_cols, dtype=dt)
-            f.create_dataset(
-                "values", data=df[feat_cols].values, dtype=values_dtype
-            )
+        with h5py.File(path, "r") as f:
+            features = [x.decode("utf-8") for x in f["features"][:]]
+            values = f["values"]
+            n_rows = values.shape[0]
+            has_key = "key" in f
+            for start in range(0, n_rows, chunksize):
+                end = min(start + chunksize, n_rows)
+                meta = {}
+                if has_key:
+                    meta["key"] = [x.decode("utf-8") for x in f["key"][start:end]]
+                meta["input"] = [x.decode("utf-8") for x in f["input"][start:end]]
+                block = pd.DataFrame(values[start:end, :], columns=features)
+                yield pd.concat([pd.DataFrame(meta), block], axis=1)
     else:
-        raise EosframesError(f"Unsupported output format '{ext}'. Expected .csv or .h5")
+        raise EosframesError(f"Unsupported input format '{ext}'. Expected .csv or .h5")
+
+
+class _StreamWriter:
+    """Incremental CSV/H5 writer — appends row-chunks without holding the file.
+
+    CSV: header on the first chunk, append thereafter. H5: resizable
+    ``values`` / ``input`` / ``key`` datasets created from the first chunk's
+    schema, then extended per chunk. Bypasses the naming-convention check
+    (the caller has already validated the output path).
+    """
+
+    def __init__(self, path: str, values_dtype: np.dtype):
+        self.path = path
+        self.ext = os.path.splitext(path)[1].lower()
+        self.values_dtype = values_dtype
+        if self.ext not in (".csv", ".h5"):
+            raise EosframesError(
+                f"Unsupported output format '{self.ext}'. Expected .csv or .h5"
+            )
+        self._csv_started = False
+        self._h5 = None
+        self._feature_cols: Optional[list] = None
+        self._has_key = False
+        self._n = 0
+
+    def write(self, df: pd.DataFrame) -> None:
+        if self.ext == ".csv":
+            df.to_csv(
+                self.path,
+                mode="w" if not self._csv_started else "a",
+                header=not self._csv_started,
+                index=False,
+            )
+            self._csv_started = True
+        else:
+            if self._h5 is None:
+                self._init_h5(df)
+            self._append_h5(df)
+
+    def _init_h5(self, df: pd.DataFrame) -> None:
+        self._feature_cols = [c for c in df.columns if c not in _META_COLS]
+        self._has_key = "key" in df.columns
+        self._h5 = h5py.File(self.path, "w")
+        dt = h5py.string_dtype(encoding="utf-8")
+        self._h5.create_dataset("features", data=self._feature_cols, dtype=dt)
+        self._values = self._h5.create_dataset(
+            "values",
+            shape=(0, len(self._feature_cols)),
+            maxshape=(None, len(self._feature_cols)),
+            dtype=self.values_dtype,
+            chunks=True,
+        )
+        self._input = self._h5.create_dataset(
+            "input", shape=(0,), maxshape=(None,), dtype=dt
+        )
+        if self._has_key:
+            self._key = self._h5.create_dataset(
+                "key", shape=(0,), maxshape=(None,), dtype=dt
+            )
+
+    def _append_h5(self, df: pd.DataFrame) -> None:
+        n = len(df)
+        new = self._n + n
+        self._values.resize(new, axis=0)
+        self._values[self._n : new, :] = df[self._feature_cols].to_numpy(
+            dtype=self.values_dtype
+        )
+        self._input.resize(new, axis=0)
+        self._input[self._n : new] = df["input"].astype(str).tolist()
+        if self._has_key:
+            self._key.resize(new, axis=0)
+            self._key[self._n : new] = df["key"].astype(str).tolist()
+        self._n = new
+
+    def close(self) -> None:
+        if self._h5 is not None:
+            self._h5.close()
+            self._h5 = None
+
+
+def _streaming_transform(
+    input_path: str,
+    transformer: dict,
+    output_path: str,
+    output_dtype: str,
+    impute: bool,
+    chunksize: int,
+) -> int:
+    """Row-chunked transform: read a chunk, scale it, write it, repeat.
+
+    Reuses the in-memory per-column dispatch on each small chunk (with its
+    per-column bar suppressed), so peak memory is one chunk in + one chunk
+    out — independent of file size. Returns the number of rows written.
+    """
+    logger = get_logger()
+    in_ext = os.path.splitext(input_path)[1].lower()
+    # Row count is free for H5 (so we get a true % bar / steady log cadence);
+    # for CSV it is unknown without a full scan, so we log every fixed number
+    # of chunks instead.
+    total_chunks = None
+    if in_ext == ".h5":
+        total_chunks = (_h5_nrows(input_path) + chunksize - 1) // chunksize
+        logger.info(
+            "Transforming %s rows in %d chunks of %d",
+            f"{_h5_nrows(input_path):,}",
+            total_chunks,
+            chunksize,
+        )
+    log_every = (
+        max(1, total_chunks // _PROGRESS_LOG_STEPS)
+        if total_chunks
+        else _PROGRESS_LOG_STEPS
+    )
+
+    writer = _StreamWriter(output_path, _values_dtype_for(output_dtype))
+    chunks = progress(
+        _iter_row_chunks(input_path, chunksize),
+        total=total_chunks,
+        desc="Transforming chunks",
+        log_every=log_every,
+    )
+
+    total_rows = 0
+    try:
+        for chunk in chunks:
+            _validate_transform(chunk, transformer, output_dtype)
+            scaled = _apply_transform(
+                chunk, transformer, output_dtype, impute, show_progress=False
+            )
+            writer.write(scaled)
+            total_rows += len(scaled)
+    finally:
+        writer.close()
+    return total_rows
 
 
 def fit_file(
@@ -1452,15 +1753,27 @@ def fit_file(
     output_path: Optional[str] = None,
     output_dtype: str = _DEFAULT_OUTPUT_DTYPE,
     impute: bool = False,
+    chunksize: int = _DEFAULT_CHUNKSIZE,
 ) -> str:
     """Fit a scaler on an Ersilia output file and save the parameters.
 
+    **Bounded memory.** The full matrix is never resident. The fit reads a
+    batch of feature columns at a time (``_FIT_COLUMN_BATCH``) and fits each
+    column from that in-memory block, so peak memory is the batch
+    (~``batch × n_rows`` floats) — a few hundred MB — regardless of how wide
+    the file is. H5 inputs are sliced directly; CSV inputs are first
+    streamed — in
+    row-chunks of *chunksize* — into a temporary columnar H5 (``float32``),
+    which is then column-sliced for the fit and removed afterwards; this is
+    the one unavoidable full read of the CSV, done without loading it whole.
+
     The scaler JSON written to *scaler_path* is dtype-agnostic — see
     :func:`fit` for the parameter set. When *output_path* is provided
     the scaled data is also written immediately (fit-then-transform in
-    one call), and *output_dtype* selects the dtype of that inline
-    output. The dtype is **not** recorded in the scaler JSON; later
-    calls to :func:`transform_file` choose the dtype independently.
+    one call, streamed row-chunk by row-chunk), and *output_dtype* selects
+    the dtype of that inline output. The dtype is **not** recorded in the
+    scaler JSON; later calls to :func:`transform_file` choose the dtype
+    independently.
 
     The scaler filename's encoded model ID and version must match the
     input file's. The transformer JSON records ``eosframes_version``
@@ -1487,6 +1800,10 @@ def fit_file(
         Only used for the inline transform when *output_path* is
         given. ``"int8"`` quantizes the scaled values into
         ``[-127, 127]`` with sentinel ``-128`` for missing.
+    chunksize : int, default ``50_000``
+        Rows per chunk for the CSV→H5 staging pass and the inline
+        transform. Bounds peak memory; tune down for extremely wide
+        frames, up for narrow ones.
 
     Returns
     -------
@@ -1540,30 +1857,71 @@ def fit_file(
             f"Output file '{output_path}' already exists. Remove it first."
         )
 
-    from .ops import _read_file
-
-    df = _read_file(input_path)
-
     if output_dtype not in _VALID_OUTPUT_DTYPES:
         raise EosframesError(
-            f"Unknown output_dtype '{output_dtype}'. "
-            f"Supported: {_VALID_OUTPUT_DTYPES}"
+            f"Unknown output_dtype '{output_dtype}'. Supported: {_VALID_OUTPUT_DTYPES}"
         )
 
+    in_ext = os.path.splitext(input_path)[1].lower()
     mode_label = "fit + transform" if output_path is not None else "fit only"
-    logger.info(
-        "%s — reading %d rows from %s", mode_label, len(df), input_path
-    )
-    fitted = fit(df)
+
+    # Resolve a column-sliceable H5 to fit from. H5 inputs are used as-is;
+    # CSV inputs are streamed into a temporary columnar H5 first.
+    staged_tmp: Optional[str] = None
+    try:
+        if in_ext == ".h5":
+            fit_source = input_path
+            feature_cols = _h5_feature_names(input_path)
+            n_rows = _h5_nrows(input_path)
+        elif in_ext == ".csv":
+            fd, staged_tmp = tempfile.mkstemp(
+                suffix=".h5", dir=os.path.dirname(os.path.abspath(input_path))
+            )
+            os.close(fd)
+            logger.info(
+                "%s — staging CSV → temporary columnar H5 for column-wise fit "
+                "(chunksize=%d)",
+                mode_label,
+                chunksize,
+            )
+            feature_cols, n_rows = _stage_csv_to_h5(input_path, staged_tmp, chunksize)
+            fit_source = staged_tmp
+        else:
+            raise EosframesError(
+                f"Unsupported input format '{in_ext}'. Expected .csv or .h5"
+            )
+
+        if not feature_cols:
+            raise EosframesError("No numeric feature columns found to fit the scaler.")
+
+        logger.info(
+            "%s — fitting %d columns over %s rows (batches of %d)",
+            mode_label,
+            len(feature_cols),
+            f"{n_rows:,}",
+            _FIT_COLUMN_BATCH,
+        )
+        columns: dict = {}
+        for name, arr in progress(
+            _iter_h5_columns(fit_source),
+            total=len(feature_cols),
+            desc="Fitting columns",
+            log_every=max(1, len(feature_cols) // _PROGRESS_LOG_STEPS),
+        ):
+            columns[name] = _fit_series(pd.Series(arr))
+        _log_fit_summary(columns, logger)
+    finally:
+        if staged_tmp is not None and os.path.exists(staged_tmp):
+            os.remove(staged_tmp)
 
     transformer = {
         "eosframes_version": _PACKAGE_VERSION,
-        "method": fitted["method"],
+        "method": _METHOD_NAME,
         "model_id": parsed["model_id"],
         "model_version": parsed["version"],
         "fitted_at": datetime.now().isoformat(timespec="seconds"),
-        "n_rows": len(df),
-        "columns": fitted["columns"],
+        "n_rows": n_rows,
+        "columns": columns,
     }
     with open(scaler_path, "w") as fh:
         json.dump(transformer, fh, indent=2)
@@ -1575,15 +1933,15 @@ def fit_file(
             output_dtype,
             impute,
         )
-        scaled_df = transform(
-            df, transformer, output_dtype=output_dtype, impute=impute
-        )
-        _write_df(
-            scaled_df,
+        written = _streaming_transform(
+            input_path,
+            transformer,
             output_path,
-            values_dtype=_values_dtype_for(output_dtype),
+            output_dtype,
+            impute,
+            chunksize,
         )
-        logger.info("Scaled output written to %s", output_path)
+        logger.info("Scaled output written to %s (%d rows)", output_path, written)
 
     return scaler_path
 
@@ -1594,9 +1952,15 @@ def transform_file(
     output_path: str,
     output_dtype: str = _DEFAULT_OUTPUT_DTYPE,
     impute: bool = False,
+    chunksize: int = _DEFAULT_CHUNKSIZE,
 ) -> str:
     """Apply a saved scaler to an Ersilia output file.
 
+    **Bounded memory.** The input is read and the output written one
+    row-chunk of *chunksize* at a time, so peak memory is a single chunk
+    in plus a single chunk out — independent of the file's size. Works for
+    CSV and H5 inputs and outputs in any combination.
+
     The scaler's recorded ``eosframes_version`` must exactly match the
     running ``eosframes.__version__``, and ``method`` must still be
     ``"robust_typed"``; any mismatch raises with a clear "re-fit"
@@ -1618,6 +1982,9 @@ def transform_file(
     output_dtype : {"float32", "int8"}, default ``"float32"``
         ``"int8"`` quantizes scaled values to ``[-127, 127]`` with
         sentinel ``-128`` for missing.
+    chunksize : int, default ``50_000``
+        Rows per streamed chunk. Bounds peak memory; tune down for
+        extremely wide frames, up for narrow ones.
 
     Returns
     -------
@@ -1673,8 +2040,7 @@ def transform_file(
 
     if output_dtype not in _VALID_OUTPUT_DTYPES:
         raise EosframesError(
-            f"Unknown output_dtype '{output_dtype}'. "
-            f"Supported: {_VALID_OUTPUT_DTYPES}"
+            f"Unknown output_dtype '{output_dtype}'. Supported: {_VALID_OUTPUT_DTYPES}"
         )
 
     t_model_id = transformer.get("model_id")
@@ -1693,21 +2059,21 @@ def transform_file(
             f"scaler was fitted on '{t_model_version}'."
         )
 
-    from .ops import _read_file
-
-    df = _read_file(input_path)
-
     logger.info(
-        "Applying scaler to %d rows from %s (output_dtype=%s, impute=%s)",
-        len(df),
+        "Applying scaler to %s in row-chunks (chunksize=%d, output_dtype=%s, "
+        "impute=%s)",
         input_path,
+        chunksize,
         output_dtype,
         impute,
     )
-    scaled_df = transform(
-        df, transformer, output_dtype=output_dtype, impute=impute
+    written = _streaming_transform(
+        input_path,
+        transformer,
+        output_path,
+        output_dtype,
+        impute,
+        chunksize,
     )
-
-    _write_df(scaled_df, output_path, values_dtype=_values_dtype_for(output_dtype))
-    logger.info("Scaled output written to %s", output_path)
+    logger.info("Scaled output written to %s (%d rows)", output_path, written)
     return output_path

eosframes-1.2.0/src/eosframes/utils.py

@@ -0,0 +1,126 @@
+"""General-purpose utilities."""
+
+import logging
+import sys
+import time
+from typing import Iterable, Iterator, Optional, TypeVar
+
+import pandas as pd
+
+from .logger import get_logger
+
+_T = TypeVar("_T")
+
+
+def chunker(df: pd.DataFrame, chunksize: int = 10000) -> Iterator[pd.DataFrame]:
+    """Yield successive non-overlapping chunks of *df*.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        The DataFrame to split.
+    chunksize : int
+        Number of rows per chunk (default 10 000).
+
+    Yields
+    ------
+    pd.DataFrame
+    """
+    for start in range(0, len(df), chunksize):
+        yield df.iloc[start : start + chunksize]
+
+
+def progress(
+    iterable: Iterable[_T],
+    total: Optional[int] = None,
+    desc: str = "",
+    log_every: Optional[int] = None,
+) -> Iterator[_T]:
+    """Wrap *iterable* with progress feedback that adapts to the environment.
+
+    Three modes, picked automatically (all gated on the ``eosframes`` logger
+    being at ``INFO`` or more verbose — a quieted logger gets a silent
+    pass-through):
+
+    * **Interactive (TTY)** — an in-place per-item bar on stderr.
+    * **Non-interactive with** *log_every* — periodic ``INFO`` log lines
+      (every *log_every* items, plus a final line). This is the path that
+      keeps long streaming jobs legible when stderr is a file or pipe:
+      ``nohup``, CI, ``build_scaler.sh``, etc. Without *log_every* a
+      non-interactive run stays silent.
+    * **Otherwise** — transparent pass-through, no output.
+
+    Both visible modes share stderr with the logger, so nothing ever leaks
+    onto piped stdout.
+
+    Parameters
+    ----------
+    iterable : Iterable
+        The items to iterate over.
+    total : int, optional
+        Expected number of items. Inferred via ``len(iterable)`` when omitted;
+        if neither is available the bar is disabled (percentages are dropped
+        from log lines but counts still flow).
+    desc : str
+        Short label shown to the left of the bar / log line.
+    log_every : int, optional
+        Emit an ``INFO`` log line every this many items when no bar is drawn.
+        Leave unset for the in-memory paths that should stay quiet off-TTY.
+
+    Yields
+    ------
+    The items of *iterable*, unchanged.
+    """
+    if total is None:
+        try:
+            total = len(iterable)  # type: ignore[arg-type]
+        except TypeError:
+            total = None
+
+    logger = get_logger()
+    info_on = logger.isEnabledFor(logging.INFO)
+    bar_on = total is not None and total > 1 and sys.stderr.isatty() and info_on
+
+    if bar_on:
+        width = 30
+        last_draw = -1.0  # force an immediate first draw
+        count = 0
+        for count, item in enumerate(iterable, 1):
+            yield item
+            now = time.monotonic()
+            # Throttle redraws to ~10 fps, but always paint the final item.
+            if count < total and now - last_draw < 0.1:
+                continue
+            last_draw = now
+            frac = count / total
+            filled = int(width * frac)
+            bar = "█" * filled + "·" * (width - filled)
+            label = f"{desc} " if desc else ""
+            sys.stderr.write(f"\r{label}|{bar}| {count}/{total}")
+            sys.stderr.flush()
+        if count:
+            sys.stderr.write("\n")
+            sys.stderr.flush()
+        return
+
+    if log_every and info_on:
+        label = desc or "progress"
+        count = 0
+        for count, item in enumerate(iterable, 1):
+            yield item
+            if count % log_every == 0:
+                if total:
+                    logger.info(
+                        "%s: %d/%d (%d%%)", label, count, total, count * 100 // total
+                    )
+                else:
+                    logger.info("%s: %d done", label, count)
+        # Final line unless the last item already landed on a logged boundary.
+        if count and count % log_every != 0:
+            if total:
+                logger.info("%s: %d/%d (100%%)", label, count, total)
+            else:
+                logger.info("%s: %d done", label, count)
+        return
+
+    yield from iterable

eosframes-1.1.0/src/eosframes/utils.py

@@ -1,23 +0,0 @@
-"""General-purpose utilities."""
-
-from typing import Iterator
-
-import pandas as pd
-
-
-def chunker(df: pd.DataFrame, chunksize: int = 10000) -> Iterator[pd.DataFrame]:
-    """Yield successive non-overlapping chunks of *df*.
-
-    Parameters
-    ----------
-    df : pd.DataFrame
-        The DataFrame to split.
-    chunksize : int
-        Number of rows per chunk (default 10 000).
-
-    Yields
-    ------
-    pd.DataFrame
-    """
-    for start in range(0, len(df), chunksize):
-        yield df.iloc[start : start + chunksize]