pytesprocess 0.1.2__tar.gz → 0.1.3__tar.gz

{pytesprocess-0.1.2/pytesprocess.egg-info → pytesprocess-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytesprocess
-Version: 0.1.2
+Version: 0.1.3
 Summary: Detector data processing for pytesdaqx acquisitions
 Author-email: Bruno Serfass <serfass@berkeley.edu>, Samuel Watkins <samwatkins@berkeley.edu>
 License: MIT License
@@ -35,7 +35,7 @@ Requires-Dist: numpy>=1.26
 Requires-Dist: scipy
 Requires-Dist: matplotlib
 Requires-Dist: PyYAML
-Requires-Dist: qetpy>=1.8.6
+Requires-Dist: qetpy>=1.8.9
 Requires-Dist: pandas
 Requires-Dist: tables
 Requires-Dist: pytesdaqx

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
   "scipy",
   "matplotlib",
   "PyYAML",
-  "qetpy>=1.8.6",
+  "qetpy>=1.8.9",
   "pandas",
   "tables",
   "pytesdaqx",

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pytesprocess/_version.py

@@ -1,2 +1,2 @@
 # special file for defining the current version of the package
-__version__ = "0.1.2"
+__version__ = "0.1.3"

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pytesprocess/process/data_source.py

@@ -311,7 +311,18 @@ class RawDataSource:
 
 
 class TriggerDataframeSource:
-    """Iterate independent trigger-dataframe tasks."""
+    """Iterate independent trigger-dataframe tasks.
+
+    Rows are read a block at a time rather than one at a time.  Asking vaex
+    for a single row makes it re-derive and re-hash its whole column set on
+    every call: profiling a 100-event unit showed nearly four million hash
+    operations, about 39,000 an event, dwarfing the feature extraction
+    itself.  One read per block removes almost all of it.
+    """
+
+    #: Rows fetched per vaex call.  A trigger dataframe row is a few hundred
+    #: scalars, so a block of these is small; the win is in the call count.
+    CHUNK_ROWS = 2000
 
     def __init__(self, trigger_files, selected_streams=None):
         self._trigger_files = trigger_files or {}
@@ -323,6 +334,12 @@ class TriggerDataframeSource:
         self._task_id = None
         self._segments = None
         self._segment_index = -1
+        self._buffer = []
+        self._buffer_start = 0
+
+    def _reset_buffer(self):
+        self._buffer = []
+        self._buffer_start = 0
 
     @property
     def task_id(self):
@@ -345,6 +362,7 @@ class TriggerDataframeSource:
         self._dataframe = vx.open_many(self._trigger_files[task_id])
         self._index = -1
         self._row_stop = len(self._dataframe)
+        self._reset_buffer()
 
     def select_segments(self, segments):
         """Iterate an explicit, ordered list of dataframe row ranges.
@@ -362,6 +380,7 @@ class TriggerDataframeSource:
         self._segment_index = -1
         self._index = -1
         self._row_stop = 0
+        self._reset_buffer()
 
     def _open_next_segment(self):
         """Open the next segment's shard.  Return False when none remain."""
@@ -381,23 +400,40 @@ class TriggerDataframeSource:
             )
         self._index = segment.row_start - 1
         self._row_stop = segment.row_stop
+        self._reset_buffer()
         return True
 
+    def _fill_buffer(self):
+        """Read the next block of rows with a single vaex call."""
+        start = self._index
+        stop = min(start + self.CHUNK_ROWS, self._row_stop)
+        self._buffer = self._dataframe[start:stop].to_records()
+        self._buffer_start = start
+
+    def _buffered_row(self):
+        """Return the next row of the open shard, or None at its end."""
+        self._index += 1
+        if self._index >= self._row_stop:
+            return None
+
+        offset = self._index - self._buffer_start
+        if not 0 <= offset < len(self._buffer):
+            self._fill_buffer()
+            offset = self._index - self._buffer_start
+        return self._buffer[offset]
+
     def _next_raw_row(self):
         """Return the next row of the current selection, before filtering."""
         if self._segments is None:
             if self._dataframe is None:
                 return None
-            self._index += 1
-            if self._index >= self._row_stop:
-                return None
-            return self._dataframe.to_records(index=self._index)
+            return self._buffered_row()
 
         while True:
             if self._dataframe is not None:
-                self._index += 1
-                if self._index < self._row_stop:
-                    return self._dataframe.to_records(index=self._index)
+                row = self._buffered_row()
+                if row is not None:
+                    return row
             if not self._open_next_segment():
                 return None
 

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pytesprocess/process/feature_errors.py

@@ -21,15 +21,22 @@ class FeatureErrorHandler:
     A failed algorithm's features are set to ``FEATURE_ERROR_VALUE``. Output
     names are registered before processing for built-ins and declared external
     algorithms. Legacy external names can still be learned on first success.
+
+    Errors are reported once per algorithm at the end of the worker, not once
+    per event: a window or noise-model problem usually repeats for thousands
+    of consecutive events and the per-event lines bury everything else. Set
+    ``max_logged`` above zero to also log that many individual failures while
+    they happen.
     """
 
-    def __init__(self, node_label='', max_logged=10):
+    def __init__(self, node_label='', max_logged=0):
         self._node_label = node_label
         self._max_logged = max_logged
         self._output_names = {}
         self._declared = set()
         self._pending_rows = {}
         self._counts = {}
+        self._first_errors = {}
 
     @property
     def total(self):
@@ -72,6 +79,13 @@ class FeatureErrorHandler:
         key = self._key(plan)
         self._counts[key] = self._counts.get(key, 0) + 1
 
+        # One failing algorithm usually fails for many events in a row, so
+        # only the first error of each algorithm is kept; report() prints it
+        # once with the total count instead of one line per event.
+        self._first_errors.setdefault(
+            key, (f'{type(error).__name__}: {error}',
+                  self._event_label(event_features, event_index)))
+
         if self.total <= self._max_logged:
             print(f'WARNING{self._node_label}: Feature algorithm "{key}" failed '
                   f'for {self._event_label(event_features, event_index)}: '
@@ -92,6 +106,15 @@ class FeatureErrorHandler:
         details = ', '.join(f'{key}: {count}' for key, count in self._counts.items())
         print(f'WARNING{self._node_label}: {self.total} feature algorithm '
               f'error(s) skipped ({details})')
+        # The same problem usually hits several channels on the same event,
+        # so identical messages are reported once for all of them.
+        grouped = {}
+        for key in self._counts:
+            grouped.setdefault(self._first_errors[key], []).append(key)
+        for (message, event_label), keys in grouped.items():
+            who = 'all' if len(keys) == len(self._counts) > 1 else ', '.join(keys)
+            print(f'WARNING{self._node_label}:   first error ({who}, '
+                  f'{event_label}): {message}')
         never_succeeded = sorted(self._pending_rows)
         if never_succeeded:
             print(f'WARNING{self._node_label}: Algorithm(s) {never_succeeded} never '
@@ -106,8 +129,13 @@ class FeatureErrorHandler:
         parts = []
         if event_features.get('stream_id') is not None:
             parts.append(f'stream {event_features["stream_id"]}')
-        parts.extend(f'{field}={event_features[field]}' for field in _EVENT_ID_FIELDS
-                     if event_features.get(field) is not None)
+        for field in _EVENT_ID_FIELDS:
+            value = event_features.get(field)
+            if value is None:
+                continue
+            if field == 'global_segment_number' and int(value) < 0:
+                continue        # continuous Zarr has no segment number
+            parts.append(f'{field}={value}')
         if event_index is not None:
             parts.append(f'worker event #{event_index}')
         return ', '.join(parts) or 'event'

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pytesprocess/process/features.py

@@ -38,6 +38,7 @@ from pytesprocess.utils import create_dataframe_group
 from pytesprocess.utils.dataframe_output import (
     attach_dataframe_group, export_empty_processing_dataframe,
 )
+from pytesprocess.utils.shard_output import export_shard
 
 warnings.filterwarnings("ignore")
 
@@ -435,7 +436,9 @@ class FeatureProcessing:
             
         # processing done
         if self._verbose:
-            print('INFO: Feature processing done!') 
+            print('INFO: Feature processing done!')
+            if output_group_path is not None:
+                print(f'INFO: Processing output group path: {output_group_path}')
                 
         
         if lgc_output:
@@ -873,9 +876,9 @@ class FeatureProcessing:
                         vx.settings.main.thread_count_io = 1
                         pa.set_cpu_count(1)
                         feature_vx = vx.from_pandas(feature_df, copy_index=False)
-                        feature_vx.export_hdf5(partial_file, mode='w')
+                        export_shard(feature_vx, output_file,
+                                     verbose=self._verbose)
                         feature_vx.close()
-                        os.replace(partial_file, output_file)
                         del feature_df, feature_vx
                         gc.collect()
                         feature_df = pd.DataFrame()

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pytesprocess/process/randoms.py

@@ -26,6 +26,7 @@ from pytesprocess.process.work_plan import (
 )
 from pytesprocess.utils import create_dataframe_group
 from pytesprocess.utils.dataframe_output import attach_dataframe_group
+from pytesprocess.utils.shard_output import export_shard
 from pytesprocess.utils.event_metadata import coordinate_metadata, metadata_value, raw_provenance
 
 
@@ -473,6 +474,11 @@ class Randoms:
                 "INFO: Randoms acquisition done! "
                 f"Generated {target_nrandoms} randoms."
             )
+            if output_path is not None:
+                print(
+                    "INFO: Processing output acquisition path: "
+                    f"{output_path}"
+                )
 
         if lgc_output:
             return output_df
@@ -895,7 +901,8 @@ class Randoms:
 
     def _save_dataframe(self, dataframe, *, dataframe_group):
         """Save the generated random-trigger dataframe as one F0001 shard."""
-        dataframe.export_hdf5(dataframe_group.file_path(1), mode="w")
+        export_shard(dataframe, dataframe_group.file_path(1),
+                     verbose=self._verbose)
 
     def _initialize_feature_dict(self, *, include_hdf5, include_partition):
         feature_dict = {

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pytesprocess/process/triggers.py

@@ -27,6 +27,7 @@ from pytesprocess.process.work_plan import (
     count_rows, find_unit, load_plan, utc_now, write_plan, write_unit_status,
 )
 from pytesprocess.utils.event_metadata import coordinate_metadata
+from pytesprocess.utils.shard_output import export_shard
 from pytesprocess.utils.dataframe_output import (
     attach_dataframe_group,
     empty_processing_dataframe, export_empty_processing_dataframe,
@@ -387,7 +388,9 @@ class TriggerProcessing:
                
         # processing done
         if self._verbose:
-            print('INFO: Trigger processing done!') 
+            print('INFO: Trigger processing done!')
+            if output_group_path is not None:
+                print(f'INFO: Processing output group path: {output_group_path}')
                 
         if lgc_output:
             return output_df 
@@ -909,17 +912,9 @@ class TriggerProcessing:
                         export_empty_processing_dataframe(partial_file, dataframe_metadata)
                         os.replace(partial_file, output_file)
                     elif lgc_save:
-                        try:
-                            process_df.export_hdf5(partial_file, mode='w')
-                            process_df.close()
-                        except Exception as e:
-                            print('WARNING: Export failed with error: ', e)
-                            print('Will try again...')
-                            df_pandas = process_df.to_pandas_df()
-                            process_df = vx.from_pandas(df_pandas, copy_index=False)
-                            process_df.export_hdf5(partial_file, mode='w')
-                            process_df.close()
-                        os.replace(partial_file, output_file)
+                        export_shard(process_df, output_file,
+                                     verbose=self._verbose)
+                        process_df.close()
                         del process_df
                         process_df = None
                             

pytesprocess-0.1.3/pytesprocess/utils/shard_output.py

@@ -0,0 +1,94 @@
+"""Writing one dataframe shard, atomically, on filesystems that resist it.
+
+Two problems are handled here, and they are unrelated to each other.
+
+Vaex's HDF5 writer memory-maps its output file with ``PROT_WRITE``.  Some
+shared filesystems refuse that: NERSC's CFS, reached from a compute node,
+raises ``OSError(ENOSYS)``, and there is no vaex option to avoid the mmap.
+When that happens the shard is written somewhere local instead and copied
+into place, because an ordinary sequential write works where mmap does not.
+
+Separately, a dataframe vaex cannot write directly sometimes survives a
+round trip through pandas.  That fallback predates this module and is kept.
+
+Either way the shard reaches its final name by rename, so a job killed
+mid-write never leaves a truncated file that looks complete.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import shutil
+import tempfile
+
+
+__all__ = ['STAGING_ENV', 'export_shard', 'staging_directory']
+
+
+STAGING_ENV = 'PYTESPROCESS_STAGING_DIR'
+
+
+def staging_directory():
+    """Where to write a shard the destination filesystem cannot mmap.
+
+    Defaults to whatever the batch system set as the job's temporary
+    directory, which on a cluster is normally node-local.
+    """
+    return os.environ.get(STAGING_ENV) or os.environ.get('TMPDIR') or None
+
+
+def export_shard(dataframe, output_file, verbose=False):
+    """Write ``dataframe`` to ``output_file``, atomically.
+
+    Return
+    ------
+    output_file : str
+    """
+    output_file = str(output_file)
+    partial = output_file + '.partial'
+
+    try:
+        _export(dataframe, partial, verbose=verbose)
+    except Exception as error:
+        if verbose:
+            print(f'WARNING: Export failed with error: {error}')
+            print('Will try again via pandas...')
+        import vaex as vx
+
+        rebuilt = vx.from_pandas(dataframe.to_pandas_df(), copy_index=False)
+        try:
+            _export(rebuilt, partial, verbose=verbose)
+        finally:
+            rebuilt.close()
+
+    os.replace(partial, output_file)
+    return output_file
+
+
+def _export(dataframe, partial, verbose=False):
+    """Export to ``partial``, staging locally if the destination cannot mmap."""
+    try:
+        dataframe.export_hdf5(partial, mode='w')
+        return
+    except OSError as error:
+        if verbose:
+            print(f'INFO: Direct export failed ({error}); this filesystem '
+                  'does not support the memory mapping vaex writes through. '
+                  'Writing locally and copying in.')
+        _discard(partial)
+
+    directory = tempfile.mkdtemp(prefix='pytesprocess-', dir=staging_directory())
+    try:
+        staged = os.path.join(directory, Path(partial).name)
+        dataframe.export_hdf5(staged, mode='w')
+        shutil.copyfile(staged, partial)
+    finally:
+        shutil.rmtree(directory, ignore_errors=True)
+
+
+def _discard(path):
+    try:
+        os.remove(path)
+    except OSError:
+        pass

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pytesprocess/workflows/salting.py

@@ -8,6 +8,7 @@ import vaex as vx
 from pytesprocess.config import ProcessingConfig
 from pytesprocess.core import FilterData
 from pytesprocess.salting import SaltGenerator
+from pytesprocess.utils.shard_output import export_shard
 from pytesprocess.utils import (
     get_trigger_template_info, create_dataframe_group,
     add_dataframe_group_columns,
@@ -127,7 +128,7 @@ def generate_salting_dataframes(
         )
         add_dataframe_group_columns(final, group, 1)
         file_name = group.file_path(1)
-        final.export_hdf5(file_name, mode="w")
+        export_shard(final, file_name)
         results.append(file_name)
 
     return results

{pytesprocess-0.1.2 → pytesprocess-0.1.3/pytesprocess.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytesprocess
-Version: 0.1.2
+Version: 0.1.3
 Summary: Detector data processing for pytesdaqx acquisitions
 Author-email: Bruno Serfass <serfass@berkeley.edu>, Samuel Watkins <samwatkins@berkeley.edu>
 License: MIT License
@@ -35,7 +35,7 @@ Requires-Dist: numpy>=1.26
 Requires-Dist: scipy
 Requires-Dist: matplotlib
 Requires-Dist: PyYAML
-Requires-Dist: qetpy>=1.8.6
+Requires-Dist: qetpy>=1.8.9
 Requires-Dist: pandas
 Requires-Dist: tables
 Requires-Dist: pytesdaqx

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pytesprocess.egg-info/SOURCES.txt

@@ -76,6 +76,7 @@ pytesprocess/utils/dataframe_output.py
 pytesprocess/utils/downsampling.py
 pytesprocess/utils/event_metadata.py
 pytesprocess/utils/filter_hdf5.py
+pytesprocess/utils/shard_output.py
 pytesprocess/utils/utils.py
 pytesprocess/vibration/__init__.py
 pytesprocess/vibration/analyzer.py
@@ -113,6 +114,7 @@ tests/test_processing_data_v15.py
 tests/test_processing_guards.py
 tests/test_randoms_work_units.py
 tests/test_restricted_processing.py
+tests/test_shard_output.py
 tests/test_signed_feature_windows.py
 tests/test_template_metadata.py
 tests/test_trigger_alignment.py

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/pytesprocess.egg-info/requires.txt

@@ -2,7 +2,7 @@ numpy>=1.26
 scipy
 matplotlib
 PyYAML
-qetpy>=1.8.6
+qetpy>=1.8.9
 pandas
 tables
 pytesdaqx

{pytesprocess-0.1.2 → pytesprocess-0.1.3}/tests/test_feature_on_error.py

@@ -160,8 +160,12 @@ def test_skip_fills_failed_features_and_counts_errors(tmp_path, monkeypatch, cap
     assert list(df['feature_error_count'].values) == [2, 2, 0, 0, 2, 0]
     assert df['maximum_Z1P1'].notna().all()
 
-    assert 'Z1P1/flaky" failed for stream I1_D20260101_T000000, event_number=4' in output
     assert '6 feature algorithm error(s) skipped (Z1P1/flaky: 3, Z1P1/other_flaky: 3)' in output
+    # both algorithms fail the same way, so the reason is reported once
+    assert ('first error (all, stream I1_D20260101_T000000, '
+            'event_number=0') in output
+    assert 'RuntimeError: bad event 0' in output
+    assert 'bad event 4' not in output
 
 
 def test_skip_only_for_configured_algorithm(tmp_path, monkeypatch):
@@ -183,19 +187,58 @@ def test_no_error_column_without_skip(tmp_path, monkeypatch):
     assert 'feature_error_count' not in df.columns
 
 
-def test_handler_limits_logged_errors_and_reports_never_succeeded(capsys):
-    class Plan:
-        channel, name = 'Z1P1', 'always_fails'
+class _AlwaysFails:
+    channel, name = 'Z1P1', 'always_fails'
 
-    handler = FeatureErrorHandler(node_label=' Node #1', max_logged=3)
-    rows = [{'stream_id': 's'} for _ in range(5)]
+
+class _AlsoFails:
+    channel, name = 'Z2P1', 'also_fails'
+
+
+def test_repeated_failures_report_once_per_algorithm(capsys):
+    """Thousands of identical failures must not produce thousands of lines."""
+    handler = FeatureErrorHandler(node_label=' Node #1')
+    rows = [{'stream_id': 's', 'trigger_index': index} for index in range(500)]
     for index, row in enumerate(rows):
-        handler.record_failure(Plan, row, ValueError('boom'), event_index=index)
+        handler.record_failure(_AlwaysFails, row, ValueError(f'boom {index}'),
+                               event_index=index)
+    handler.report()
+    output = capsys.readouterr().out
+
+    # nothing is printed per event, only the end-of-worker report
+    assert '" failed for' not in output
+    assert '500 feature algorithm error(s) skipped (Z1P1/always_fails: 500)' in output
+    # the first error is still shown once, with the event that produced it
+    assert 'first error (Z1P1/always_fails, ' in output
+    assert 'trigger_index=0, worker event #0' in output
+    assert 'ValueError: boom 0' in output
+    assert 'boom 1' not in output
+    assert len(output.strip().splitlines()) == 3  # summary, detail, never-succeeded
+    assert "['Z1P1/always_fails'] never succeeded" in output
+    assert rows[0] == {'stream_id': 's', 'trigger_index': 0}
+
+
+def test_each_failing_algorithm_is_reported_separately(capsys):
+    handler = FeatureErrorHandler(node_label=' Node #1')
+    handler.record_failure(_AlwaysFails, {'stream_id': 's'}, ValueError('bad window'))
+    handler.record_failure(_AlsoFails, {'stream_id': 's'}, KeyError('no csd'))
+    handler.record_failure(_AlsoFails, {'stream_id': 's'}, KeyError('no csd'))
+    handler.report()
+    output = capsys.readouterr().out
+
+    assert '3 feature algorithm error(s) skipped (Z1P1/always_fails: 1, Z2P1/also_fails: 2)' in output
+    assert 'first error (Z1P1/always_fails, stream s): ValueError: bad window' in output
+    assert "first error (Z2P1/also_fails, stream s): KeyError: 'no csd'" in output
+
+
+def test_max_logged_restores_per_event_lines(capsys):
+    handler = FeatureErrorHandler(node_label=' Node #1', max_logged=3)
+    for index in range(5):
+        handler.record_failure(_AlwaysFails, {'stream_id': 's'}, ValueError('boom'),
+                               event_index=index)
     handler.report()
     output = capsys.readouterr().out
 
     assert output.count('" failed for') == 3
     assert 'Further feature algorithm errors are counted but not displayed' in output
     assert '5 feature algorithm error(s) skipped (Z1P1/always_fails: 5)' in output
-    assert "['Z1P1/always_fails'] never succeeded" in output
-    assert rows == [{'stream_id': 's'}] * 5

pytesprocess-0.1.3/tests/test_shard_output.py

@@ -0,0 +1,151 @@
+"""Writing a shard where the destination filesystem will not be memory-mapped.
+
+Vaex's HDF5 writer mmaps its output with PROT_WRITE.  NERSC's CFS refuses
+that from a compute node with OSError(ENOSYS), so the writer has to be able
+to produce the file somewhere else and copy it in.
+"""
+
+import errno
+import os
+from pathlib import Path
+
+import numpy as np
+import pytest
+import vaex as vx
+
+from pytesprocess.utils.shard_output import (
+    STAGING_ENV,
+    export_shard,
+    staging_directory,
+)
+
+
+@pytest.fixture
+def frame():
+    return vx.from_arrays(
+        event_number=np.arange(5, dtype=np.int64),
+        value=np.linspace(0.0, 1.0, 5),
+    )
+
+
+def read_back(path):
+    frame = vx.open(str(path))
+    try:
+        return frame.event_number.tolist()
+    finally:
+        frame.close()
+
+
+def refusing_export(refuse_under, refused):
+    """Stand in for a filesystem that will not be mmapped, like CFS.
+
+    A plain function, so monkeypatch binds it as a method and the dataframe
+    still arrives as self.
+    """
+    real = vx.dataframe.DataFrameLocal.export_hdf5
+
+    def export(self, path, *args, **kwargs):
+        if str(path).startswith(str(refuse_under)):
+            refused.append(str(path))
+            raise OSError(errno.ENOSYS, "Function not implemented")
+        return real(self, path, *args, **kwargs)
+
+    return export
+
+
+def test_a_shard_is_written_and_readable(tmp_path, frame):
+    output = tmp_path / 'trigger_F0001.hdf5'
+
+    export_shard(frame, output)
+
+    assert read_back(output) == [0, 1, 2, 3, 4]
+
+
+def test_no_partial_survives_a_successful_write(tmp_path, frame):
+    export_shard(frame, tmp_path / 'out.hdf5')
+
+    assert list(tmp_path.glob('*.partial')) == []
+
+
+def test_the_shard_appears_only_when_complete(tmp_path, frame, monkeypatch):
+    """A killed job must not leave a file that looks finished."""
+    output = tmp_path / 'out.hdf5'
+    seen = {}
+
+    real = vx.dataframe.DataFrameLocal.export_hdf5
+
+    def export(self, path, *args, **kwargs):
+        result = real(self, path, *args, **kwargs)
+        # Once vaex has written, but before the rename.
+        seen['during'] = sorted(p.name for p in tmp_path.iterdir())
+        return result
+
+    monkeypatch.setattr(vx.dataframe.DataFrameLocal, 'export_hdf5', export)
+    export_shard(frame, output)
+
+    assert seen['during'] == ['out.hdf5.partial'], \
+        'the final name must not exist until the shard is complete'
+    assert output.is_file()
+
+
+def test_a_destination_that_cannot_mmap_is_staged(tmp_path, frame, monkeypatch):
+    destination = tmp_path / 'cfs'
+    destination.mkdir()
+    staging = tmp_path / 'local'
+    staging.mkdir()
+    monkeypatch.setenv(STAGING_ENV, str(staging))
+
+    refused = []
+    monkeypatch.setattr(vx.dataframe.DataFrameLocal, 'export_hdf5',
+                        refusing_export(destination, refused))
+
+    output = destination / 'out.hdf5'
+    export_shard(frame, output)
+
+    assert refused, 'the direct write should have been attempted first'
+    assert read_back(output) == [0, 1, 2, 3, 4]
+
+
+def test_staging_leaves_nothing_behind(tmp_path, frame, monkeypatch):
+    destination = tmp_path / 'cfs'
+    destination.mkdir()
+    staging = tmp_path / 'local'
+    staging.mkdir()
+    monkeypatch.setenv(STAGING_ENV, str(staging))
+    monkeypatch.setattr(vx.dataframe.DataFrameLocal, 'export_hdf5',
+                        refusing_export(destination, []))
+
+    export_shard(frame, destination / 'out.hdf5')
+
+    assert list(staging.iterdir()) == []
+    assert sorted(p.name for p in destination.iterdir()) == ['out.hdf5']
+
+
+def test_staging_directory_follows_the_environment(monkeypatch):
+    monkeypatch.delenv(STAGING_ENV, raising=False)
+    monkeypatch.setenv('TMPDIR', '/scratch/job')
+    assert staging_directory() == '/scratch/job'
+
+    monkeypatch.setenv(STAGING_ENV, '/node/local')
+    assert staging_directory() == '/node/local'
+
+
+def test_a_dataframe_vaex_cannot_write_goes_via_pandas(tmp_path, frame,
+                                                       monkeypatch):
+    """The older fallback, kept for a different class of failure."""
+    calls = []
+    real = vx.dataframe.DataFrameLocal.export_hdf5
+
+    def export(self, path, *args, **kwargs):
+        calls.append(str(path))
+        if len(calls) == 1:
+            raise ValueError('cannot export column of type')
+        return real(self, path, *args, **kwargs)
+
+    monkeypatch.setattr(vx.dataframe.DataFrameLocal, 'export_hdf5', export)
+    output = tmp_path / 'out.hdf5'
+
+    export_shard(frame, output)
+
+    assert len(calls) == 2, 'the pandas round trip should have retried'
+    assert read_back(output) == [0, 1, 2, 3, 4]