pyKES 0.2.0__tar.gz → 0.2.2__tar.gz

{pykes-0.2.0 → pykes-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyKES
-Version: 0.2.0
+Version: 0.2.2
 Summary: Python package for kinetic modelling of chemical reaction networks.
 Keywords: kinetics,photocatalysis,water-splitting,reaction-networks,ode
 Author: Jacob Schneidewind

{pykes-0.2.0 → pykes-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "pyKES"
-version = "0.2.0"
+version = "0.2.2"
 description = "Python package for kinetic modelling of chemical reaction networks."
 keywords = ["kinetics", "photocatalysis", "water-splitting", "reaction-networks", "ode"]
 authors = [

{pykes-0.2.0 → pykes-0.2.2}/src/pyKES/database/data_processing.py

@@ -131,6 +131,155 @@ def read_in_single_experiment(file_name: str,
         }
 
 
+def ensure_processed_column(database: ExperimentalDataset) -> None:
+    """
+    Make sure ``overview_df`` carries the ``Processed`` flag as text.
+
+    Idempotent, and called by both entry points that touch the flag, so
+    neither depends on the other having run first.
+
+    Parameters
+    ----------
+    database : ExperimentalDataset
+        Dataset whose ``overview_df`` is given the column if it lacks one.
+
+    Returns
+    -------
+    None : None
+    """
+
+    # The flag is text, not a bool: it round-trips through Excel and HDF5, and
+    # comes back as the strings this module compares against. Seeding it with a
+    # bool gives the column bool dtype, which pandas then refuses to write
+    # 'True' into once the first experiment succeeds.
+    if "Processed" not in database.overview_df.columns:
+        database.overview_df["Processed"] = "False"
+
+
+def select_unprocessed_experiments(database: ExperimentalDataset,
+                                   overview_df_experiment_column: Optional[str] = 'Experiment') -> list:
+    """
+    List the experiments of the overview sheet that still need processing.
+
+    Parameters
+    ----------
+    database : ExperimentalDataset
+        Dataset whose ``overview_df`` lists the experiments. A missing
+        ``Processed`` column is added to it in place.
+    overview_df_experiment_column : str, optional
+        Column of ``overview_df`` holding the experiment names.
+
+    Returns
+    -------
+    list of str
+        Experiments that are either not flagged as processed or not yet held
+        by ``database.experiments``.
+    """
+
+    ensure_processed_column(database)
+
+    unprocessed = (
+        database.overview_df["Processed"].ne('True')
+        | ~database.overview_df[overview_df_experiment_column].isin(database.experiments))
+
+    return database.overview_df.loc[unprocessed,
+                    overview_df_experiment_column].astype(str).tolist()
+
+
+def ingest_experiment(experiment_name: str,
+                      database: ExperimentalDataset,
+                      metadata_retrival_function: callable,
+                      raw_data_reading_function: callable,
+                      processing_function: callable,
+                      overview_df_experiment_column: Optional[str] = 'Experiment',
+                      directory: Optional[Path] = None,
+                      external_version: Optional[dict] = None) -> dict:
+    """
+    Process one experiment of the overview sheet and add it to the dataset.
+
+    One step of `read_in_experiments_single_threaded`, exposed separately so a
+    caller can drive the ingestion one experiment at a time — which is what the
+    Streamlit page does to keep its progress bar alive in the browser.
+
+    Parameters
+    ----------
+    experiment_name : str
+        Experiment to process, as named in ``overview_df``.
+    database : ExperimentalDataset
+        Dataset the experiment is added to; mutated in place on success.
+    metadata_retrival_function : callable
+        Returns the metadata dict for an experiment name.
+    raw_data_reading_function : callable
+        Reads the raw data for an experiment from ``directory``.
+    processing_function : callable
+        Turns raw data and metadata into the processed-data dict.
+    overview_df_experiment_column : str, optional
+        Column of ``overview_df`` holding the experiment names.
+    directory : Path, optional
+        Directory the raw-data files are read from.
+    external_version : dict, optional
+        Provenance of the external app, stamped onto the experiment. Already
+        resolved — unlike the loop functions, this is not defaulted to the
+        dataset's own entry.
+
+    Returns
+    -------
+    result : dict
+        Result of `read_in_single_experiment`.
+    """
+
+    ensure_processed_column(database)
+
+    result = read_in_single_experiment(
+        file_name = experiment_name,
+        database = database,
+        metadata_retrival_function = metadata_retrival_function,
+        raw_data_reading_function = raw_data_reading_function,
+        processing_function = processing_function,
+        directory = directory,
+        legacy_mode = False,
+        external_version = external_version
+    )
+
+    if not result['success']:
+        print(f"Failed to process {result['file']}: {result['error']}")
+        return result
+
+    database.add_experiment(result['data'])
+
+    database.overview_df.loc[
+        database.overview_df[overview_df_experiment_column].eq(result['data'].experiment_name),
+            "Processed",
+        ] = 'True'
+
+    return result
+
+
+def finalize_processing_run(database: ExperimentalDataset,
+                            results: list,
+                            external_version: Optional[dict] = None) -> None:
+    """
+    Stamp the dataset version after a processing or reprocessing run.
+
+    Parameters
+    ----------
+    database : ExperimentalDataset
+        Dataset whose ``version`` dict is updated in place.
+    results : list of dict
+        Results of the run; the dataset is only stamped when at least one
+        experiment succeeded.
+    external_version : dict, optional
+        Provenance of the external app.
+
+    Returns
+    -------
+    None : None
+    """
+
+    if any(result['success'] for result in results):
+        database.stamp_version(processed=True, external_version=external_version)
+
+
 def read_in_experiments_single_threaded(database: ExperimentalDataset,
                                         metadata_retrival_function: callable,
                                         raw_data_reading_function: callable,
@@ -158,8 +307,8 @@ def read_in_experiments_single_threaded(database: ExperimentalDataset,
     directory : Path, optional
         Directory the raw-data files are read from.
     external_version : dict, optional
-        Provenance of the external app (e.g. its git commit), stamped onto
-        every processed experiment. Defaults to the dataset's own
+        Provenance of the external app (e.g. its own project version), stamped
+        onto every processed experiment. Defaults to the dataset's own
         ``version['external_version']``.
     progress_callback : callable, optional
         Called as ``(completed, total, experiment_name)`` once before the loop
@@ -173,24 +322,8 @@ def read_in_experiments_single_threaded(database: ExperimentalDataset,
         `read_in_single_experiment`.
     """
 
-    # The flag is text, not a bool: it round-trips through Excel and HDF5, and
-    # comes back as the strings this module compares against below. Seeding it
-    # with a bool gives the column bool dtype, which pandas then refuses to
-    # write 'True' into once the first experiment succeeds.
-    if "Processed" not in database.overview_df.columns:
-        database.overview_df["Processed"] = "False"
-
-    # Returning only the experiments which have not been processed 
-    # Do not contain "Processed" column or "Processed" is not True
-    # also returns experiments which are not in the database.experiments dict,
+    experiments = select_unprocessed_experiments(database, overview_df_experiment_column)
 
-    mask = (
-    database.overview_df["Processed"].ne('True')
-    | ~database.overview_df[overview_df_experiment_column].isin(database.experiments))
-    
-    experiments = database.overview_df.loc[mask, 
-                    overview_df_experiment_column].astype(str).tolist()
-    
     total_experiments = len(experiments)
     external_version = resolve_external_version(database, external_version)
 
@@ -201,37 +334,21 @@ def read_in_experiments_single_threaded(database: ExperimentalDataset,
 
     for completed, experiment_name in enumerate(experiments, start=1):
 
-        result = read_in_single_experiment(
-            file_name = experiment_name,
+        results.append(ingest_experiment(
+            experiment_name = experiment_name,
             database = database,
             metadata_retrival_function = metadata_retrival_function,
             raw_data_reading_function = raw_data_reading_function,
             processing_function = processing_function,
+            overview_df_experiment_column = overview_df_experiment_column,
             directory = directory,
-            legacy_mode = False,
             external_version = external_version
-        )
-
-        results.append(result)
-
-        if result['success']:
-            # Add experiment to database
-            database.add_experiment(result['data'])
-
-            # Setting "Processed" to True in dataframe
-            database.overview_df.loc[
-                database.overview_df[overview_df_experiment_column].eq(result['data'].experiment_name),
-                    "Processed",
-                ] = 'True'
-
-        else:
-            print(f"Failed to process {result['file']}: {result['error']}")
+        ))
 
         if progress_callback is not None:
             progress_callback(completed, total_experiments, experiment_name)
 
-    if any(result['success'] for result in results):
-        database.stamp_version(processed=True, external_version=external_version)
+    finalize_processing_run(database, results, external_version)
 
     return results
 
@@ -270,8 +387,7 @@ def read_in_experiments_multiprocessing(database: ExperimentalDataset,
         else:
             print(f"Failed to process {result['file']}: {result['error']}")
 
-    if any(result['success'] for result in results):
-        database.stamp_version(processed=True, external_version=external_version)
+    finalize_processing_run(database, results, external_version)
 
     return results
 
@@ -351,6 +467,80 @@ def reprocess_single_experiment(experiment: Experiment,
         }
 
 
+def reprocess_experiment_by_name(experiment_name: str,
+                                 database: ExperimentalDataset,
+                                 processing_function: callable,
+                                 metadata_retrival_function: Optional[callable] = None,
+                                 external_version: Optional[dict] = None) -> dict:
+    """
+    Reprocess one experiment of a dataset, addressed by name.
+
+    Name-addressed counterpart of `reprocess_single_experiment`, so that a
+    caller stepping through the experiments one at a time can reach both
+    pipelines through the same ``(experiment_name, **context)`` call shape as
+    `ingest_experiment`.
+
+    Parameters
+    ----------
+    experiment_name : str
+        Experiment to reprocess; must be present in ``database.experiments``.
+    database : ExperimentalDataset
+        Dataset holding the experiment and the overview sheet.
+    processing_function : callable
+        ``(raw_data_dict, metadata_dict) -> processed_data_dict``.
+    metadata_retrival_function : callable, optional
+        When given, metadata is refreshed from ``database.overview_df``.
+    external_version : dict, optional
+        Provenance of the external app, already resolved.
+
+    Returns
+    -------
+    result : dict
+        Result of `reprocess_single_experiment`.
+    """
+
+    return reprocess_single_experiment(
+        experiment = database.experiments[experiment_name],
+        processing_function = processing_function,
+        metadata_retrival_function = metadata_retrival_function,
+        overview_df = database.overview_df,
+        external_version = external_version
+    )
+
+
+def select_experiments_to_reprocess(database: ExperimentalDataset,
+                                    experiment_names: Optional[list] = None) -> list:
+    """
+    Resolve which experiments of a dataset a reprocessing run covers.
+
+    Parameters
+    ----------
+    database : ExperimentalDataset
+        Dataset holding the experiments.
+    experiment_names : list of str, optional
+        Requested experiments. Defaults to every experiment in the dataset.
+
+    Returns
+    -------
+    list of str
+        Experiments to reprocess.
+
+    Raises
+    ------
+    ValueError
+        If any requested experiment is not held by the dataset.
+    """
+
+    if experiment_names is None:
+        return sorted(database.experiments.keys())
+
+    unknown_experiments = [name for name in experiment_names if name not in database.experiments]
+    if unknown_experiments:
+        raise ValueError(f"Experiments not present in the dataset: {unknown_experiments}")
+
+    return list(experiment_names)
+
+
 def reprocess_experiments(database: ExperimentalDataset,
                           processing_function: callable,
                           metadata_retrival_function: Optional[callable] = None,
@@ -390,12 +580,7 @@ def reprocess_experiments(database: ExperimentalDataset,
         One result dict per experiment, as returned by
         `reprocess_single_experiment`.
     """
-    if experiment_names is None:
-        experiment_names = sorted(database.experiments.keys())
-
-    unknown_experiments = [name for name in experiment_names if name not in database.experiments]
-    if unknown_experiments:
-        raise ValueError(f"Experiments not present in the dataset: {unknown_experiments}")
+    experiment_names = select_experiments_to_reprocess(database, experiment_names)
 
     total_experiments = len(experiment_names)
     external_version = resolve_external_version(database, external_version)
@@ -407,21 +592,18 @@ def reprocess_experiments(database: ExperimentalDataset,
 
     for completed, experiment_name in enumerate(experiment_names, start=1):
 
-        result = reprocess_single_experiment(
-            experiment = database.experiments[experiment_name],
+        results.append(reprocess_experiment_by_name(
+            experiment_name = experiment_name,
+            database = database,
             processing_function = processing_function,
             metadata_retrival_function = metadata_retrival_function,
-            overview_df = database.overview_df,
             external_version = external_version
-        )
-
-        results.append(result)
+        ))
 
         if progress_callback is not None:
             progress_callback(completed, total_experiments, experiment_name)
 
-    if any(result['success'] for result in results):
-        database.stamp_version(processed=True, external_version=external_version)
+    finalize_processing_run(database, results, external_version)
 
     return results
 

{pykes-0.2.0 → pykes-0.2.2}/src/pyKES/database/database_experiments.py

@@ -292,8 +292,8 @@ class ExperimentalDataset:
             Whether a processing function was just run, which additionally
             sets ``'last_processed'``.
         external_version : dict, optional
-            Provenance of the external app (e.g. its git commit), merged into
-            the existing ``'external_version'`` entry.
+            Provenance of the external app (e.g. its own project version),
+            merged into the existing ``'external_version'`` entry.
 
         Returns
         -------
@@ -318,7 +318,7 @@ class ExperimentalDataset:
         alongside the data::
 
             dataset.set_external_version({'app': 'photocat',
-                                          'commit': get_git_commit(__file__)})
+                                          'version': get_project_version(__file__)})
 
         Parameters
         ----------
@@ -648,7 +648,7 @@ def usage_example():
     # Add dataset-level attributes
     dataset.plotting_instruction = {'xlabel': 'Time (s)', 'ylabel': 'Current (mA)'}
     dataset.group_mapping = {'GroupA': ['Exp1'], 'GroupB': ['Exp2']}
-    dataset.set_external_version({'app': 'usage_example', 'commit': 'abc123'})
+    dataset.set_external_version({'app': 'usage_example', 'version': '0.3.0'})
 
     exp1 = Experiment(
         experiment_name="Exp1",

pykes-0.2.2/src/pyKES/streamlit_app/chunked_processing.py

@@ -0,0 +1,280 @@
+"""
+Drive a long processing run one experiment per Streamlit rerun.
+
+The pyKES app is also deployed as a static browser page via stlite, which runs
+Streamlit inside Pyodide on the browser's *single* event loop — there is no
+separate script-runner thread. A loop that processes every experiment inside
+one script run therefore never lets Streamlit deliver anything: the progress
+messages it writes are queued and only flushed once the run yields to the
+event loop, which happens after the loop has already finished. The progress
+bar was invisible in the browser for exactly this reason.
+
+A job started here is instead advanced by one experiment per rerun. Each rerun
+ends, the event loop gets control, and the bar painted by that rerun reaches
+the screen.
+
+The reruns are app-scoped, so the page that hosts a job has to skip its
+expensive sections while `any_active_job` is true — the data-upload page
+rewrites the whole HDF5 file in its download section, which must not happen
+once per experiment. Fragment-scoped reruns would avoid that, but they are not
+usable here: ``st.rerun(scope="fragment")`` is rejected outside a fragment
+rerun, and the only way into one is a widget interaction or a ``run_every``
+timer that would then poll for the entire duration of the job.
+
+The same code path is used when running under ``streamlit run``; nothing here
+is browser-specific.
+
+See [docs/browser_deployment.md](../../../docs/browser_deployment.md).
+"""
+
+import shutil
+from typing import Callable, Optional
+
+import streamlit as st
+
+from pyKES.database.data_processing import finalize_processing_run
+
+
+# Session-state key holding the results of the most recently finished job,
+# derived from the job's own key. The results outlive the job itself so the page
+# can report them after the final rerun has torn the job down.
+JOB_RESULTS_KEY_TEMPLATE = "{job_key}_results"
+
+
+def job_results_key(job_key: str) -> str:
+    """
+    Session-state key under which a finished job parks its results.
+
+    Parameters
+    ----------
+    job_key : str
+        Key identifying the job.
+
+    Returns
+    -------
+    str
+        Session-state key of the results list.
+    """
+
+    return JOB_RESULTS_KEY_TEMPLATE.format(job_key=job_key)
+
+
+def start_chunked_job(job_key: str,
+                      experiment_names: list,
+                      context: dict,
+                      staging_directory: Optional[str] = None) -> None:
+    """
+    Register a processing run to be advanced one experiment per rerun.
+
+    The caller is expected to trigger a full ``st.rerun`` afterwards, so the
+    page re-renders with the job active.
+
+    Parameters
+    ----------
+    job_key : str
+        Session-state key identifying this job. One per page section.
+    experiment_names : list of str
+        Experiments to process, in order.
+    context : dict
+        Keyword arguments handed to the step function alongside the experiment
+        name — the processing callables, the staging directory and the
+        already-resolved external version. Must include the
+        `ExperimentalDataset` under ``'database'``: both step functions take it,
+        and the job is stamped against it when it finishes.
+    staging_directory : str, optional
+        Directory holding uploaded files for the duration of the job. Removed
+        when the job finishes.
+
+    Returns
+    -------
+    None : None
+    """
+
+    st.session_state.pop(job_results_key(job_key), None)
+
+    st.session_state[job_key] = {
+        'experiment_names': experiment_names,
+        'completed': 0,
+        'results': [],
+        'painted': False,
+        'staging_directory': staging_directory,
+        'context': context,
+    }
+
+
+def active_job(job_key: str) -> Optional[dict]:
+    """
+    Return the job registered under a key, if there is one.
+
+    Parameters
+    ----------
+    job_key : str
+        Session-state key identifying the job.
+
+    Returns
+    -------
+    dict or None
+        The job state, or None when no job is running.
+    """
+
+    return st.session_state.get(job_key)
+
+
+def collect_job_results(job_key: str) -> Optional[list]:
+    """
+    Take the results parked by the most recently finished job.
+
+    Parameters
+    ----------
+    job_key : str
+        Session-state key identifying the job.
+
+    Returns
+    -------
+    list of dict or None
+        The results, or None when no job has finished since the last call.
+    """
+
+    return st.session_state.pop(job_results_key(job_key), None)
+
+
+def paint_job_progress(job: dict) -> None:
+    """
+    Draw the progress bar for the experiment that is about to be processed.
+
+    Painting is a rerun of its own, doing no work, so that the bar reaches the
+    screen *before* the next experiment occupies the event loop. Painting and
+    processing in the same rerun would leave the bar one experiment behind and
+    never show the first one — which is the whole bug this module exists for.
+
+    Parameters
+    ----------
+    job : dict
+        Job state; ``painted`` is set in place.
+
+    Returns
+    -------
+    None : None
+    """
+
+    completed = job['completed']
+    total = len(job['experiment_names'])
+
+    st.progress(
+        completed / total,
+        text = f"Processing experiment {completed + 1}/{total}: {job['experiment_names'][completed]}",
+    )
+
+    job['painted'] = True
+
+
+def run_job_step(job: dict, step_function: Callable[..., dict]) -> None:
+    """
+    Process the next experiment of a job.
+
+    Parameters
+    ----------
+    job : dict
+        Job state; ``results``, ``completed`` and ``painted`` are updated in
+        place.
+    step_function : callable
+        ``(experiment_name, **context) -> result_dict``, e.g.
+        `pyKES.database.data_processing.ingest_experiment`.
+
+    Returns
+    -------
+    None : None
+    """
+
+    experiment_name = job['experiment_names'][job['completed']]
+
+    job['results'].append(step_function(experiment_name, **job['context']))
+    job['completed'] += 1
+    job['painted'] = False
+
+
+def finish_chunked_job(job_key: str, job: dict) -> None:
+    """
+    Stamp the dataset, clean up the staging directory and retire the job.
+
+    Parameters
+    ----------
+    job_key : str
+        Session-state key identifying the job.
+    job : dict
+        Job state to retire.
+
+    Returns
+    -------
+    None : None
+    """
+
+    # Stamped once for the whole run, as the non-chunked loop functions do,
+    # rather than after every experiment.
+    finalize_processing_run(job['context']['database'],
+                            job['results'],
+                            job['context'].get('external_version'))
+
+    if job['staging_directory'] is not None:
+        shutil.rmtree(job['staging_directory'])
+
+    st.session_state[job_results_key(job_key)] = job['results']
+    del st.session_state[job_key]
+
+
+def render_chunked_job(job_key: str, step_function: Callable[..., dict]) -> None:
+    """
+    Advance a job by one phase and rerun for the next.
+
+    Alternates painting and processing, so every experiment is announced
+    before it runs and the announcement reaches the screen while it runs.
+
+    Parameters
+    ----------
+    job_key : str
+        Session-state key identifying the job.
+    step_function : callable
+        ``(experiment_name, **context) -> result_dict``.
+
+    Returns
+    -------
+    None : None
+    """
+
+    job = active_job(job_key)
+
+    if job is None:
+        return
+
+    # st.rerun raises, so each branch below ends the script run.
+    if job['completed'] == len(job['experiment_names']):
+        finish_chunked_job(job_key, job)
+        st.rerun()
+
+    if job['painted']:
+        run_job_step(job, step_function)
+    else:
+        paint_job_progress(job)
+
+    st.rerun()
+
+
+def any_active_job(job_keys: list) -> bool:
+    """
+    Report whether any of the given jobs is currently running.
+
+    Pages use this to skip sections that are too expensive to re-render on
+    every step of a job.
+
+    Parameters
+    ----------
+    job_keys : list of str
+        Session-state keys of the jobs the page can start.
+
+    Returns
+    -------
+    bool
+        True while at least one of them is active.
+    """
+
+    return any(active_job(job_key) is not None for job_key in job_keys)