pyerrors 2.15.1__tar.gz → 2.17.0__tar.gz

{pyerrors-2.15.1 → pyerrors-2.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyerrors
-Version: 2.15.1
+Version: 2.17.0
 Summary: Error propagation and statistical analysis for Monte Carlo simulations
 Home-page: https://github.com/fjosw/pyerrors
 Author: Fabian Joswig
@@ -12,13 +12,13 @@ Project-URL: Changelog, https://github.com/fjosw/pyerrors/blob/master/CHANGELOG.
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Physics
-Requires-Python: >=3.9.0
+Requires-Python: >=3.10.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy>=2.0
@@ -52,7 +52,7 @@ Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary
 
-[![](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![arXiv](https://img.shields.io/badge/arXiv-2209.14371-b31b1b.svg)](https://arxiv.org/abs/2209.14371) [![DOI](https://img.shields.io/badge/DOI-10.1016%2Fj.cpc.2023.108750-blue)](https://doi.org/10.1016/j.cpc.2023.108750)
+[![](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![arXiv](https://img.shields.io/badge/arXiv-2209.14371-b31b1b.svg)](https://arxiv.org/abs/2209.14371) [![DOI](https://img.shields.io/badge/DOI-10.1016%2Fj.cpc.2023.108750-blue)](https://doi.org/10.1016/j.cpc.2023.108750)
 # pyerrors
 `pyerrors` is a python framework for error computation and propagation of Markov chain Monte Carlo data from lattice field theory and statistical mechanics simulations.
 

{pyerrors-2.15.1 → pyerrors-2.17.0}/README.md

@@ -1,4 +1,4 @@
-[![](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![arXiv](https://img.shields.io/badge/arXiv-2209.14371-b31b1b.svg)](https://arxiv.org/abs/2209.14371) [![DOI](https://img.shields.io/badge/DOI-10.1016%2Fj.cpc.2023.108750-blue)](https://doi.org/10.1016/j.cpc.2023.108750)
+[![](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![arXiv](https://img.shields.io/badge/arXiv-2209.14371-b31b1b.svg)](https://arxiv.org/abs/2209.14371) [![DOI](https://img.shields.io/badge/DOI-10.1016%2Fj.cpc.2023.108750-blue)](https://doi.org/10.1016/j.cpc.2023.108750)
 # pyerrors
 `pyerrors` is a python framework for error computation and propagation of Markov chain Monte Carlo data from lattice field theory and statistical mechanics simulations.
 

{pyerrors-2.15.1 → pyerrors-2.17.0}/pyerrors/fits.py

@@ -472,7 +472,7 @@ def least_squares(x, y, func, priors=None, silent=False, **kwargs):
             hat_vector = prepare_hat_matrix()
             A = W @ hat_vector
             P_phi = A @ np.linalg.pinv(A.T @ A) @ A.T
-            expected_chisquare = np.trace((np.identity(y_all.shape[-1]) - P_phi) @ W @ cov @ W)
+            expected_chisquare = np.trace((np.identity(y_all.shape[-1]) - P_phi) @ W @ cov @ W) + len(loc_priors)
             output.chisquare_by_expected_chisquare = output.chisquare / expected_chisquare
             if not silent:
                 print('chisquare/expected_chisquare:', output.chisquare_by_expected_chisquare)

{pyerrors-2.15.1 → pyerrors-2.17.0}/pyerrors/input/pandas.py

@@ -145,9 +145,9 @@ def _serialize_df(df, gz=False):
         serialize = _need_to_serialize(out[column])
 
         if serialize is True:
-            out[column] = out[column].transform(lambda x: create_json_string(x, indent=0) if x is not None else None)
+            out[column] = out[column].transform(lambda x: create_json_string(x, indent=0) if not _is_null(x) else None)
             if gz is True:
-                out[column] = out[column].transform(lambda x: gzip.compress((x if x is not None else '').encode('utf-8')))
+                out[column] = out[column].transform(lambda x: gzip.compress(x.encode('utf-8')) if not _is_null(x) else gzip.compress(b''))
     return out
 
 
@@ -166,37 +166,49 @@ def _deserialize_df(df, auto_gamma=False):
     ------
     In case any column of the DataFrame is gzipped it is gunzipped in the process.
     """
-    for column in df.select_dtypes(include="object"):
-        if isinstance(df[column][0], bytes):
-            if df[column][0].startswith(b"\x1f\x8b\x08\x00"):
-                df[column] = df[column].transform(lambda x: gzip.decompress(x).decode('utf-8'))
-
-        if not all([e is None for e in df[column]]):
+    # In pandas 3+, string columns use 'str' dtype instead of 'object'
+    string_like_dtypes = ["object", "str"] if int(pd.__version__.split(".")[0]) >= 3 else ["object"]
+    for column in df.select_dtypes(include=string_like_dtypes):
+        if len(df[column]) == 0:
+            continue
+        if isinstance(df[column].iloc[0], bytes):
+            if df[column].iloc[0].startswith(b"\x1f\x8b\x08\x00"):
+                df[column] = df[column].transform(lambda x: gzip.decompress(x).decode('utf-8') if not pd.isna(x) else '')
+
+        if df[column].notna().any():
             df[column] = df[column].replace({r'^$': None}, regex=True)
             i = 0
-            while df[column][i] is None:
+            while i < len(df[column]) and pd.isna(df[column].iloc[i]):
                 i += 1
-            if isinstance(df[column][i], str):
-                if '"program":' in df[column][i][:20]:
-                    df[column] = df[column].transform(lambda x: import_json_string(x, verbose=False) if x is not None else None)
+            if i < len(df[column]) and isinstance(df[column].iloc[i], str):
+                if '"program":' in df[column].iloc[i][:20]:
+                    df[column] = df[column].transform(lambda x: import_json_string(x, verbose=False) if not pd.isna(x) else None)
                     if auto_gamma is True:
-                        if isinstance(df[column][i], list):
-                            df[column].apply(lambda x: [o.gm() if o is not None else x for o in x])
+                        if isinstance(df[column].iloc[i], list):
+                            df[column].apply(lambda x: [o.gm() if o is not None else x for o in x] if x is not None else x)
                         else:
                             df[column].apply(lambda x: x.gm() if x is not None else x)
+        # Convert NA values back to Python None for compatibility with `x is None` checks
+        if df[column].isna().any():
+            df[column] = df[column].astype(object).where(df[column].notna(), None)
     return df
 
 
 def _need_to_serialize(col):
     serialize = False
     i = 0
-    while i < len(col) and col[i] is None:
+    while i < len(col) and _is_null(col.iloc[i]):
         i += 1
     if i == len(col):
         return serialize
-    if isinstance(col[i], (Obs, Corr)):
+    if isinstance(col.iloc[i], (Obs, Corr)):
         serialize = True
-    elif isinstance(col[i], list):
-        if all(isinstance(o, Obs) for o in col[i]):
+    elif isinstance(col.iloc[i], list):
+        if all(isinstance(o, Obs) for o in col.iloc[i]):
             serialize = True
     return serialize
+
+
+def _is_null(val):
+    """Check if a value is null (None or NA), handling list/array values."""
+    return False if isinstance(val, (list, np.ndarray)) else pd.isna(val)

{pyerrors-2.15.1 → pyerrors-2.17.0}/pyerrors/input/sfcf.py

@@ -5,12 +5,13 @@ import numpy as np  # Thinly-wrapped numpy
 from ..obs import Obs
 from .utils import sort_names, check_idl
 import itertools
+import warnings
 
 
 sep = "/"
 
 
-def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs):
+def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", cfg_func=None, silent=False, **kwargs):
     """Read sfcf files from given folder structure.
 
     Parameters
@@ -71,11 +72,11 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0,
     """
     ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type],
                           noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version,
-                          cfg_separator=cfg_separator, silent=silent, **kwargs)
+                          cfg_separator=cfg_separator, cfg_func=cfg_func, silent=silent, **kwargs)
     return ret[name][quarks][str(noffset)][str(wf)][str(wf2)]
 
 
-def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", silent=False, keyed_out=False, **kwargs):
+def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", cfg_func=None, silent=False, keyed_out=False, **kwargs):
     """Read sfcf files from given folder structure.
 
     Parameters
@@ -245,6 +246,16 @@ def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=
     for key in needed_keys:
         internal_ret_dict[key] = []
 
+    def _default_idl_func(cfg_string, cfg_sep):
+        return int(cfg_string.split(cfg_sep)[-1])
+
+    if cfg_func is None:
+        print("Default idl function in use.")
+        cfg_func = _default_idl_func
+        cfg_func_args = [cfg_separator]
+    else:
+        cfg_func_args = kwargs.get("cfg_func_args", [])
+
     if not appended:
         for i, item in enumerate(ls):
             rep_path = path + '/' + item
@@ -268,7 +279,7 @@ def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=
             for cfg in sub_ls:
                 try:
                     if compact:
-                        rep_idl.append(int(cfg.split(cfg_separator)[-1]))
+                        rep_idl.append(cfg_func(cfg, *cfg_func_args))
                     else:
                         rep_idl.append(int(cfg[3:]))
                 except Exception:
@@ -351,7 +362,7 @@ def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=
             for rep, file in enumerate(name_ls):
                 rep_idl = []
                 filename = path + '/' + file
-                T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single'])
+                T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], im, intern[name]['single'], cfg_func, cfg_func_args)
                 if rep == 0:
                     intern[name]['T'] = T
                     for t in range(intern[name]['T']):
@@ -581,12 +592,7 @@ def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im):
     return return_vals
 
 
-def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single):
-    try:
-        idl = int(chunk[gauge_line].split(cfg_sep)[-1])
-    except Exception:
-        raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line)
-
+def _read_chunk_data(chunk, start_read, T, corr_line, b2b, pattern, im, single):
     found_pat = ""
     data = []
     for li in chunk[corr_line + 1:corr_line + 6 + b2b]:
@@ -595,46 +601,90 @@ def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, patte
         for t, line in enumerate(chunk[start_read:start_read + T]):
             floats = list(map(float, line.split()))
             data.append(floats[im + 1 - single])
-    return idl, data
+    return data
 
 
-def _read_append_rep(filename, pattern, b2b, cfg_separator, im, single):
+def _check_append_rep(content, start_list):
+    data_len_list = []
+    header_len_list = []
+    has_regular_len_heads = True
+    for chunk_num in range(len(start_list)):
+        start = start_list[chunk_num]
+        if chunk_num == len(start_list) - 1:
+            stop = len(content)
+        else:
+            stop = start_list[chunk_num + 1]
+        chunk = content[start:stop]
+        for linenumber, line in enumerate(chunk):
+            if line.startswith("[correlator]"):
+                header_len = linenumber
+                break
+        header_len_list.append(header_len)
+        data_len_list.append(len(chunk) - header_len)
+
+    if len(set(header_len_list)) > 1:
+        warnings.warn("Not all headers have the same length. Data parts do.")
+        has_regular_len_heads = False
+
+    if len(set(data_len_list)) > 1:
+        raise Exception("Irregularities in file structure found, not all run data are of the same output length")
+    return has_regular_len_heads
+
+
+def _read_chunk_structure(chunk, pattern, b2b):
+    start_read = 0
+    for linenumber, line in enumerate(chunk):
+        if line.startswith("gauge_name"):
+            gauge_line = linenumber
+        elif line.startswith("[correlator]"):
+            corr_line = linenumber
+            found_pat = ""
+            for li in chunk[corr_line + 1: corr_line + 6 + b2b]:
+                found_pat += li
+            if re.search(pattern, found_pat):
+                start_read = corr_line + 7 + b2b
+                break
+    if start_read == 0:
+        raise ValueError("Did not find pattern\n", pattern)
+    endline = corr_line + 6 + b2b
+    while not chunk[endline] == "\n":
+        endline += 1
+    T = endline - start_read
+    return gauge_line, corr_line, start_read, T
+
+
+def _read_append_rep(filename, pattern, b2b, im, single, idl_func, cfg_func_args):
     with open(filename, 'r') as fp:
         content = fp.readlines()
-        data_starts = []
+        chunk_start_lines = []
         for linenumber, line in enumerate(content):
             if "[run]" in line:
-                data_starts.append(linenumber)
-        if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1:
-            raise Exception("Irregularities in file structure found, not all runs have the same output length")
-        chunk = content[:data_starts[1]]
-        for linenumber, line in enumerate(chunk):
-            if line.startswith("gauge_name"):
-                gauge_line = linenumber
-            elif line.startswith("[correlator]"):
-                corr_line = linenumber
-                found_pat = ""
-                for li in chunk[corr_line + 1: corr_line + 6 + b2b]:
-                    found_pat += li
-                if re.search(pattern, found_pat):
-                    start_read = corr_line + 7 + b2b
-                    break
-                else:
-                    raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename)
-        endline = corr_line + 6 + b2b
-        while not chunk[endline] == "\n":
-            endline += 1
-        T = endline - start_read
-
-        # all other chunks should follow the same structure
+                chunk_start_lines.append(linenumber)
+        has_regular_len_heads = _check_append_rep(content, chunk_start_lines)
+        if has_regular_len_heads:
+            chunk = content[:chunk_start_lines[1]]
+            try:
+                gauge_line, corr_line, start_read, T = _read_chunk_structure(chunk, pattern, b2b)
+            except ValueError:
+                raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename, "lines", 1, "to", chunk_start_lines[1] + 1)
+        # if has_regular_len_heads is true, all other chunks should follow the same structure
         rep_idl = []
         rep_data = []
 
-        for cnfg in range(len(data_starts)):
-            start = data_starts[cnfg]
-            stop = start + data_starts[1]
+        for chunk_num in range(len(chunk_start_lines)):
+            start = chunk_start_lines[chunk_num]
+            if chunk_num == len(chunk_start_lines) - 1:
+                stop = len(content)
+            else:
+                stop = chunk_start_lines[chunk_num + 1]
             chunk = content[start:stop]
-            idl, data = _read_chunk(chunk, gauge_line, cfg_separator, start_read, T, corr_line, b2b, pattern, im, single)
+            if not has_regular_len_heads:
+                gauge_line, corr_line, start_read, T = _read_chunk_structure(chunk, pattern, b2b)
+            try:
+                idl = idl_func(chunk[gauge_line], *cfg_func_args)
+            except Exception:
+                raise Exception("Couldn't parse idl from file", filename, ", problem with chunk of lines", start + 1, "to", stop + 1)
+            data = _read_chunk_data(chunk, start_read, T, corr_line, b2b, pattern, im, single)
             rep_idl.append(idl)
             rep_data.append(data)
 

pyerrors-2.17.0/pyerrors/version.py

@@ -0,0 +1 @@
+__version__ = "2.17.0"

{pyerrors-2.15.1 → pyerrors-2.17.0}/pyerrors.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyerrors
-Version: 2.15.1
+Version: 2.17.0
 Summary: Error propagation and statistical analysis for Monte Carlo simulations
 Home-page: https://github.com/fjosw/pyerrors
 Author: Fabian Joswig
@@ -12,13 +12,13 @@ Project-URL: Changelog, https://github.com/fjosw/pyerrors/blob/master/CHANGELOG.
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Physics
-Requires-Python: >=3.9.0
+Requires-Python: >=3.10.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy>=2.0
@@ -52,7 +52,7 @@ Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary
 
-[![](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![arXiv](https://img.shields.io/badge/arXiv-2209.14371-b31b1b.svg)](https://arxiv.org/abs/2209.14371) [![DOI](https://img.shields.io/badge/DOI-10.1016%2Fj.cpc.2023.108750-blue)](https://doi.org/10.1016/j.cpc.2023.108750)
+[![](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![arXiv](https://img.shields.io/badge/arXiv-2209.14371-b31b1b.svg)](https://arxiv.org/abs/2209.14371) [![DOI](https://img.shields.io/badge/DOI-10.1016%2Fj.cpc.2023.108750-blue)](https://doi.org/10.1016/j.cpc.2023.108750)
 # pyerrors
 `pyerrors` is a python framework for error computation and propagation of Markov chain Monte Carlo data from lattice field theory and statistical mechanics simulations.
 

{pyerrors-2.15.1 → pyerrors-2.17.0}/setup.py

@@ -24,18 +24,18 @@ setup(name='pyerrors',
       author_email='fabian.joswig@ed.ac.uk',
       license="MIT",
       packages=find_packages(),
-      python_requires='>=3.9.0',
+      python_requires='>=3.10.0',
       install_requires=['numpy>=2.0', 'autograd>=1.7.0', 'numdifftools>=0.9.41', 'matplotlib>=3.9', 'scipy>=1.13', 'iminuit>=2.28', 'h5py>=3.11', 'lxml>=5.0', 'python-rapidjson>=1.20', 'pandas>=2.2'],
       extras_require={'test': ['pytest', 'pytest-cov', 'pytest-benchmark', 'hypothesis', 'nbmake', 'flake8']},
       classifiers=[
           'Development Status :: 5 - Production/Stable',
           'Intended Audience :: Science/Research',
           'Programming Language :: Python :: 3',
-          'Programming Language :: Python :: 3.9',
           'Programming Language :: Python :: 3.10',
           'Programming Language :: Python :: 3.11',
           'Programming Language :: Python :: 3.12',
           'Programming Language :: Python :: 3.13',
+          'Programming Language :: Python :: 3.14',
           'Topic :: Scientific/Engineering :: Physics'
       ],
      )

pyerrors-2.15.1/pyerrors/version.py

@@ -1 +0,0 @@
-__version__ = "2.15.1"