nnpdf-data 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

nnpdf_data/commondata/ZEUS_2JET_319GEV_374PB-1_DIF/metadata.yaml

@@ -17,40 +17,39 @@ version_comment: "Separated luminosity uncertainty"
 
 implemented_observables:
 
-  - observable: {description: "double differential cross section w.r.t. E_T and q^2", label: '$\frac{d^2\sigma}{dE_Tdq^2}$', units: "$pb/GeV^3$"}
-    observable_name: ETQ2
-    process_type: HERADIJET
-    ndata: 22
-    tables: [13, 14, 15, 16, 17, 18]
-    kinematics:
-      variables:
-        ET: {description: "mean jet transverse energy of the dijet system in the Breit frame", label: '$\langle E_{T, B} \rangle$', units: "$GeV$"}
-        Q2: {description: "momentum transferred squared", label: "$q^2$", units: "$GeV^2$"}
-        sqrts: {description: "center of mass energy", label: '$\sqrt{s}$', units: "$GeV$"}
-      file: kinematics_q2_et.yaml
-    data_central: data_q2_et.yaml
-    data_uncertainties:
+- observable: {description: "double differential cross section w.r.t. E_T and q^2", label: '$\frac{d^2\sigma}{dE_Tdq^2}$', units: "$pb/GeV^3$"}
+  observable_name: ETQ2
+  process_type: HERADIJET
+  ndata: 22
+  tables: [13, 14, 15, 16, 17, 18]
+  kinematics:
+    variables:
+      ET: {description: "mean jet transverse energy of the dijet system in the Breit frame", label: '$\langle E_{T, B} \rangle$', units: "$GeV$"}
+      Q2: {description: "momentum transferred squared", label: "$q^2$", units: "$GeV^2$"}
+      sqrts: {description: "center of mass energy", label: '$\sqrt{s}$', units: "$GeV$"}
+    file: kinematics_q2_et.yaml
+  data_central: data_q2_et.yaml
+  data_uncertainties:
+  - uncertainties_q2_et_wo-lumi.yaml
+  - uncertainties_q2_et_lumi.yaml
+  kinematic_coverage: [ET, Q2, sqrts]
+  plotting:
+    dataset_label: '$ZEUS\ DiJet\ 374\ pb^{-1}:\ \frac{d^2\sigma}{dE_Tdq^2}$'
+    plot_x: ET
+    y_label: '$\frac{d^2\sigma}{dE_Tdq^2}$ ($pb/GeV^3$)'
+    x_scale: log
+    figure_by:
+    - Q2
+  theory:
+    FK_tables:
+    - - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN1
+      - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN2
+      - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN3
+      - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN4
+      - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN5
+      - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN6
+    operation: 'null'
+  variants:
+    wo-lumi:
+      data_uncertainties:
       - uncertainties_q2_et_wo-lumi.yaml
-      - uncertainties_q2_et_lumi.yaml
-    kinematic_coverage: [ET, Q2, sqrts]
-    plotting:
-      kinematics_override: identity
-      dataset_label: '$ZEUS\ DiJet\ 374\ pb^{-1}:\ \frac{d^2\sigma}{dE_Tdq^2}$'
-      plot_x: ET
-      y_label: '$\frac{d^2\sigma}{dE_Tdq^2}$ ($pb/GeV^3$)'
-      x_scale: log
-      figure_by:
-        - Q2
-    theory:
-      FK_tables:
-        - - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN1
-          - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN2
-          - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN3
-          - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN4
-          - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN5
-          - ZEUS_2JET_319GEV_374PB-1_DIF_ETQ2_BIN6
-      operation: 'null'
-    variants:
-      wo-lumi:
-        data_uncertainties:
-          - uncertainties_q2_et_wo-lumi.yaml

nnpdf_data/commondata/dataset_names.yml

@@ -330,7 +330,7 @@ CMS_SINGLETOP_TCH_TOT_7TEV:
   dataset: CMS_SINGLETOP_7TEV_TCHANNEL-XSEC
   variant: legacy
 CMS_TTB_DIFF_13TEV_2016_LJ_TRAP:
-  dataset: CMS_TTBAR_13TEV_LJ_2016_DIF_YTTBAR
+  dataset: CMS_TTBAR_13TEV_LJ_2016_DIF_YT
   variant: legacy
 POSDYS:
   dataset: NNPDF_POS_2P24GEV_DYS
@@ -412,3 +412,6 @@ ATLAS_WJ_JET_8TEV_WM-PT:
 ATLAS_WJ_JET_8TEV_WP-PT:
   dataset: ATLAS_WJ_8TEV_WP-PT
   variant: legacy
+CMS_TTBAR_13TEV_LJ_2016_DIF_YTTBAR:
+  dataset: CMS_TTBAR_13TEV_LJ_2016_DIF_YT
+  variant: legacy

nnpdf_data/commondataparser.py

@@ -37,22 +37,20 @@ by modifying the CommonMetaData using one of the loaded Variants one can change
 """
 
 import dataclasses
-from functools import cached_property, lru_cache
+from functools import cache, cached_property
 import logging
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Optional
 
 import numpy as np
 import pandas as pd
 from validobj import ValidationError, parse_input
 from validobj.custom import Parser
 
-# We cannot use ruamel directly due to the ambiguity ruamel.yaml / ruamel_yaml
-# of some versions which are pinned in some of the conda packages we use...
-from . import new_to_legacy_map, path_commondata
 from .coredata import KIN_NAMES, CommonData
 from .process_options import ValidProcess
 from .utils import parse_yaml_inp, quick_yaml_load
+from .validphys_compatibility import new_to_legacy_map, path_commondata
 
 try:
     from validphys.plotoptions.plottingoptions import PlottingOptions, labeler_functions
@@ -66,7 +64,6 @@ except ModuleNotFoundError:
 
     VP_AVAILABLE = False
 
-
 # JCM:
 # Some notes for developers
 # The usage of `frozen` in the definitions of the dataclass is not strictly necessary
@@ -107,6 +104,10 @@ KINLABEL_LATEX = {
     "SHP_ASY": ("$\\eta$", "$p_T (GeV)$", "$\\sqrt{s} (GeV)$"),
     "JET_POL": ("$\\eta$", "$p_T^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"),
     "DIJET_POL": ("$\\m_{1,2} (GeV)", "$\\eta_1$", "$\\eta_2$"),
+    "DY_Z_Y": ("$y_Z$", "$\\M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"),
+    "SINGLETOP": ("$y$", "$m_t^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"),
+    "DY_MLL": ("$M_{ll} (GeV)$", "$M_{ll}^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"),
+    "DY_W_ETA": ("$abs_\\eta$", "$M_W^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"),
 }
 
 PROCESS_DESCRIPTION_LABEL = {
@@ -134,6 +135,8 @@ PROCESS_DESCRIPTION_LABEL = {
     "JET_POL": "Inclusive Jet longitudinal double-spin asymmetry",
     "DIJET_POL": "Dijets longitudinal double-spin asymmetry",
     "SHP_ASY": "double spin asymmetry in single hadron production",
+    "DY_MLL": "Drell-Yan Mass Distribution of Lepton Pairs",
+    "DY_W_ETA": "Drell-Yan W boson rapidity distribution",
 }
 
 
@@ -226,9 +229,9 @@ class TheoryMeta:
 
     Example
     -------
-    >>> from validphys.commondataparser import TheoryMeta
+    >>> from nnpdf_data.commondataparser import TheoryMeta
     ... from validobj import parse_input
-    ... from nnpdf_data.utils import yaml
+    ... from ruamel.yaml import YAML
     ... theory_raw = '''
     ... FK_tables:
     ...   - - fk1
@@ -236,7 +239,7 @@ class TheoryMeta:
     ...     - fk3
     ... operation: ratio
     ... '''
-    ... theory = yaml.safe_load(theory_raw)
+    ... theory = YAML(typ='safe').load(theory_raw)
     ... parse_input(theory, TheoryMeta)
     TheoryMeta(FK_tables=[['fk1'], ['fk2', 'fk3']], operation='RATIO', shifts = None, conversion_factor=1.0, comment=None, normalization=None))
     """
@@ -307,7 +310,7 @@ class Variant:
     experiment: Optional[str] = None
 
 
-ValidVariants = Dict[str, Variant]
+ValidVariants = dict[str, Variant]
 
 
 ### Kinematic data
@@ -349,7 +352,7 @@ class ValidKinematics:
     """
 
     file: ValidPath
-    variables: Dict[str, ValidVariable]
+    variables: dict[str, ValidVariable]
 
     def get_label(self, var):
         """For the given variable, return the label as label (unit)
@@ -439,10 +442,8 @@ class ObservableMetaData:
                 raise ValidationError(f"Missing `data_central` field for {self.name}")
 
             if not self.data_uncertainties:
-                ermsg = f"Missing `data_uncertainties` for {self.name}."
-                # be polite
-                if "legacy" in self.variants:
-                    ermsg += " Maybe you intended to use `variant: legacy`?"
+                ermsg = f"""Missing `data_uncertainties` for {self.name}.
+                    Select one of the variants: {list(self.variants.keys())}"""
                 raise ValidationError(ermsg)
 
         # Check that plotting.plot_x is being filled
@@ -468,8 +469,16 @@ class ObservableMetaData:
     def apply_variant(self, variant_name):
         """Return a new instance of this class with the variant applied
 
-        This class also defines how the variant is applied to the commondata
+        This class also defines how the variant is applied to the commondata.
+        If more than a variant is being used, this function will be called recursively
+        until all variants are applied.
         """
+        if not isinstance(variant_name, str):
+            observable = self
+            for single_variant in variant_name:
+                observable = observable.apply_variant(single_variant)
+            return observable
+
         try:
             variant = self.variants[variant_name]
         except KeyError as e:
@@ -485,7 +494,6 @@ class ObservableMetaData:
 
         # This section should only be used for the purposes of reproducibility
         # of legacy data, no new data should use these
-
         if variant.experiment is not None:
             new_nnpdf_metadata = dict(self._parent.nnpdf_metadata.items())
             new_nnpdf_metadata["experiment"] = variant.experiment
@@ -493,7 +501,13 @@ class ObservableMetaData:
             variant_replacement["_parent"] = setmetadata_copy
             variant_replacement["plotting"] = dataclasses.replace(self.plotting)
 
-        return dataclasses.replace(self, applied_variant=variant_name, **variant_replacement)
+        # Keep track of applied variants:
+        if self.applied_variant is None:
+            varname = variant_name
+        else:
+            varname = f"{self.applied_variant}_{variant_name}"
+
+        return dataclasses.replace(self, applied_variant=varname, **variant_replacement)
 
     @property
     def is_positivity(self):
@@ -553,7 +567,6 @@ class ObservableMetaData:
         all_df = []
         for ufile in self.paths_uncertainties:
             uncyaml = quick_yaml_load(ufile)
-
             mindex = pd.MultiIndex.from_tuples(
                 [(k, v["treatment"], v["type"]) for k, v in uncyaml["definitions"].items()],
                 names=["name", "treatment", "type"],
@@ -652,10 +665,6 @@ class ObservableMetaData:
         if self.ported_from is None:
             return False
 
-        # If it is using a legacy variant and has a ported_from field, then it is a ported one
-        if self.applied_variant is not None and self.applied_variant.startswith("legacy"):
-            return True
-
         # If not using a legacy variant, we consider it ported if the kin variables are still k1,k2,k3
         return {"k1", "k2", "k3"} == set(self.kinematic_coverage)
 
@@ -745,6 +754,18 @@ class ObservableMetaData:
                 new_line_by.append(self.digest_plotting_variable(var))
             self.plotting.line_by = new_line_by
 
+        # And do it also within the normalize dictionary
+        if self.plotting.normalize is not None:
+            # Copy the normalize dictionary and update the figure and line by
+            tmp = dict(self.plotting.normalize)
+            tmp["figure_by"] = []
+            tmp["line_by"] = []
+            for var in self.plotting.normalize.get("figure_by", []):
+                tmp["figure_by"].append(self.digest_plotting_variable(var))
+            for var in self.plotting.normalize.get("line_by", []):
+                tmp["line_by"].append(self.digest_plotting_variable(var))
+            self.plotting.normalize = tmp
+
         self.plotting.already_digested = True
         return self.plotting
 
@@ -818,22 +839,36 @@ class SetMetaData:
         try:
             observable = self.allowed_observables[obs_name]
         except KeyError:
-            raise ValueError(
-                f"The selected observable {obs_name_raw} does not exist in {self.setname}"
-            )
+            std_err = f"The selected observable {obs_name_raw} does not exist in {self.setname}"
+            # If the error is due to the usage of an old name, give some extra information
+            name = f"{self.setname}_{obs_name_raw}"
+            new_name, map_variant = legacy_to_new_map(name, None)
+            if new_name == name:
+                # It was just a mistake, raise the original error
+                raise ValueError(std_err)
+            if map_variant is None:
+                dinput_str = f"- {{ dataset: {new_name} }}"
+            else:
+                dinput_str = f"- {{ dataset: {new_name}, variant: {map_variant} }}"
+            std_err += f"""
+It might be that you used an older name for the dataset: '{name}'.
+If this is a mistake, please use '{new_name}' instead. E.g.,
+    {dinput_str}
+"""
+            raise ValueError(std_err)
 
         # Now burn the _parent key into the observable and apply checks
         object.__setattr__(observable, "_parent", self)
         return observable
 
 
-@lru_cache
+@cache
 def parse_set_metadata(metadata_file):
     """Read the metadata file"""
     return parse_yaml_inp(metadata_file, SetMetaData)
 
 
-@lru_cache
+@cache
 def parse_new_metadata(metadata_file, observable_name, variant=None):
     """Given a metadata file in the new format and the specific observable to be read
     load and parse the metadata and select the observable. If any variants are selected, apply them.
@@ -846,7 +881,7 @@ def parse_new_metadata(metadata_file, observable_name, variant=None):
     # Select one observable from the entire metadata
     metadata = set_metadata.select_observable(observable_name)
 
-    # And apply variant if given
+    # And apply variant or variants if given
     if variant is not None:
         metadata = metadata.apply_variant(variant)
 

nnpdf_data/coredata.py

@@ -1,13 +1,14 @@
 """
-    The all-important CommonData object
+The all-important CommonData object
 """
 
 import dataclasses
 from typing import Optional
 
+import numpy as np
 import pandas as pd
 
-from .utils import yaml
+from .utils import yaml_safe
 
 KIN_NAMES = ["kin1", "kin2", "kin3"]
 
@@ -198,7 +199,7 @@ class CommonData:
     def export_data(self, buffer):
         """Exports the central data defined by this commondata instance to the given buffer"""
         ret = {"data_central": self.central_values.tolist()}
-        yaml.safe_dump(ret, buffer)
+        yaml_safe.dump(ret, buffer)
 
     def export_uncertainties(self, buffer):
         """Exports the uncertainties defined by this commondata instance to the given buffer"""
@@ -213,6 +214,7 @@ class CommonData:
             k: v for k, v in sorted(definitions.items(), key=lambda item: item[1]["treatment"])
         }
         bins = []
+
         for idx, row in self.systematic_errors().iterrows():
             tmp = {"stat": float(self.stat_errors[idx])}
             # Hope things come in the right order...
@@ -227,7 +229,7 @@ class CommonData:
             "type": "UNCORR",
         }
         ret = {"definitions": sorted_definitions, "bins": bins}
-        yaml.safe_dump(ret, buffer)
+        yaml_safe.dump(ret, buffer)
 
     def export(self, folder_path):
         """Wrapper around export_data and export_uncertainties
@@ -240,6 +242,8 @@ class CommonData:
         data_path = folder_path / data_path.name
         unc_path = folder_path / unc_path.name
         # Export data and uncertainties
-        self.export_data(data_path.open("w", encoding="utf-8"))
-        self.export_uncertainties(unc_path.open("w", encoding="utf-8"))
+        with open(data_path, "w", encoding="utf-8") as file:
+            self.export_data(file)
+        with open(unc_path, "w", encoding="utf-8") as file:
+            self.export_uncertainties(file)
         return data_path, unc_path

nnpdf_data/filter_utils/hera_utils.py

@@ -0,0 +1,73 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List
+
+import numpy as np
+import yaml
+
+
+@dataclass
+class commondata:
+    central_values: np.ndarray
+    kinematics: np.ndarray
+    statistical_uncertainties: np.ndarray
+    systematic_uncertainties: np.ndarray
+    systypes: List[tuple[str, str]]
+    process: str
+    dataset_name: str
+    kinematic_quantities: List[str]
+
+    # Procedure to create data_*.yaml, kinematics_*.yaml and uncertainties_*.yaml
+    def write_new_commondata(
+        self, data_filename: Path, kinematics_filename: Path, uncertainties_filename: Path
+    ):
+        # central data values
+        data = {"data_central": self.central_values.tolist()}
+        with data_filename.open("w+") as f:
+            yaml.dump(data, f, default_flow_style=False, sort_keys=False)
+
+        # kinematic quantieties
+        # TODO add arrays for min and max values to derived type?
+        bins = []
+        for kin in self.kinematics.tolist():
+            bins.append(
+                {
+                    self.kinematic_quantities[0]: {"min": None, "mid": kin[0], "max": None},
+                    self.kinematic_quantities[1]: {"min": None, "mid": kin[1], "max": None},
+                    self.kinematic_quantities[2]: {"min": None, "mid": kin[2], "max": None},
+                }
+            )
+        data = {"bins": bins}
+        with kinematics_filename.open("w+") as f:
+            yaml.dump(data, f, default_flow_style=False, sort_keys=False)
+
+        # uncertainties
+        # There is only one statistical uncertainty per datapoint.
+        definitions = {
+            "stat": {
+                "description": "Statistical uncertainty.",
+                "treatment": "ADD",
+                "type": "UNCORR",
+            }
+        }
+        for isys, sys in enumerate(self.systypes):
+            definitions.update(
+                {
+                    f"sys_corr_{isys}": {
+                        "description": f"Systematic uncertainty {isys}",
+                        "treatment": sys[0],
+                        "type": sys[1],
+                    }
+                }
+            )
+        bins = {"bins": []}
+        for i, _ in enumerate(self.central_values):
+            systematics = {"stat": self.statistical_uncertainties.tolist()[i]}
+            for isys, sys in enumerate(self.systematic_uncertainties[i].tolist()):
+                systematics.update({f"sys_corr_{isys}": sys})
+            bins["bins"].append(systematics)
+        data = {"definitions": definitions}
+
+        with uncertainties_filename.open("w+") as f:
+            yaml.dump(data, f, sort_keys=False)
+            yaml.dump(bins, f, sort_keys=False)

nnpdf_data/filter_utils/nmc_hepdata_utils.py

@@ -0,0 +1,59 @@
+""""Common functions to parse NMC data from Hepdata."""
+
+import pandas as pd
+import yaml
+
+from .utils import check_xq2_degenearcy
+
+
+def read_tables(store_path, header_line):
+    """Parse Tables."""
+    dfs = pd.DataFrame()
+    for file in store_path.iterdir():
+        with open(file, "r", encoding="utf-8") as f:
+            lines = f.readlines()
+            df = pd.DataFrame(
+                [l.split(",") for l in lines[header_line:-1]],
+                columns=["Q2", "R", "F2", "stat+", "stat-", "sys+", "sys-"],
+            )
+            df["x"] = float(lines[header_line - 2].split(",")[1])
+            dfs = pd.concat([dfs, df], ignore_index=True) if not dfs.empty else df
+
+    dfs = dfs.astype(float)
+    check_xq2_degenearcy(dfs.Q2.values, dfs.x.values)
+    return dfs.sort_values(["x", "Q2"])
+
+
+def write_files(df, store_path):
+    """Write kinematics, central value and uncertainties files."""
+
+    # Write central data
+    data_central_yaml = {"data_central": [float(x) for x in df["F2"]]}
+    with open(store_path / "data_EM-F2-HEPDATA.yaml", "w", encoding="utf-8") as file:
+        yaml.dump(data_central_yaml, file)
+
+    # Write kin file
+    kin = []
+    for _, row in df.iterrows():
+        kin_value = {
+            "x": {"min": None, "mid": float(row.x), "max": None},
+            "Q2": {"min": None, "mid": float(row.Q2), "max": None},
+        }
+        kin.append(kin_value)
+    kinematics_yaml = {"bins": kin}
+    with open(store_path / "kinematics_EM-F2-HEPDATA.yaml", "w", encoding="utf-8") as file:
+        yaml.dump(kinematics_yaml, file, sort_keys=False)
+
+    # loop on data points
+    error_definition = {
+        "stat": {"description": "statistical uncertainty", "treatment": "ADD", "type": "UNCORR"},
+        "sys": {"description": "systematical uncertainty", "treatment": "MULT", "type": "CORR"},
+    }
+    error = []
+    for _, row in df.iterrows():
+        e = {"stat": float(row["stat+"]), "sys": float(row["sys+"])}
+        error.append(e)
+
+    uncertainties_yaml = {"definitions": error_definition, "bins": error}
+    with open(store_path / "uncertainties_EM-F2-HEPDATA.yaml", "w", encoding="utf-8") as file:
+        yaml.dump(uncertainties_yaml, file, sort_keys=False)

nnpdf_data/filter_utils/utils.py

@@ -162,7 +162,7 @@ def covmat_to_artunc(ndata, covmat_list, no_of_norm_mat=0):
         elif eigval[j] > 0:
             continue
     if not psd_check:
-        raise ValueError('The covariance matrix is not positive-semidefinite')
+        raise ValueError("The covariance matrix is not positive-semidefinite")
     else:
         for i in range(ndata):
             for j in range(ndata):
@@ -244,7 +244,7 @@ def matlist_to_matrix(rows, columns, mat_list):
         matrix = np.array(matrix)
         return matrix
     else:
-        raise Exception('rows * columns != len(mat_list)')
+        raise Exception("rows * columns != len(mat_list)")
 
 
 def concat_matrices(rows, columns, list_of_matrices):
@@ -363,7 +363,7 @@ def trimat_to_fullmat(mode, tri_mat_list):
                     matrix[i][j] = tri_mat_list[list_el]
                     matrix[j][i] = tri_mat_list[list_el]
     else:
-        raise Exception('Mode should be 0 or 1, refer to the function for usage')
+        raise Exception("Mode should be 0 or 1, refer to the function for usage")
     mat_list = []
     for i in range(dim):
         for j in range(dim):
@@ -422,10 +422,22 @@ def prettify_float(dumper, value):
     ret = dumper.represent_float(value)
     if len(ret.value) > 8:
         ret_str = f"{value:.8e}"
-        ret = dumper.represent_scalar('tag:yaml.org,2002:float', ret_str)
+        ret = dumper.represent_scalar("tag:yaml.org,2002:float", ret_str)
     return ret
 
 
+def check_xq2_degenearcy(Q2, x):
+    """Check is the pair of (x,Q2) is unique."""
+    size = len(x)
+    unique_pairs = np.unique(np.array([Q2, x]), axis=1)
+    try:
+        assert unique_pairs.shape[1] == size
+    except AssertionError as exc:
+        raise ValueError(
+            f"""(x,Q2) kinematic is degenerate need to store 3rd kinematic variable as well.
+            unique kinematics are: {unique_pairs.shape[1]}, original size: {size}"""
+        ) from exc
+
 def uncert_skip_variant(source_file, skip_file, uncert_file, uncert_name, remove_source=True):
     r"""
     Create two new uncertainty files, one where the specified uncertainty