correctionlib 2.4.0__tar.gz → 2.6.0__tar.gz

correctionlib-2.6.0/.github/workflows/ci.yml

@@ -0,0 +1,83 @@
+name: CI
+
+on:
+  workflow_dispatch:
+  pull_request:
+  merge_group:
+  push:
+    branches:
+    - master
+    - main
+    - develop
+
+jobs:
+  pre-commit:
+    name: Format
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - uses: actions/setup-python@v3
+      with:
+        python-version: 3.11.3
+    - uses: pre-commit/action@v3.0.0
+      with:
+        extra_args: --hook-stage manual --all-files
+
+  checks:
+    name: Check Python ${{ matrix.python-version }} on ${{ matrix.runs-on }}
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7", "3.11", "3.12"]
+        runs-on: [ubuntu-latest, macos-13, windows-latest]
+
+        include:
+        - python-version: "3.12"
+          runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install package
+      run: python -m pip install .[test]
+
+    - name: Test package
+      run: python -m pytest -ra
+
+  checkroot:
+    name: Check ROOT bindings
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up Conda
+        uses: conda-incubator/setup-miniconda@v3.0.4
+        with:
+          python-version: 3.11
+          miniforge-variant: Mambaforge
+          channels: conda-forge,defaults
+          channel-priority: true
+
+      - name: Install ROOT
+        shell: bash -l {0}
+        run: |
+          mamba install root
+
+      - name: Install package
+        shell: bash -l {0}
+        run: python -m pip install .[test]
+
+      - name: Test package
+        shell: bash -l {0}
+        run: python -m pytest -ra

correctionlib-2.6.0/.github/workflows/docs.yml

@@ -0,0 +1,40 @@
+name: Docs
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - master
+    - main
+
+jobs:
+  docs:
+    name: Build and deploy docs
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+
+    - name: Install pandoc
+      uses: r-lib/actions/setup-pandoc@v2
+
+    - name: Install package
+      run: python -m pip install .[docs]
+
+    - name: Build documentation
+      run: |
+        cd docs
+        make
+        touch build/html/.nojekyll
+
+    - name: Deploy documentation
+      uses: peaceiris/actions-gh-pages@v4.0.0
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./docs/build/html

correctionlib-2.6.0/.github/workflows/wheels.yml

@@ -0,0 +1,68 @@
+name: wheels
+
+on:
+  workflow_dispatch:
+  release:
+    types:
+      - published
+  pull_request:
+    paths:
+      - ".github/workflows/wheels.yml"
+
+jobs:
+  make_sdist:
+    name: Make SDist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        fetch-tags: true
+        fetch-depth: 0
+
+    - name: Build SDist
+      run: pipx run build --sdist
+
+    - uses: actions/upload-artifact@v2
+      with:
+        path: dist/*.tar.gz
+
+  build_wheels:
+    name: Wheel on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        fetch-tags: true
+        fetch-depth: 0
+
+    - uses: pypa/cibuildwheel@v2.19.1
+      env:
+        MACOSX_DEPLOYMENT_TARGET: 10.14
+
+    - name: Upload wheels
+      uses: actions/upload-artifact@v2
+      with:
+        path: wheelhouse/*.whl
+
+  upload_all:
+    needs: [build_wheels, make_sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release' && github.event.action == 'published'
+
+    steps:
+    - uses: actions/download-artifact@v2
+      with:
+        name: artifact
+        path: dist
+
+    - uses: pypa/gh-action-pypi-publish@v1.9.0
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_token }}

correctionlib-2.6.0/.pre-commit-config.yaml

@@ -0,0 +1,44 @@
+repos:
+- repo: https://github.com/psf/black
+  rev: 22.3.0
+  hooks:
+  - id: black
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.1.0
+  hooks:
+  - id: check-added-large-files
+  - id: check-case-conflict
+  - id: check-merge-conflict
+  - id: check-symlinks
+  - id: check-yaml
+  - id: debug-statements
+  - id: end-of-file-fixer
+  - id: mixed-line-ending
+  - id: requirements-txt-fixer
+  - id: trailing-whitespace
+
+- repo: https://github.com/PyCQA/isort
+  rev: 5.12.0
+  hooks:
+  - id: isort
+
+- repo: https://github.com/asottile/pyupgrade
+  rev: v2.31.0
+  hooks:
+  - id: pyupgrade
+    args: ["--py36-plus"]
+
+- repo: https://github.com/pycqa/flake8
+  rev: 7.0.0
+  hooks:
+  - id: flake8
+    exclude: docs/conf.py
+    additional_dependencies: [flake8-bugbear, flake8-print]
+
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.4.1
+  hooks:
+  - id: mypy
+    files: src
+    additional_dependencies: [uhi, pydantic, numpy, types-setuptools]

correctionlib-2.6.0/PKG-INFO

@@ -0,0 +1,150 @@
+Metadata-Version: 2.1
+Name: correctionlib
+Version: 2.6.0
+Summary: A generic correction library
+Home-page: https://github.com/cms-nanoAOD/correctionlib
+Author: Nick Smith
+Author-email: nick.smith@cern.ch
+Maintainer: Nick Smith
+Maintainer-email: nick.smith@cern.ch
+License: BSD 3-Clause License
+Platform: Any
+Classifier: Topic :: Scientific/Engineering
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Development Status :: 5 - Production/Stable
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.13.3
+Requires-Dist: typing-extensions; python_version < "3.8"
+Requires-Dist: dataclasses; python_version < "3.7"
+Requires-Dist: pydantic>=2
+Requires-Dist: rich
+Requires-Dist: packaging
+Provides-Extra: test
+Requires-Dist: pytest>=4.6; extra == "test"
+Requires-Dist: scipy; extra == "test"
+Requires-Dist: awkward>=2.2.2; python_version > "3.7" and extra == "test"
+Requires-Dist: awkward<2; python_version <= "3.7" and extra == "test"
+Requires-Dist: dask-awkward>=2024.1.1; python_version > "3.7" and extra == "test"
+Provides-Extra: dev
+Requires-Dist: pytest>=4.6; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: uhi; extra == "dev"
+Requires-Dist: awkward>=2; extra == "dev"
+Requires-Dist: dask-awkward; python_version > "3.7" and extra == "dev"
+Provides-Extra: docs
+Requires-Dist: recommonmark>=0.5.0; extra == "docs"
+Requires-Dist: Sphinx>=3.0.0; extra == "docs"
+Requires-Dist: sphinx_copybutton; extra == "docs"
+Requires-Dist: sphinx_rtd_theme>=0.5.0; extra == "docs"
+Requires-Dist: sphinx-jsonschema; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
+Requires-Dist: nbsphinx; extra == "docs"
+Provides-Extra: convert
+Requires-Dist: pandas; extra == "convert"
+Requires-Dist: uproot>=4.0.4; extra == "convert"
+Requires-Dist: requests; extra == "convert"
+Requires-Dist: scipy; extra == "convert"
+
+# correctionlib
+
+[![Actions Status][actions-badge]][actions-link]
+[![Documentation Status][rtd-badge]][rtd-link]
+[![Code style: black][black-badge]][black-link]
+
+[![conda version][conda-badge]][conda-link]
+[![PyPI version][pypi-version]][pypi-link]
+[![PyPI platforms][pypi-platforms]][pypi-link]
+
+[![GitHub Discussion][github-discussions-badge]][github-discussions-link]
+
+## Introduction
+The purpose of this library is to provide a well-structured JSON data format for a
+wide variety of ad-hoc correction factors encountered in a typical HEP analysis and
+a companion evaluation tool suitable for use in C++ and python programs.
+Here we restrict our definition of correction factors to a class of functions with
+scalar inputs that produce a scalar output.
+
+In python, the function signature is:
+
+```python
+def f(*args: str | int | float) -> float:
+    return ...
+```
+
+In C++, the evaluator implements this currently as:
+```cpp
+double Correction::evaluate(const std::vector<std::variant<int, double, std::string>>& values) const;
+```
+
+The supported function classes include:
+
+  * multi-dimensional binned lookups;
+  * binned lookups pointing to multi-argument formulas with a restricted
+    math function set (`exp`, `sqrt`, etc.);
+  * categorical (string or integer enumeration) maps;
+  * input transforms (updating one input value in place); and
+  * compositions of the above.
+
+Each function type is represented by a "node" in a call graph and holds all
+of its parameters in a JSON structure, described by the JSON schema.
+Possible future extension nodes might include weigted sums (which, when composed with
+the others, could represent a BDT) and perhaps simple MLPs.
+
+The tool should provide:
+
+  * standardized, versioned [JSON schemas](https://json-schema.org/);
+  * forward-porting tools (to migrate data written in older schema versions); and
+  * a well-optimized C++ evaluator and python bindings (with numpy vectorization support).
+
+This tool will definitely not provide:
+
+  * support for `TLorentzVector` or other object-type inputs (such tools should be written
+    as a higher-level tool depending on this library as a low-level tool)
+
+Formula support currently includes a mostly-complete subset of the ROOT library `TFormula` class,
+and is implemented in a threadsafe standalone manner. The parsing grammar is formally defined
+and parsed through the use of a header-only [PEG parser library](https://github.com/yhirose/cpp-peglib).
+The supported features mirror CMSSW's [reco::formulaEvaluator](https://github.com/cms-sw/cmssw/pull/11516)
+and fully passes the test suite for that utility with the purposeful exception of the `TMath::` namespace.
+The python bindings may be able to call into [numexpr](https://numexpr.readthedocs.io/en/latest/user_guide.html),
+though, due to the tree-like structure of the corrections, it may prove difficult to exploit vectorization
+at levels other than the entrypoint.
+
+Detailed instructions for installing and using this package are provided in the [documentation][rtd-link].
+
+## Creating new corrections
+
+A demo/tutorial of the features is available in the [documentation][rtd-link] and also available interactively
+on [binder](https://mybinder.org/v2/gh/cms-nanoAOD/correctionlib/HEAD?labpath=binder%2Fcorrectionlib_tutorial.ipynb)
+
+The `correctionlib.schemav2` module provides a helpful framework for defining correction objects
+and `correctionlib.convert` includes select conversion routines for common types. Nodes can be type-checked as they are
+constructed using the [parse_obj](https://pydantic-docs.helpmanual.io/usage/models/#helper-functions)
+class method or by directly constructing them using keyword arguments.
+
+## Developing
+See CONTRIBUTING.md
+
+[actions-badge]:            https://github.com/cms-nanoAOD/correctionlib/workflows/CI/badge.svg
+[actions-link]:             https://github.com/cms-nanoAOD/correctionlib/actions
+[black-badge]:              https://img.shields.io/badge/code%20style-black-000000.svg
+[black-link]:               https://github.com/psf/black
+[conda-badge]:              https://img.shields.io/conda/vn/conda-forge/correctionlib.svg
+[conda-link]:               https://github.com/conda-forge/correctionlib-feedstock
+[github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
+[github-discussions-link]:  https://github.com/cms-nanoAOD/correctionlib/discussions
+[gitter-badge]:             https://badges.gitter.im/https://github.com/cms-nanoAOD/correctionlib/community.svg
+[gitter-link]:              https://gitter.im/https://github.com/cms-nanoAOD/correctionlib/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge
+[pypi-link]:                https://pypi.org/project/correctionlib/
+[pypi-platforms]:           https://img.shields.io/pypi/pyversions/correctionlib
+[pypi-version]:             https://badge.fury.io/py/correctionlib.svg
+[rtd-badge]:                https://github.com/cms-nanoAOD/correctionlib/actions/workflows/docs.yml/badge.svg
+[rtd-link]:                 https://cms-nanoAOD.github.io/correctionlib/

correctionlib-2.6.0/data/conversion.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python
+import gzip
+
+import pandas
+import requests
+
+from correctionlib import convert
+from correctionlib.schemav2 import (
+    VERSION,
+    Binning,
+    Category,
+    Correction,
+    CorrectionSet,
+    Formula,
+)
+
+examples = "https://raw.githubusercontent.com/CoffeaTeam/coffea/master/tests/samples"
+
+# `sf` is a binned event weight to be applied as a function of $\eta$ and $p_T$
+corr1 = convert.from_uproot_THx(
+    f"{examples}/testSF2d.histo.root:scalefactors_Tight_Electron"
+)
+
+
+sf = pandas.read_csv(
+    f"{examples}/DeepCSV_2016LegacySF_V1_TuneCP5.btag.csv.gz", skipinitialspace=True
+)
+
+
+def build_formula(sf):
+    if len(sf) != 1:
+        raise ValueError(sf)
+
+    value = sf.iloc[0]["formula"]
+    if "x" in value:
+        return Formula.model_validate(
+            {
+                "nodetype": "formula",
+                "expression": value,
+                "parser": "TFormula",
+                # For this case, since this is a "reshape" SF, we know the parameter is the discriminant
+                "variables": ["discriminant"],
+                "parameters": [],
+            }
+        )
+    else:
+        return float(value)
+
+
+def build_discrbinning(sf):
+    edges = sorted(set(sf["discrMin"]) | set(sf["discrMax"]))
+    return Binning.model_validate(
+        {
+            "nodetype": "binning",
+            "input": "discriminant",
+            "edges": edges,
+            "content": [
+                build_formula(sf[(sf["discrMin"] >= lo) & (sf["discrMax"] <= hi)])
+                for lo, hi in zip(edges[:-1], edges[1:])
+            ],
+            "flow": "clamp",
+        }
+    )
+
+
+def build_ptbinning(sf):
+    edges = sorted(set(sf["ptMin"]) | set(sf["ptMax"]))
+    return Binning.model_validate(
+        {
+            "nodetype": "binning",
+            "input": "pt",
+            "edges": edges,
+            "content": [
+                build_discrbinning(sf[(sf["ptMin"] >= lo) & (sf["ptMax"] <= hi)])
+                for lo, hi in zip(edges[:-1], edges[1:])
+            ],
+            "flow": "clamp",
+        }
+    )
+
+
+def build_etabinning(sf):
+    edges = sorted(set(sf["etaMin"]) | set(sf["etaMax"]))
+    return Binning.model_validate(
+        {
+            "nodetype": "binning",
+            "input": "abseta",
+            "edges": edges,
+            "content": [
+                build_ptbinning(sf[(sf["etaMin"] >= lo) & (sf["etaMax"] <= hi)])
+                for lo, hi in zip(edges[:-1], edges[1:])
+            ],
+            "flow": "error",
+        }
+    )
+
+
+def build_flavor(sf):
+    keys = sorted(sf["jetFlavor"].unique())
+    return Category.model_validate(
+        {
+            "nodetype": "category",
+            "input": "flavor",
+            "content": [
+                {"key": key, "value": build_etabinning(sf[sf["jetFlavor"] == key])}
+                for key in keys
+            ],
+        }
+    )
+
+
+def build_systs(sf):
+    keys = list(sf["sysType"].unique())
+    return Category.model_validate(
+        {
+            "nodetype": "category",
+            "input": "systematic",
+            "content": [
+                {"key": key, "value": build_flavor(sf[sf["sysType"] == key])}
+                for key in keys
+            ],
+        }
+    )
+
+
+corr2 = Correction.model_validate(
+    {
+        "version": 1,
+        "name": "DeepCSV_2016LegacySF",
+        "description": "A btagging scale factor",
+        "inputs": [
+            {"name": "systematic", "type": "string"},
+            {
+                "name": "flavor",
+                "type": "int",
+                "description": "BTV flavor definiton: 0=b, 1=c, 2=udsg",
+            },
+            {"name": "abseta", "type": "real"},
+            {"name": "pt", "type": "real"},
+            {
+                "name": "discriminant",
+                "type": "real",
+                "description": "DeepCSV output value",
+            },
+        ],
+        "output": {"name": "weight", "type": "real"},
+        "data": build_systs(sf),
+    }
+)
+
+
+sf = requests.get(f"{examples}/EIDISO_WH_out.histo.json").model_dump_json()
+
+
+def build_syst(sf):
+    return Category.model_validate(
+        {
+            "nodetype": "category",
+            "input": "systematic",
+            "content": [
+                {"key": "nominal", "value": sf["value"]},
+                {"key": "up", "value": sf["value"] + sf["error"]},
+                {"key": "down", "value": sf["value"] - sf["error"]},
+            ],
+        }
+    )
+
+
+def parse_str(key, prefix="eta:"):
+    if not key.startswith(prefix + "["):
+        raise ValueError(f"{key} missing prefix {prefix}")
+    lo, hi = map(float, key[len(prefix + "[") : -1].split(","))
+    return lo, hi
+
+
+def build_pts(sf):
+    edges = []
+    content = []
+    for binstr, data in sf.items():
+        if not binstr.startswith("pt:["):
+            raise ValueError
+        lo, hi = map(float, binstr[len("pt:[") : -1].split(","))
+        if len(edges) == 0:
+            edges.append(lo)
+        if edges[-1] != lo:
+            raise ValueError
+        edges.append(hi)
+        content.append(build_syst(data))
+
+    return Binning.model_validate(
+        {
+            "nodetype": "binning",
+            "input": "pt",
+            "edges": edges,
+            "content": content,
+            "flow": "clamp",
+        }
+    )
+
+
+def build_etas(sf):
+    bins = [parse_str(s, "eta:") for s in sf]
+    edges = sorted({edge for bin in bins for edge in bin})
+    content = [None] * (len(edges) - 1)
+    for s, data in sf.items():
+        lo, hi = parse_str(s, "eta:")
+        found = False
+        for i, bin in enumerate(bins):
+            if bin[0] >= lo and bin[1] <= hi:
+                content[i] = build_pts(data)
+                found = True
+        if not found:
+            raise ValueError("eta edges not in binning?")
+
+    return Binning.model_validate(
+        {
+            "nodetype": "binning",
+            "input": "eta",
+            "edges": edges,
+            "content": content,
+            "flow": "error",
+        }
+    )
+
+
+corr3 = Correction.model_validate(
+    {
+        "version": 1,
+        "name": "EIDISO_WH_out",
+        "description": "An electron scale factor",
+        "inputs": [
+            {"name": "eta", "type": "real"},
+            {"name": "pt", "type": "real"},
+            {"name": "systematic", "type": "string"},
+        ],
+        "output": {"name": "weight", "type": "real"},
+        "data": build_etas(sf["EIDISO_WH"]["eta_pt_ratio"]),
+    }
+)
+
+
+cset = CorrectionSet.model_validate(
+    {
+        "schema_version": VERSION,
+        "corrections": [
+            corr1,
+            corr2,
+            corr3,
+        ],
+    }
+)
+
+with gzip.open("data/examples.json.gz", "wt") as fout:
+    fout.write(cset.model_dump_json(exclude_unset=True))