atlas-schema 0.2.2__tar.gz → 0.2.4__tar.gz

{atlas_schema-0.2.2 → atlas_schema-0.2.4}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: atlas-schema
-Version: 0.2.2
+Version: 0.2.4
 Summary: Helper python package for ATLAS Common NTuple Analysis work.
 Project-URL: Homepage, https://github.com/scipp-atlas/atlas-schema
 Project-URL: Bug Tracker, https://github.com/scipp-atlas/atlas-schema/issues
 Project-URL: Discussions, https://github.com/scipp-atlas/atlas-schema/discussions
-Project-URL: Documentation, https://atlas-schema.readthedocs.io/en/v0.2.2/
+Project-URL: Documentation, https://atlas-schema.readthedocs.io/en/v0.2.4/
 Project-URL: Releases, https://github.com/scipp-atlas/atlas-schema/releases
 Project-URL: Release Notes, https://atlas-schema.readthedocs.io/en/latest/history.html
 Author-email: Giordon Stark <kratsg@gmail.com>
@@ -251,7 +251,7 @@ Requires-Dist: tbump>=6.7.0; extra == 'test'
 Requires-Dist: twine; extra == 'test'
 Description-Content-Type: text/markdown
 
-# atlas-schema v0.2.2
+# atlas-schema v0.2.4
 
 [![Actions Status][actions-badge]][actions-link]
 [![Documentation Status][rtd-badge]][rtd-link]
@@ -279,6 +279,129 @@ Description-Content-Type: text/markdown
 
 <!-- prettier-ignore-end -->
 
+This is the python package containing schemas and helper functions enabling
+analyzers to work with ATLAS datasets (Monte Carlo and Data), using
+[coffea](https://coffea-hep.readthedocs.io/en/latest/).
+
+## Hello World
+
+The simplest example is to just get started processing the file as expected:
+
+```python
+from atlas_schema.schema import NtupleSchema
+from coffea import dataset_tools
+import awkward as ak
+
+fileset = {"ttbar": {"files": {"path/to/ttbar.root": "tree_name"}}}
+samples, report = dataset_tools.preprocess(fileset)
+
+
+def noop(events):
+    return ak.fields(events)
+
+
+fields = dataset_tools.apply_to_fileset(noop, samples, schemaclass=NtupleSchema)
+print(fields)
+```
+
+which produces something similar to
+
+```python
+{
+    "ttbar": [
+        "dataTakingYear",
+        "mcChannelNumber",
+        "runNumber",
+        "eventNumber",
+        "lumiBlock",
+        "actualInteractionsPerCrossing",
+        "averageInteractionsPerCrossing",
+        "truthjet",
+        "PileupWeight",
+        "RandomRunNumber",
+        "met",
+        "recojet",
+        "truth",
+        "generatorWeight",
+        "beamSpotWeight",
+        "trigPassed",
+        "jvt",
+    ]
+}
+```
+
+However, a more involved example to apply a selection and fill a histogram looks
+like below:
+
+```python
+import awkward as ak
+import dask
+import hist.dask as had
+import matplotlib.pyplot as plt
+from coffea import processor
+from coffea.nanoevents import NanoEventsFactory
+from distributed import Client
+
+from atlas_schema.schema import NtupleSchema
+
+
+class MyFirstProcessor(processor.ProcessorABC):
+    def __init__(self):
+        pass
+
+    def process(self, events):
+        dataset = events.metadata["dataset"]
+        h_ph_pt = (
+            had.Hist.new.StrCat(["all", "pass", "fail"], name="isEM")
+            .Regular(200, 0.0, 2000.0, name="pt", label="$pt_{\gamma}$ [GeV]")
+            .Int64()
+        )
+
+        cut = ak.all(events.ph.isEM, axis=1)
+        h_ph_pt.fill(isEM="all", pt=ak.firsts(events.ph.pt / 1.0e3))
+        h_ph_pt.fill(isEM="pass", pt=ak.firsts(events[cut].ph.pt / 1.0e3))
+        h_ph_pt.fill(isEM="fail", pt=ak.firsts(events[~cut].ph.pt / 1.0e3))
+
+        return {
+            dataset: {
+                "entries": ak.num(events, axis=0),
+                "ph_pt": h_ph_pt,
+            }
+        }
+
+    def postprocess(self, accumulator):
+        pass
+
+
+if __name__ == "__main__":
+    client = Client()
+
+    fname = "ntuple.root"
+    events = NanoEventsFactory.from_root(
+        {fname: "analysis"},
+        schemaclass=NtupleSchema,
+        metadata={"dataset": "700352.Zqqgamma.mc20d.v1"},
+    ).events()
+
+    p = MyFirstProcessor()
+    out = p.process(events)
+    (computed,) = dask.compute(out)
+    print(computed)
+
+    fig, ax = plt.subplots()
+    computed["700352.Zqqgamma.mc20d.v1"]["ph_pt"].plot1d(ax=ax)
+    ax.set_xscale("log")
+    ax.legend(title="Photon pT for Zqqgamma")
+
+    fig.savefig("ph_pt.pdf")
+```
+
+which produces
+
+<img src="https://raw.githubusercontent.com/scipp-atlas/atlas-schema/main/docs/_static/img/ph_pt.png" alt="three stacked histograms of photon pT, with each stack corresponding to: no selection, requiring the isEM flag, and inverting the isEM requirement" width="500" style="display: block; margin-left: auto; margin-right: auto;">
+
+<!-- SPHINX-END -->
+
 ## Developer Notes
 
 ### Converting Enums from C++ to Python

atlas_schema-0.2.4/README.md

@@ -0,0 +1,160 @@
+# atlas-schema v0.2.4
+
+[![Actions Status][actions-badge]][actions-link]
+[![Documentation Status][rtd-badge]][rtd-link]
+
+[![PyPI version][pypi-version]][pypi-link]
+[![Conda-Forge][conda-badge]][conda-link]
+[![PyPI platforms][pypi-platforms]][pypi-link]
+
+[![GitHub Discussion][github-discussions-badge]][github-discussions-link]
+
+<!-- SPHINX-START -->
+
+<!-- prettier-ignore-start -->
+[actions-badge]:            https://github.com/scipp-atlas/atlas-schema/workflows/CI/badge.svg
+[actions-link]:             https://github.com/scipp-atlas/atlas-schema/actions
+[conda-badge]:              https://img.shields.io/conda/vn/conda-forge/atlas-schema
+[conda-link]:               https://github.com/conda-forge/atlas-schema-feedstock
+[github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
+[github-discussions-link]:  https://github.com/scipp-atlas/atlas-schema/discussions
+[pypi-link]:                https://pypi.org/project/atlas-schema/
+[pypi-platforms]:           https://img.shields.io/pypi/pyversions/atlas-schema
+[pypi-version]:             https://img.shields.io/pypi/v/atlas-schema
+[rtd-badge]:                https://readthedocs.org/projects/atlas-schema/badge/?version=latest
+[rtd-link]:                 https://atlas-schema.readthedocs.io/en/latest/?badge=latest
+
+<!-- prettier-ignore-end -->
+
+This is the python package containing schemas and helper functions enabling
+analyzers to work with ATLAS datasets (Monte Carlo and Data), using
+[coffea](https://coffea-hep.readthedocs.io/en/latest/).
+
+## Hello World
+
+The simplest example is to just get started processing the file as expected:
+
+```python
+from atlas_schema.schema import NtupleSchema
+from coffea import dataset_tools
+import awkward as ak
+
+fileset = {"ttbar": {"files": {"path/to/ttbar.root": "tree_name"}}}
+samples, report = dataset_tools.preprocess(fileset)
+
+
+def noop(events):
+    return ak.fields(events)
+
+
+fields = dataset_tools.apply_to_fileset(noop, samples, schemaclass=NtupleSchema)
+print(fields)
+```
+
+which produces something similar to
+
+```python
+{
+    "ttbar": [
+        "dataTakingYear",
+        "mcChannelNumber",
+        "runNumber",
+        "eventNumber",
+        "lumiBlock",
+        "actualInteractionsPerCrossing",
+        "averageInteractionsPerCrossing",
+        "truthjet",
+        "PileupWeight",
+        "RandomRunNumber",
+        "met",
+        "recojet",
+        "truth",
+        "generatorWeight",
+        "beamSpotWeight",
+        "trigPassed",
+        "jvt",
+    ]
+}
+```
+
+However, a more involved example to apply a selection and fill a histogram looks
+like below:
+
+```python
+import awkward as ak
+import dask
+import hist.dask as had
+import matplotlib.pyplot as plt
+from coffea import processor
+from coffea.nanoevents import NanoEventsFactory
+from distributed import Client
+
+from atlas_schema.schema import NtupleSchema
+
+
+class MyFirstProcessor(processor.ProcessorABC):
+    def __init__(self):
+        pass
+
+    def process(self, events):
+        dataset = events.metadata["dataset"]
+        h_ph_pt = (
+            had.Hist.new.StrCat(["all", "pass", "fail"], name="isEM")
+            .Regular(200, 0.0, 2000.0, name="pt", label="$pt_{\gamma}$ [GeV]")
+            .Int64()
+        )
+
+        cut = ak.all(events.ph.isEM, axis=1)
+        h_ph_pt.fill(isEM="all", pt=ak.firsts(events.ph.pt / 1.0e3))
+        h_ph_pt.fill(isEM="pass", pt=ak.firsts(events[cut].ph.pt / 1.0e3))
+        h_ph_pt.fill(isEM="fail", pt=ak.firsts(events[~cut].ph.pt / 1.0e3))
+
+        return {
+            dataset: {
+                "entries": ak.num(events, axis=0),
+                "ph_pt": h_ph_pt,
+            }
+        }
+
+    def postprocess(self, accumulator):
+        pass
+
+
+if __name__ == "__main__":
+    client = Client()
+
+    fname = "ntuple.root"
+    events = NanoEventsFactory.from_root(
+        {fname: "analysis"},
+        schemaclass=NtupleSchema,
+        metadata={"dataset": "700352.Zqqgamma.mc20d.v1"},
+    ).events()
+
+    p = MyFirstProcessor()
+    out = p.process(events)
+    (computed,) = dask.compute(out)
+    print(computed)
+
+    fig, ax = plt.subplots()
+    computed["700352.Zqqgamma.mc20d.v1"]["ph_pt"].plot1d(ax=ax)
+    ax.set_xscale("log")
+    ax.legend(title="Photon pT for Zqqgamma")
+
+    fig.savefig("ph_pt.pdf")
+```
+
+which produces
+
+<img src="https://raw.githubusercontent.com/scipp-atlas/atlas-schema/main/docs/_static/img/ph_pt.png" alt="three stacked histograms of photon pT, with each stack corresponding to: no selection, requiring the isEM flag, and inverting the isEM requirement" width="500" style="display: block; margin-left: auto; margin-right: auto;">
+
+<!-- SPHINX-END -->
+
+## Developer Notes
+
+### Converting Enums from C++ to Python
+
+This useful `vim` substitution helps:
+
+```
+%s/    \([A-Za-z]\+\)\s\+=  \(\d\+\),\?/    \1: Annotated[int, "\1"] = \2
+```

{atlas_schema-0.2.2 → atlas_schema-0.2.4}/pyproject.toml

@@ -60,7 +60,7 @@ docs = [
 Homepage = "https://github.com/scipp-atlas/atlas-schema"
 "Bug Tracker" = "https://github.com/scipp-atlas/atlas-schema/issues"
 Discussions = "https://github.com/scipp-atlas/atlas-schema/discussions"
-Documentation = "https://atlas-schema.readthedocs.io/en/v0.2.2/"
+Documentation = "https://atlas-schema.readthedocs.io/en/v0.2.4/"
 Releases = "https://github.com/scipp-atlas/atlas-schema/releases"
 "Release Notes" = "https://atlas-schema.readthedocs.io/en/latest/history.html"
 
@@ -111,13 +111,18 @@ addopts = [
 ]
 xfail_strict = true
 filterwarnings = [
-  "error",
+    "error",
 ]
+
 log_cli_level = "INFO"
 testpaths = [
+  "src",
   "tests",
   "docs",
 ]
+norecursedirs = [
+  "tests/helpers"
+]
 
 [tool.coverage]
 run.source = ["atlas_schema"]

{atlas_schema-0.2.2 → atlas_schema-0.2.4}/src/atlas_schema/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.2'
-__version_tuple__ = version_tuple = (0, 2, 2)
+__version__ = version = '0.2.4'
+__version_tuple__ = version_tuple = (0, 2, 4)

{atlas_schema-0.2.2 → atlas_schema-0.2.4}/src/atlas_schema/methods.py

@@ -230,12 +230,24 @@ JetArray.MomentumClass = vector.LorentzVectorArray  # noqa: F821
 
 __all__ = [
     "Electron",
+    "ElectronArray",  # noqa: F822
+    "ElectronRecord",  # noqa: F822
     "Jet",
+    "JetArray",  # noqa: F822
+    "JetRecord",  # noqa: F822
     "MissingET",
+    "MissingETArray",  # noqa: F822
+    "MissingETRecord",  # noqa: F822
     "Muon",
+    "MuonArray",  # noqa: F822
+    "MuonRecord",  # noqa: F822
     "NtupleEvents",
     "Particle",
+    "ParticleArray",  # noqa: F822
+    "ParticleRecord",  # noqa: F822
     "Pass",
     "Photon",
+    "PhotonArray",  # noqa: F822
+    "PhotonRecord",  # noqa: F822
     "Weight",
 ]

atlas_schema-0.2.4/src/atlas_schema/schema.py

@@ -0,0 +1,405 @@
+from __future__ import annotations
+
+import difflib
+import warnings
+from collections.abc import KeysView, ValuesView
+from typing import Any, ClassVar
+
+from coffea.nanoevents.schemas.base import BaseSchema, zip_forms
+
+from atlas_schema.typing_compat import Behavior, Self
+
+
+class NtupleSchema(BaseSchema):  # type: ignore[misc]
+    """The schema for building ATLAS ntuples following the typical centralized formats.
+
+    This schema is built from all branches found in a tree in the supplied
+    file, based on the naming pattern of the branches. This naming pattern is
+    typically assumed to be
+
+    .. code-block:: bash
+
+       {collection:str}_{subcollection:str}_{systematic:str}
+
+    where:
+      * ``collection`` is assumed to be a prefix with typical characters, following the regex ``[a-zA-Z][a-zA-Z0-9]*``; that is starting with a case-insensitive letter, and proceeded by zero or more alphanumeric characters,
+      * ``subcollection`` is assumed to be anything with typical characters (allowing for underscores) following the regex ``[a-zA-Z_][a-zA-Z0-9_]*``; that is starting with a case-insensitive letter or underscore, and proceeded by zero or more alphanumeric characters including underscores, and
+      * ``systematic`` is assumed to be either ``NOSYS`` to indicate a branch with potential systematic variariations, or anything with typical characters (allowing for underscores) following the same regular expression as the ``subcollection``.
+
+    Here, a collection refers to the top-level entry to access an item - a collection called ``el`` will be accessible under the ``el`` attributes via ``events['el']`` or ``events.el``. A subcollection called ``pt`` will be accessible under that collection, such as ``events['el']['pt']`` or ``events.el.pt``. This is the power of the schema providing a more user-friendly (and programmatic) access to the underlying branches.
+
+    The above logic means that the following branches below will be categorized as follows:
+
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | branch                        | collection        | subcollection         | systematic       |
+    +===============================+===================+=======================+==================+
+    | ``'eventNumber'``             | ``'eventNumber'`` | ``None``              | ``None``         |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'runNumber'``               | ``'runNumber'``   | ``None``              | ``None``         |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'el_pt_NOSYS'``             | ``'el'``          | ``'pt'``              | ``'NOSYS'``      |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'jet_cleanTightBad_NOSYS'`` | ``'jet'``         | ``'cleanTightBad'``   | ``'NOSYS'``      |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'jet_select_btag_NOSYS'``   | ``'jet'``         | ``'select_btag'``     | ``'NOSYS'``      |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'jet_e_NOSYS'``             | ``'jet'``         | ``'e'``               | ``'NOSYS'``      |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'truthel_phi'``             | ``'truthel'``     | ``'phi'``             | ``None``         |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'truthel_pt'``              | ``'truthel'``     | ``'pt'``              | ``None``         |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'ph_eta'``                  | ``'ph'``          | ``'eta'``             | ``None``         |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'ph_phi_SCALE__1up'``       | ``'ph'``          | ``'phi'``             | ``'SCALE__1up'`` |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'mu_TTVA_effSF_NOSYS'``     | ``'mu'``          | ``'TTVA_effSF'``      | ``'NOSYS'``      |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'recojet_antikt4PFlow_pt'`` | ``'recojet'``     | ``'antikt4PFlow_pt'`` | ``'NOSYS'``      |
+    +-------------------------------+-------------------+-----------------------+------------------+
+    | ``'recojet_antikt10UFO_m'``   | ``'recojet'``     | ``'antikt10UFO_m'``   | ``None``         |
+    +-------------------------------+-------------------+-----------------------+------------------+
+
+    Sometimes this logic is not what you want, and there are ways to teach ``NtupleSchema`` how to group some of these better for atypical cases. We can address these case-by-case.
+
+    **Singletons**
+
+     Sometimes you have particular branches that you don't want to be treated as a collection (with subcollections). And sometimes you will see warnings about this (see :ref:`faq`). There are some pre-defined ``singletons`` stored under :attr:`event_ids`, and these will be lazily treated as a _singleton_. For other cases where you add your own branches, you can additionally extend this class to add your own :attr:`singletons`:
+
+     .. code-block:: python
+
+        from atlas_schema.schema import NtupleSchema
+
+
+        class MySchema(NtupleSchema):
+            singletons = {"RandomRunNumber"}
+
+     and use this schema in your analysis code. The rest of the logic will be handled for you, and you can access your singletons under ``events.RandomRunNumber`` as expected.
+
+    **Mixins (collections, subcollections)**
+
+     In more complicated scenarios, you might need to teach :class:`NtupleSchema` how to handle collections that end up having underscores in their name, or other characters that make the grouping non-trivial. In some other scenarios, you want to tell the schema to assign a certain set of behaviors to a collection - rather than the default :class:`atlas_schema.methods.Particle` behavior. This is where :attr:`mixins` comes in. Similar to how :attr:`singletons` are handled, you extend this schema to include your own ``mixins`` pointing them at one of the behaviors defined in :mod:`atlas_schema.methods`.
+
+     Let's demonstrate both cases. Imagine you want to have your ``truthel`` collections above treated as :class:`atlas_schema.methods.Electron`, then you would extend the existing :attr:`mixins`:
+
+     .. code-block:: python
+
+        from atlas_schema.schema import NtupleSchema
+
+
+        class MySchema(NtupleSchema):
+            mixins = {"truthel": "Electron", **NtupleSchema.mixins}
+
+     Now, ``events.truthel`` will give you arrays zipped up with :class:`atlas_schema.methods.Electron` behaviors.
+
+     If instead, you run into problems with mixing different branches in the same collection, because the default behavior of this schema described above is not smart enough to handle the atypical cases, you can explicitly fix this by defining your collections:
+
+     .. code-block:: python
+
+        from atlas_schema.schema import NtupleSchema
+
+
+        class MySchema(NtupleSchema):
+            mixins = {
+                "recojet_antikt4PFlow": "Jet",
+                "recojet_antikt10UFO": "Jet",
+                **NtupleSchema.mixins,
+            }
+
+     Now, ``events.recojet_antikt4PFlow`` and ``events.recojet_antikt10UFO`` will be separate collections, instead of a single ``events.recojet`` that incorrectly merged branches from each of these collections.
+    """
+
+    __dask_capable__: ClassVar[bool] = True
+
+    warn_missing_crossrefs: ClassVar[bool] = True
+
+    #: Treat missing event-level branches as error instead of warning (default is ``False``)
+    error_missing_event_ids: ClassVar[bool] = False
+    #: Determine closest behavior for a given branch or treat branch as :attr:`default_behavior` (default is ``True``)
+    identify_closest_behavior: ClassVar[bool] = True
+
+    #: event IDs to expect in data datasets
+    event_ids_data: ClassVar[set[str]] = {
+        "lumiBlock",
+        "averageInteractionsPerCrossing",
+        "actualInteractionsPerCrossing",
+        "dataTakingYear",
+    }
+    #: event IDs to expect in MC datasets
+    event_ids_mc: ClassVar[set[str]] = {
+        "mcChannelNumber",
+        "runNumber",
+        "eventNumber",
+        "mcEventWeights",
+    }
+    #: all event IDs to expect in the dataset
+    event_ids: ClassVar[set[str]] = {*event_ids_data, *event_ids_mc}
+
+    #: mixins defining the mapping from collection name to behavior to use for that collection
+    mixins: ClassVar[dict[str, str]] = {
+        "el": "Electron",
+        "jet": "Jet",
+        "met": "MissingET",
+        "mu": "Muon",
+        "pass": "Pass",
+        "ph": "Photon",
+        "trigPassed": "Trigger",
+        "weight": "Weight",
+    }
+
+    #: additional branches to pass-through with no zipping or additional interpretation (such as those stored as length-1 vectors)
+    singletons: ClassVar[set[str]] = set()
+
+    #: docstrings to assign for specific subcollections across the various collections identified by this schema
+    docstrings: ClassVar[dict[str, str]] = {
+        "charge": "charge",
+        "eta": "pseudorapidity",
+        "met": "missing transverse energy [MeV]",
+        "mass": "invariant mass [MeV]",
+        "pt": "transverse momentum [MeV]",
+        "phi": "azimuthal angle",
+    }
+
+    #: default behavior to use for any collection (default ``"NanoCollection"``, from :class:`coffea.nanoevents.methods.base.NanoCollection`)
+    default_behavior: ClassVar[str] = "NanoCollection"
+
+    def __init__(self, base_form: dict[str, Any], version: str = "latest"):
+        super().__init__(base_form)
+        self._version = version
+        if version == "latest":
+            pass
+        else:
+            pass
+        self._form["fields"], self._form["contents"] = self._build_collections(
+            self._form["fields"], self._form["contents"]
+        )
+        self._form["parameters"]["metadata"]["version"] = self._version
+
+    @classmethod
+    def v1(cls, base_form: dict[str, Any]) -> Self:
+        """Build the NtupleEvents
+
+        For example, one can use ``NanoEventsFactory.from_root("file.root", schemaclass=NtupleSchema.v1)``
+        to ensure NanoAODv7 compatibility.
+        """
+        return cls(base_form, version="1")
+
+    def _build_collections(
+        self, field_names: list[str], input_contents: list[Any]
+    ) -> tuple[KeysView[str], ValuesView[dict[str, Any]]]:
+        branch_forms = dict(zip(field_names, input_contents))
+
+        # parse into high-level records (collections, list collections, and singletons)
+        collections = {k.split("_")[0] for k in branch_forms}
+        collections -= self.event_ids
+        collections -= set(self.singletons)
+
+        # now handle any collections that we identified that are substrings of the items in the mixins
+        # convert all valid branch_forms into strings to make the lookups a bit faster
+        bf_str = ",".join(branch_forms.keys())
+        for mixin in self.mixins:
+            if mixin in collections:
+                continue
+            if f",{mixin}_" not in bf_str and not bf_str.startswith(f"{mixin}_"):
+                continue
+            if "_" in mixin:
+                warnings.warn(
+                    f"I identified a mixin that I did not automatically identify as a collection because it contained an underscore: '{mixin}'. I will add this to the known collections. To suppress this warning next time, please create your ntuples with collections without underscores. [mixin-underscore]",
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
+            collections.add(mixin)
+            for collection in list(collections):
+                if mixin.startswith(f"{collection}_"):
+                    warnings.warn(
+                        f"I found a misidentified collection: '{collection}'. I will remove this from the known collections. To suppress this warning next time, please create your ntuples with collections that are not similarly named with underscores. [collection-subset]",
+                        RuntimeWarning,
+                        stacklevel=2,
+                    )
+                    collections.remove(collection)
+                    break
+
+        # rename needed because easyjet breaks the AMG assumptions
+        # https://gitlab.cern.ch/easyjet/easyjet/-/issues/246
+        for k in list(branch_forms):
+            if "NOSYS" not in k:
+                continue
+            branch_forms[k.replace("_NOSYS", "") + "_NOSYS"] = branch_forms.pop(k)
+
+        # these are collections with systematic variations
+        subcollections = {
+            k.split("__")[0].split("_", 1)[1].replace("_NOSYS", "")
+            for k in branch_forms
+            if "NOSYS" in k
+        }
+
+        # Check the presence of the event_ids
+        missing_event_ids = [
+            event_id for event_id in self.event_ids if event_id not in branch_forms
+        ]
+
+        if len(missing_event_ids) > 0:
+            if self.error_missing_event_ids:
+                msg = f"There are missing event ID fields: {missing_event_ids} \n\n\
+                    The event ID fields {self.event_ids} are necessary to perform sub-run identification \
+                    (e.g. for corrections and sub-dividing data during different detector conditions),\
+                    to cross-validate MC and Data (i.e. matching events for comparison), and to generate event displays. \
+                    It's advised to never drop these branches from the dataformat.\n\n\
+                    This error can be demoted to a warning by setting the class level variable error_missing_event_ids to False."
+                raise RuntimeError(msg)
+            warnings.warn(
+                f"Missing event_ids : {missing_event_ids}",
+                RuntimeWarning,
+                stacklevel=2,
+            )
+
+        output = {}
+
+        # first, register singletons (event-level, others)
+        for name in {*self.event_ids, *self.singletons}:
+            if name in missing_event_ids:
+                continue
+            output[name] = branch_forms[name]
+
+        # next, go through and start grouping up collections
+        for name in collections:
+            content = {}
+            used = set()
+
+            for subname in subcollections:
+                prefix = f"{name}_{subname}_"
+                used.update({k for k in branch_forms if k.startswith(prefix)})
+                subcontent = {
+                    k[len(prefix) :]: branch_forms[k]
+                    for k in branch_forms
+                    if k.startswith(prefix)
+                }
+                if subcontent:
+                    # create the nominal version
+                    content[subname] = branch_forms[f"{prefix}NOSYS"]
+                    # create a collection of the systematic variations for the given variable
+                    content[f"{subname}_syst"] = zip_forms(
+                        subcontent, f"{name}_syst", record_name="NanoCollection"
+                    )
+
+            content.update(
+                {
+                    k[len(name) + 1 :]: branch_forms[k]
+                    for k in branch_forms
+                    if k.startswith(name + "_") and k not in used
+                }
+            )
+
+            if not used and not content:
+                warnings.warn(
+                    f"I identified a branch that likely does not have any leaves: '{name}'. I will treat this as a 'singleton'. To suppress this warning next time, please define your singletons explicitly. [singleton-undefined]",
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
+                self.singletons.add(name)
+                output[name] = branch_forms[name]
+
+            else:
+                behavior = self.mixins.get(name, "")
+                if not behavior:
+                    behavior = self.suggested_behavior(name)
+                    warnings.warn(
+                        f"I found a collection with no defined mixin: '{name}'. I will assume behavior: '{behavior}'. To suppress this warning next time, please define mixins for your custom collections. [mixin-undefined]",
+                        RuntimeWarning,
+                        stacklevel=2,
+                    )
+
+                output[name] = zip_forms(content, name, record_name=behavior)
+
+            output[name].setdefault("parameters", {})
+            output[name]["parameters"].update({"collection_name": name})
+
+            if output[name]["class"] == "ListOffsetArray":
+                if output[name]["class"] == "RecordArray":
+                    parameters = output[name]["content"]["fields"]
+                    contents = output[name]["content"]["contents"]
+                else:
+                    # these are also singletons of another kind that we just pass through
+                    continue
+            elif output[name]["class"] == "RecordArray":
+                parameters = output[name]["fields"]
+                contents = output[name]["contents"]
+            elif output[name]["class"] == "NumpyArray":
+                # these are singletons that we just pass through
+                continue
+            else:
+                msg = f"Unhandled class {output[name]['class']}"
+                raise RuntimeError(msg)
+
+            # update docstrings as needed
+            # NB: must be before flattening for easier logic
+            for index, parameter in enumerate(parameters):
+                if "parameters" not in contents[index]:
+                    continue
+
+                parsed_name = parameter.replace("_NOSYS", "")
+                contents[index]["parameters"]["__doc__"] = self.docstrings.get(
+                    parsed_name,
+                    contents[index]["parameters"].get(
+                        "__doc__", "no docstring available"
+                    ),
+                )
+
+        return output.keys(), output.values()
+
+    @classmethod
+    def behavior(cls) -> Behavior:
+        """Behaviors necessary to implement this schema
+
+        Returns:
+            dict[str | tuple['*', str], type[awkward.Record]]: an :data:`awkward.behavior` dictionary
+        """
+        from atlas_schema.methods import behavior as roaster
+
+        return roaster
+
+    @classmethod
+    def suggested_behavior(cls, key: str, cutoff: float = 0.4) -> str:
+        """
+        Suggest e behavior to use for a provided collection or branch name.
+
+        Default behavior: :class:`~coffea.nanoevents.methods.base.NanoCollection`.
+
+        Note:
+            If :attr:`identify_closest_behavior` is ``False``, then this function will return the default behavior ``NanoCollection``.
+
+        Warning:
+            If no behavior is found above the *cutoff* score, then this function will return the default behavior.
+
+        Args:
+            key (str): collection name to suggest a matching behavior for
+            cutoff (float): o ptional argument cutoff (default ``0.4``) is a float in the range ``[0, 1]``. Possibilities that don't score at least that similar to *key* are ignored.
+
+        Returns:
+            str: suggested behavior to use by string
+
+        Example:
+            >>> from atlas_schema.schema import NtupleSchema
+            >>> NtupleSchema.suggested_behavior("truthjet")
+            'Jet'
+            >>> NtupleSchema.suggested_behavior("SignalElectron")
+            'Electron'
+            >>> NtupleSchema.suggested_behavior("generatorWeight")
+            'Weight'
+            >>> NtupleSchema.suggested_behavior("aVeryStrangelyNamedBranchWithNoMatch")
+            'NanoCollection'
+        """
+        if cls.identify_closest_behavior:
+            # lowercase everything to do case-insensitive matching
+            behaviors = [b for b in cls.behavior() if isinstance(b, str)]
+            behaviors_l = [b.lower() for b in behaviors]
+            results = difflib.get_close_matches(
+                key.lower(), behaviors_l, n=1, cutoff=cutoff
+            )
+            if not results:
+                return cls.default_behavior
+
+            behavior = results[0]
+            # need to identify the index and return the unlowered version
+            return behaviors[behaviors_l.index(behavior)]
+        return cls.default_behavior

{atlas_schema-0.2.2 → atlas_schema-0.2.4}/src/atlas_schema/typing_compat.py

@@ -5,7 +5,7 @@ Typing helpers.
 from __future__ import annotations
 
 import sys
-from typing import Annotated
+from typing import Annotated, Literal, Union
 
 import awkward
 
@@ -19,6 +19,6 @@ if sys.version_info >= (3, 11):
 else:
     from typing_extensions import Self
 
-Behavior: TypeAlias = dict[str, type[awkward.Record]]
+Behavior: TypeAlias = dict[Union[str, tuple[Literal["*"]], str], type[awkward.Record]]
 
 __all__ = ("Annotated", "Behavior", "Self")

atlas_schema-0.2.4/src/atlas_schema/utils.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import TypeVar, Union, cast
+
+import awkward as ak
+import dask_awkward as dak
+
+Array = TypeVar("Array", bound=Union[dak.Array, ak.Array])
+_E = TypeVar("_E", bound=Enum)
+
+
+def isin(element: Array, test_elements: dak.Array | ak.Array, axis: int = -1) -> Array:
+    """
+    Find test_elements in element. Similar in API as :func:`numpy.isin`.
+
+    Calculates `element in test_elements`, broadcasting over *element elements only*. Returns a boolean array of the same shape as *element* that is `True` where an element of *element* is in *test_elements* and `False` otherwise.
+
+    This works by first transforming *test_elements* to an array with one more
+    dimension than the *element*, placing the *test_elements* at *axis*, and then doing a
+    comparison.
+
+    Args:
+        element (dask_awkward.Array or ak.Array): input array of values.
+        test_elements (dask_awkward.Array or ak.Array): one-dimensional set of values against which to test each value of *element*.
+        axis (int): the axis along which the comparison is performed
+
+    Returns:
+        dask_awkward.Array or ak.Array: result of comparison for test_elements in *element*
+
+    Example:
+        >>> import awkward as ak
+        >>> import atlas_schema as ats
+        >>> truth_origins = ak.Array([[1, 2, 3], [4], [5, 6, 7], [1]])
+        >>> prompt_origins = ak.Array([1, 2, 7])
+        >>> ats.isin(truth_origins, prompt_origins).to_list()
+        [[True, True, False], [False], [False, False, True], [True]]
+    """
+    assert test_elements.ndim == 1, "test_elements must be one-dimensional"
+    assert axis >= -1, "axis must be -1 or positive-valued"
+    assert axis < element.ndim + 1, "axis too large for the element"
+
+    # First, build up the transformation, with slice(None) indicating where to stick the test_elements
+    reshaper: list[None | slice] = [None] * element.ndim
+    axis = element.ndim if axis == -1 else axis
+    reshaper.insert(axis, slice(None))
+
+    # Note: reshaper needs to be a tuple for indexing purposes
+    return cast(Array, ak.any(element == test_elements[tuple(reshaper)], axis=-1))

atlas_schema-0.2.2/README.md

@@ -1,37 +0,0 @@
-# atlas-schema v0.2.2
-
-[![Actions Status][actions-badge]][actions-link]
-[![Documentation Status][rtd-badge]][rtd-link]
-
-[![PyPI version][pypi-version]][pypi-link]
-[![Conda-Forge][conda-badge]][conda-link]
-[![PyPI platforms][pypi-platforms]][pypi-link]
-
-[![GitHub Discussion][github-discussions-badge]][github-discussions-link]
-
-<!-- SPHINX-START -->
-
-<!-- prettier-ignore-start -->
-[actions-badge]:            https://github.com/scipp-atlas/atlas-schema/workflows/CI/badge.svg
-[actions-link]:             https://github.com/scipp-atlas/atlas-schema/actions
-[conda-badge]:              https://img.shields.io/conda/vn/conda-forge/atlas-schema
-[conda-link]:               https://github.com/conda-forge/atlas-schema-feedstock
-[github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
-[github-discussions-link]:  https://github.com/scipp-atlas/atlas-schema/discussions
-[pypi-link]:                https://pypi.org/project/atlas-schema/
-[pypi-platforms]:           https://img.shields.io/pypi/pyversions/atlas-schema
-[pypi-version]:             https://img.shields.io/pypi/v/atlas-schema
-[rtd-badge]:                https://readthedocs.org/projects/atlas-schema/badge/?version=latest
-[rtd-link]:                 https://atlas-schema.readthedocs.io/en/latest/?badge=latest
-
-<!-- prettier-ignore-end -->
-
-## Developer Notes
-
-### Converting Enums from C++ to Python
-
-This useful `vim` substitution helps:
-
-```
-%s/    \([A-Za-z]\+\)\s\+=  \(\d\+\),\?/    \1: Annotated[int, "\1"] = \2
-```

atlas_schema-0.2.2/src/atlas_schema/schema.py

@@ -1,206 +0,0 @@
-from __future__ import annotations
-
-import warnings
-from collections.abc import KeysView, ValuesView
-from typing import Any, ClassVar
-
-from coffea.nanoevents.schemas.base import BaseSchema, zip_forms
-
-from atlas_schema.typing_compat import Behavior, Self
-
-
-class NtupleSchema(BaseSchema):  # type: ignore[misc]
-    """Ntuple schema builder
-
-    The Ntuple schema is built from all branches found in the supplied file, based on
-    the naming pattern of the branches. The following additional arrays are constructed:
-
-    - n/a
-    """
-
-    __dask_capable__ = True
-
-    warn_missing_crossrefs = True
-    error_missing_event_ids = False
-
-    event_ids_data: ClassVar[set[str]] = {
-        "lumiBlock",
-        "averageInteractionsPerCrossing",
-        "actualInteractionsPerCrossing",
-        "dataTakingYear",
-    }
-    event_ids_mc: ClassVar[set[str]] = {
-        "mcChannelNumber",
-        "runNumber",
-        "eventNumber",
-        "mcEventWeights",
-    }
-    event_ids: ClassVar[set[str]] = {*event_ids_data, *event_ids_mc}
-
-    mixins: ClassVar[dict[str, str]] = {
-        "el": "Electron",
-        "jet": "Jet",
-        "met": "MissingET",
-        "mu": "Muon",
-        "pass": "Pass",
-        "ph": "Photon",
-        "trigPassed": "Trigger",
-        "weight": "Weight",
-    }
-
-    # These are stored as length-1 vectors unnecessarily
-    singletons: ClassVar[list[str]] = []
-
-    docstrings: ClassVar[dict[str, str]] = {
-        "charge": "charge",
-        "eta": "pseudorapidity",
-        "met": "missing transverse energy [MeV]",
-        "mass": "invariant mass [MeV]",
-        "pt": "transverse momentum [MeV]",
-        "phi": "azimuthal angle",
-    }
-
-    def __init__(self, base_form: dict[str, Any], version: str = "latest"):
-        super().__init__(base_form)
-        self._version = version
-        if version == "latest":
-            pass
-        else:
-            pass
-        self._form["fields"], self._form["contents"] = self._build_collections(
-            self._form["fields"], self._form["contents"]
-        )
-        self._form["parameters"]["metadata"]["version"] = self._version
-
-    @classmethod
-    def v1(cls, base_form: dict[str, Any]) -> Self:
-        """Build the NtupleEvents
-
-        For example, one can use ``NanoEventsFactory.from_root("file.root", schemaclass=NtupleSchema.v1)``
-        to ensure NanoAODv7 compatibility.
-        """
-        return cls(base_form, version="1")
-
-    def _build_collections(
-        self, field_names: list[str], input_contents: list[Any]
-    ) -> tuple[KeysView[str], ValuesView[dict[str, Any]]]:
-        branch_forms = dict(zip(field_names, input_contents))
-
-        # parse into high-level records (collections, list collections, and singletons)
-        collections = {k.split("_")[0] for k in branch_forms}
-        collections -= self.event_ids
-        collections -= set(self.singletons)
-
-        # rename needed because easyjet breaks the AMG assumptions
-        # https://gitlab.cern.ch/easyjet/easyjet/-/issues/246
-        for k in list(branch_forms):
-            if "NOSYS" not in k:
-                continue
-            branch_forms[k.replace("_NOSYS", "") + "_NOSYS"] = branch_forms.pop(k)
-
-        # these are collections with systematic variations
-        subcollections = {
-            k.split("__")[0].split("_", 1)[1].replace("_NOSYS", "")
-            for k in branch_forms
-            if "NOSYS" in k
-        }
-
-        # Check the presence of the event_ids
-        missing_event_ids = [
-            event_id for event_id in self.event_ids if event_id not in branch_forms
-        ]
-
-        if len(missing_event_ids) > 0:
-            if self.error_missing_event_ids:
-                msg = f"There are missing event ID fields: {missing_event_ids} \n\n\
-                    The event ID fields {self.event_ids} are necessary to perform sub-run identification \
-                    (e.g. for corrections and sub-dividing data during different detector conditions),\
-                    to cross-validate MC and Data (i.e. matching events for comparison), and to generate event displays. \
-                    It's advised to never drop these branches from the dataformat.\n\n\
-                    This error can be demoted to a warning by setting the class level variable error_missing_event_ids to False."
-                raise RuntimeError(msg)
-            warnings.warn(
-                f"Missing event_ids : {missing_event_ids}",
-                RuntimeWarning,
-                stacklevel=2,
-            )
-
-        output = {}
-
-        # first, register the event-level stuff directly
-        for name in self.event_ids:
-            if name in missing_event_ids:
-                continue
-            output[name] = branch_forms[name]
-
-        # next, go through and start grouping up collections
-        for name in collections:
-            mixin = self.mixins.get(name, "NanoCollection")
-            content = {}
-            used = set()
-
-            for subname in subcollections:
-                prefix = f"{name}_{subname}_"
-                used.update({k for k in branch_forms if k.startswith(prefix)})
-                subcontent = {
-                    k[len(prefix) :]: branch_forms[k]
-                    for k in branch_forms
-                    if k.startswith(prefix)
-                }
-                if subcontent:
-                    # create the nominal version
-                    content[subname] = branch_forms[f"{prefix}NOSYS"]
-                    # create a collection of the systematic variations for the given variable
-                    content[f"{subname}_syst"] = zip_forms(
-                        subcontent, f"{name}_syst", record_name="NanoCollection"
-                    )
-
-            content.update(
-                {
-                    k[len(name) + 1 :]: branch_forms[k]
-                    for k in branch_forms
-                    if k.startswith(name + "_") and k not in used
-                }
-            )
-
-            output[name] = zip_forms(content, name, record_name=mixin)
-
-            output[name].setdefault("parameters", {})
-            output[name]["parameters"].update({"collection_name": name})
-
-            if output[name]["class"] == "ListOffsetArray":
-                parameters = output[name]["content"]["fields"]
-                contents = output[name]["content"]["contents"]
-            elif output[name]["class"] == "RecordArray":
-                parameters = output[name]["fields"]
-                contents = output[name]["contents"]
-            else:
-                msg = f"Unhandled class {output[name]['class']}"
-                raise RuntimeError(msg)
-            # update docstrings as needed
-            # NB: must be before flattening for easier logic
-            for index, parameter in enumerate(parameters):
-                if "parameters" not in contents[index]:
-                    continue
-
-                parsed_name = parameter.replace("_NOSYS", "")
-                contents[index]["parameters"]["__doc__"] = self.docstrings.get(
-                    parsed_name,
-                    contents[index]["parameters"].get(
-                        "__doc__", "no docstring available"
-                    ),
-                )
-
-            if name in self.singletons:
-                # flatten! this 'promotes' the content of an inner dimension
-                # upwards, effectively hiding one nested dimension
-                output[name] = output[name]["content"]
-
-        return output.keys(), output.values()
-
-    @classmethod
-    def behavior(cls) -> Behavior:
-        """Behaviors necessary to implement this schema"""
-        from atlas_schema.methods import behavior as roaster
-
-        return roaster

atlas_schema-0.2.2/src/atlas_schema/utils.py

@@ -1,39 +0,0 @@
-from __future__ import annotations
-
-from enum import Enum
-from typing import TypeVar, Union, cast
-
-import awkward as ak
-import dask_awkward as dak
-
-Array = TypeVar("Array", bound=Union[dak.Array, ak.Array])
-_E = TypeVar("_E", bound=Enum)
-
-
-def isin(haystack: Array, needles: dak.Array | ak.Array, axis: int = -1) -> Array:
-    """
-    Find needles in haystack.
-
-    This works by first transforming needles to an array with one more
-    dimension than the haystack, placing the needles at axis, and then doing a
-    comparison.
-
-    Args:
-        haystack (dak.Array or ak.Array): haystack of values.
-        needles (dak.Array or ak.Array): one-dimensional set of needles to find in haystack.
-        axis (int): the axis along which the comparison is performed
-
-    Returns:
-        dak.Array or ak.Array: result of comparison for needles in haystack
-    """
-    assert needles.ndim == 1, "Needles must be one-dimensional"
-    assert axis >= -1, "axis must be -1 or positive-valued"
-    assert axis < haystack.ndim + 1, "axis too large for the haystack"
-
-    # First, build up the transformation, with slice(None) indicating where to stick the needles
-    reshaper: list[None | slice] = [None] * haystack.ndim
-    axis = haystack.ndim if axis == -1 else axis
-    reshaper.insert(axis, slice(None))
-
-    # Note: reshaper needs to be a tuple for indexing purposes
-    return cast(Array, ak.any(haystack == needles[tuple(reshaper)], axis=-1))