triggerflow 0.3.4__py3-none-any.whl

triggerflow/starter/{{ cookiecutter.repo_name }}/environment.yml

@@ -0,0 +1,23 @@
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.11
+  - pip
+  - pip:
+    - kedro
+    - kedro-viz
+    - kedro-datasets
+    - matplotlib
+    - mplhep
+    - shap
+    - scikit-learn
+    - pandas
+    - dvc
+    - shap
+    - ruff
+    - uproot
+    - awkward
+    - triggerflow
+    - coffea
+    - dask[distributed]

triggerflow/starter/{{ cookiecutter.repo_name }}/pyproject.toml

@@ -0,0 +1,50 @@
+[build-system]
+requires = [ "setuptools",]
+build-backend = "setuptools.build_meta"
+
+[project]
+requires-python = ">=3.10"
+name = "{{ cookiecutter.python_package }}"
+readme = "README.md"
+dynamic = [ "version",]
+dependencies = [ "ipython>=8.10", "jupyterlab>=3.0", "notebook", "kedro~=1.0.0",]
+
+[project.scripts]
+{{ cookiecutter.project_name }} = "{{ cookiecutter.python_package }}.__main__:main"
+
+[project.optional-dependencies]
+dev = [ "pytest-cov~=3.0", "pytest-mock>=1.7.1, <2.0", "pytest~=7.2", "ruff~=0.1.8",]
+
+[tool.kedro]
+package_name = "{{ cookiecutter.python_package }}"
+project_name = "{{ cookiecutter.project_name }}"
+kedro_init_version = "1.0.0"
+tools = "['Linting', 'Testing', 'Custom Logging', 'Data Structure']"
+example_pipeline = "False"
+source_dir = "src"
+
+[tool.ruff]
+line-length = 88
+show-fixes = true
+select = [ "F", "W", "E", "I", "UP", "PL", "T201",]
+ignore = [ "E501",]
+
+[project.entry-points."kedro.hooks"]
+
+[tool.pytest.ini_options]
+addopts = "--cov-report term-missing --cov src/{{ cookiecutter.python_package }} -ra"
+
+[tool.coverage.report]
+fail_under = 0
+show_missing = true
+exclude_lines = [ "pragma: no cover", "raise NotImplementedError",]
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.setuptools.dynamic.version]
+attr = "{{ cookiecutter.python_package }}.__version__"
+
+[tool.setuptools.packages.find]
+where = [ "src",]
+namespaces = false

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/__init__.py

@@ -0,0 +1,3 @@
+"""{{ cookiecutter.project_name }}"""
+
+__version__ = "0.1"

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/__main__.py

@@ -0,0 +1,25 @@
+"""{{ cookiecutter.project_name }} file for ensuring the package is executable
+as `{{ cookiecutter.project_name }}` and `python -m {{ cookiecutter.python_package }}`
+"""
+
+import sys
+from pathlib import Path
+from typing import Any
+
+from kedro.framework.cli.utils import find_run_command
+from kedro.framework.project import configure_project
+
+
+def main(*args, **kwargs) -> Any:
+    package_name = Path(__file__).parent.name
+    configure_project(package_name)
+
+    interactive = hasattr(sys, "ps1")
+    kwargs["standalone_mode"] = not interactive
+
+    run = find_run_command(package_name)
+    return run(*args, **kwargs)
+
+
+if __name__ == "__main__":
+    main()

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/any_object.py

@@ -0,0 +1,20 @@
+from kedro.io import AbstractDataset
+from typing import Any
+
+
+class AnyObject(AbstractDataset):
+    """
+    Abstract class which can be used for passing "Any" object
+    """
+
+    def __init__(self):
+        pass
+
+    def _load(self) -> None:
+        pass
+
+    def _save(self, data: Any) -> Any:
+        return data
+
+    def _describe(self) -> dict:
+        return {}

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/base_dataset.py

@@ -0,0 +1,137 @@
+import logging, uproot, json, os
+import pandas as pd
+import numpy as np
+from abc import abstractmethod
+from fnmatch import filter as fnmatch_filter
+from kedro.io import AbstractDataset
+
+
+class BaseDataset(AbstractDataset):
+    """
+    Abstract Base Class for loading data from ROOT files.
+
+    Users must inherit from this class and implement the abstract methods.
+    The core processing logic in `_load` is fixed and cannot be overridden.
+    """
+
+    def __init__(self, sample_info: str, sample_key: str):
+        with open(sample_info, "r") as f:
+            data = json.load(f)
+        self._sample_info = data[sample_key]
+        self._sample_key = sample_key
+
+        # get logger for reporting
+        self.logger = logging.getLogger(__name__)
+        self.logger.info(f"Initializing dataset: {self.__class__.__name__}")
+
+    @abstractmethod
+    def get_branches_to_keep(self) -> list[str]:
+        """
+        USER MUST IMPLEMENT: Return a list of branch names or patterns (with wildcards)
+        to keep from the ROOT file.
+
+        Example:
+            return ["Jet_*", "PuppiMET_pt", "nJet"]
+        """
+        pass
+
+    @abstractmethod
+    def get_cut(self) -> str | None:
+        """
+        USER MUST IMPLEMENT: Return a string representing the cuts to apply to the data.
+        """
+        pass
+
+    @abstractmethod
+    def convert_to_pandas(self, data: dict) -> pd.DataFrame:
+        """
+        USER MUST IMPLEMENT: Convert the loaded data from a dictionary format to a pandas DataFrame.
+        """
+        pass
+
+    def get_tree_name(self) -> str:
+        return "Events"
+
+    def _resolve_branches(self, all_branches: list) -> list[str]:
+        """Internal method to resolve wildcard patterns."""
+        selected = []
+        for pattern in self.get_branches_to_keep():
+            matched = fnmatch_filter(all_branches, pattern)
+            if not matched:
+                self.logger.warning(f"Pattern '{pattern}' did not match any branches.")
+            selected.extend(matched)
+        return sorted(list(set(selected)))
+
+    def _load(self) -> pd.DataFrame:
+        """
+        CORE LOGIC (NOT OVERRIDABLE): Loads and processes a single ROOT file.
+        """
+
+        # Process all files in sample
+        df = pd.DataFrame()
+
+        all_root_files = []
+        for key in self._sample_info.keys():
+            files = os.listdir(self._sample_info[key]["folder"])
+            cur_files = []
+            for file_pattern in self._sample_info[key]["file_pattern"]:
+                for f in fnmatch_filter(files, file_pattern):
+                    cur_files.append(os.path.join(self._sample_info[key]["folder"], f))
+            all_root_files.append(cur_files)
+
+        is_signals = [
+            self._sample_info[key]["is_signal"] for key in self._sample_info.keys()
+        ]
+        self.logger.info("Processing files")
+        for root_files, is_signal in zip(all_root_files, is_signals):
+            self.logger.info(f"Processing files: {root_files}")
+            for root_file in root_files:
+                if f"{root_file}" == "data/01_raw/samples_dummy.json":
+                    n = 100
+                    # generate dummy features
+                    dummy_data = {}
+                    for branch in self.get_branches_to_keep():
+                        dummy_data[branch] = np.random.randn(n)
+                    if is_signal:
+                        dummy_data["is_signal"] = np.ones(n)
+                    else:
+                        dummy_data["is_signal"] = np.zeros(n)
+
+                    cur_df = pd.DataFrame(dummy_data)
+
+                    # generate a binary target (0/1)
+                    cur_df["y"] = np.random.choice([0, 1], size=n)
+
+                    df = pd.concat([df, cur_df])
+
+                else:
+                    with uproot.open(f"{root_file}") as f:
+                        tree = f[self.get_tree_name()]
+                        all_branches = tree.keys()
+                        branches_to_load = self._resolve_branches(all_branches)
+
+                        if not branches_to_load:
+                            self.logger.warning(
+                                f"No valid branches to load for {root_file}. Skipping."
+                            )
+                            continue
+
+                        data = tree.arrays(branches_to_load, cut=self.get_cut())
+
+                        cur_df = self.convert_to_pandas(data)
+
+                        # set background or signal
+                        if is_signal:
+                            cur_df["is_signal"] = [1 for _ in range(len(cur_df))]
+                        else:
+                            cur_df["is_signal"] = [0 for _ in range(len(cur_df))]
+
+                        df = pd.concat([df, cur_df])
+
+        return df
+
+    def _save(self, data: pd.DataFrame) -> pd.DataFrame:
+        return data
+
+    def _describe(self) -> dict:
+        return {"output_sample_info": self._sample_info, "sample_key": self._sample_key}

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/base_loader.py

@@ -0,0 +1,101 @@
+import logging, json
+from abc import abstractmethod
+from kedro.io import AbstractDataset
+from trigger_loader.loader import TriggerLoader
+import pandas as pd
+import numpy as np
+from pathlib import Path
+
+
+class BaseLoader(AbstractDataset):
+    """
+    Abstract Base Class for using the TriggerLoader.
+
+    Users must inherit from this class and implement the abstract methods.
+    The core processing logic in `_load` is fixed and cannot be overridden.
+    """
+
+    def __init__(self, sample_json: str, settings: str, config: str):
+        self.sample_json = sample_json
+        with open(settings, "r") as f:
+            self.settings = json.load(f)
+        with open(config, "r") as f:
+            self.config = json.load(f)
+
+        # get logger for reporting
+        self.logger = logging.getLogger(__name__)
+        self.logger.info(f"Initializing loader: {self.__class__.__name__}")
+
+    @abstractmethod
+    def transform(self, events):
+        """
+        USER MUST IMPLEMENT.
+        """
+        pass
+
+    def _load(self) -> pd.DataFrame:
+        """
+        CORE LOGIC (NOT OVERRIDABLE): Loads and processes a single ROOT file.
+        """
+
+        self.logger.info(f"Start Loading...")
+        loader = TriggerLoader(
+            sample_json=self.sample_json,
+            transform=self.transform,
+            output_path=self.settings["output_dir"]
+        )
+
+        if self.settings["run_local"]:
+            loader.run_local(
+                num_workers=self.settings["num_workers"],
+                chunksize=self.settings["chunksize"]
+            )
+        else:
+            loader.run_distributed(
+                cluster_type=self.settings["cluster_type"],
+                cluster_config=self.config,
+                chunksize=self.settings["chunksize"],
+                jobs=self.settings["jobs"]
+            )
+
+        # load last parquet file from manifest file for each dataset key
+        # from the meta_data
+        dataset_keys = set(loader.meta_data.keys())
+        manifest_path = Path(self.settings["output_dir"]) / "manifest.json"
+
+        last_records = {key: None for key in dataset_keys}
+
+        with manifest_path.open() as f:
+            for line in f:
+                record = json.loads(line)
+                dataset = record.get("dataset")
+
+                if dataset in last_records:
+                    last_records[dataset] = record
+
+        # sanity check
+        missing = [k for k, v in last_records.items() if v is None]
+        if missing:
+            raise ValueError(f"No manifest entry found for datasets: {missing}")
+
+        final_dfs = []
+        for dataset_key, record in last_records.items():
+            file_path = record["parquet_file"]
+            df = pd.read_parquet(file_path)
+
+            if loader.meta_data[dataset_key]["is_signal"]:
+                df["is_signal"] = np.ones(len(df), dtype=int)
+                df["y"] = np.ones(len(df), dtype=int)
+            else:
+                df["is_signal"] = np.zeros(len(df), dtype=int)
+                df["y"] = np.zeros(len(df), dtype=int)
+
+            final_dfs.append(df)
+
+        return pd.concat(final_dfs, ignore_index=True)
+
+    def _save(self, data: pd.DataFrame) -> pd.DataFrame:
+        return data
+
+    def _describe(self) -> dict:
+        return {"sample_json": self.sample_json, "settings": self.settings, "config": self.config}

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/meta_dataset.py

@@ -0,0 +1,49 @@
+import logging, json
+from glob import glob
+from kedro.io import AbstractDataset
+
+METADATA_CONFIG = {"x": 0}
+
+class MetaDataset(AbstractDataset):
+    def __init__(self, filepath: str, sample_key: str):
+        self._filepath = filepath
+        self._sample_key = sample_key
+        self.logger = logging.getLogger(__name__)
+
+    def get_dasgoclient_metadata(self, das_name: str, config: dict) -> dict:
+        self.logger.info(f"Fetching DAS metadata for dataset: {das_name}")
+        return {"gridpack": "0.0.0"}
+
+    def _load(self) -> dict:
+        self.logger.info(f"Processing file: {self._filepath}")
+        with open(self._filepath, "r") as f:
+            data = json.load(f)
+        return data
+
+    def _save(self, samples: dict) -> None:
+        metadata = {}
+
+        dataset_content = samples.get(self._sample_key, {})
+        
+        for sample_name, sample_info in dataset_content.items():
+            self.logger.info(f"Processing sample: {sample_name}")
+
+            sample_files = sample_info.get("files", "")
+            
+            resolved_files = glob(sample_files) if isinstance(sample_files, str) else sample_files
+            sample_info["files"] = resolved_files
+            
+            self.logger.info(f"Found {len(resolved_files)} files for {sample_name}.")
+
+            metadata[sample_name] = self.get_dasgoclient_metadata(
+                sample_info.get("DAS", "Unknown"), 
+                METADATA_CONFIG
+            )
+
+        with open(self._filepath, "w") as f:
+            json.dump(metadata, f, indent=4)
+
+    def _describe(self) -> dict:
+        return {"filepath": self._filepath, "sample_key": self._sample_key}
+    
+    

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/{{ cookiecutter.python_package }}_dataset.py

@@ -0,0 +1,35 @@
+import pandas as pd
+from .base_dataset import BaseDataset
+
+
+class {{ cookiecutter.python_package }}Dataset(BaseDataset):
+    """
+    A custom dataset example.
+    """
+
+    def get_branches_to_keep(self) -> list[str]:
+        """
+        Define the branches you needed.
+        """
+        return [
+            "PuppiMET_pt",
+            "CaloMET_pt",
+            "event",  # <-- we need this for meta data
+            # "Jet_pt",
+            # "Jet_eta",
+            # "Jet_phi",
+            # "Jet_btag*", # Use a wildcard to get all b-tagging info
+            "nJet",
+        ]
+
+    def get_cut(self) -> str | None:
+        """
+        Apply a pre-selection cut to keep only events with exactly 1 jet.
+        """
+        return "nJet == 1"
+
+    def convert_to_pandas(self, data: dict):
+        """
+        Logic to convert from dict of (potentially nested) arrays to a pandas DataFrame.
+        """
+        return pd.DataFrame(data)

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/{{ cookiecutter.python_package }}_loader.py

@@ -0,0 +1,32 @@
+import pandas as pd
+import awkward as ak
+from .base_loader import BaseLoader
+
+
+class {{ cookiecutter.python_package }}Loader(BaseLoader):
+    """
+    A custom loader example.
+    """
+
+    def transform(self, events):
+
+        jets = events.Jet
+        pt  = ak.fill_none(ak.pad_none(jets.pt , 2, axis=1, clip=True), -9999.9)
+        eta = ak.fill_none(ak.pad_none(jets.eta, 2, axis=1, clip=True), -9999.9)
+        phi = ak.fill_none(ak.pad_none(jets.phi, 2, axis=1, clip=True), -9999.9)
+
+        met = events.MET
+
+        result = ak.zip({
+            "event": events.event,
+            "jet_pt_1":  pt[:, 0],
+            "jet_pt_2":  pt[:, 1],
+            "jet_eta_1": eta[:, 0],
+            "jet_eta_2": eta[:, 1],
+            "jet_phi_1": phi[:, 0],
+            "jet_phi_2": phi[:, 1],
+            "met_pt":    met.pt,
+            "met_phi":   met.phi,
+        })
+
+        return result

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/models/base_model.py

@@ -0,0 +1,155 @@
+import inspect
+import pandas as pd
+from abc import ABC, abstractmethod
+from typing import Any
+from sklearn.base import BaseEstimator
+
+
+class BaseModel(ABC, BaseEstimator):
+    """
+    Standard Wrapper for a model
+    """
+
+    def __init__(self, name: str, hps: dict):
+        self.name = name
+        # this will be overwritten after training
+        self.model = None
+        self.history = None
+        self.callbacks = []
+        self.hps = hps
+
+    @abstractmethod
+    def train(self, X: pd.DataFrame, y: pd.DataFrame, hps: dict, **kwargs):
+        """
+        User code function.
+        Args:
+            X: features
+            y: label
+            hps: hyperparameters
+            kwargs: anything else needed for training
+        """
+        pass
+
+    @abstractmethod
+    def build(self):
+        """
+        User code function to build the model.
+        """
+        pass
+
+    def predict(self, X: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        """
+        Calculates predictions of the model
+        Args:
+            X: features
+
+        Returns:
+            predictions
+            (optional in user code) kwargs: anything else needed for predicting
+        """
+        y_pred = self.model.predict(X.astype("float32"))
+        return pd.DataFrame(y_pred)
+
+    def predict_proba(self, X: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        """
+        Calculates proba predictions of the model
+        Args:
+            X: features
+
+        Returns:
+            predictions
+            (optional in user code) kwargs: anything else needed for predicting
+        """
+        y_pred = self.model.predict_proba(X.astype("float32"))
+        return pd.DataFrame(y_pred)
+
+    def fit(self, X: pd.DataFrame, y: pd.DataFrame):
+        """
+        Same as train but get kwargs from __init__ for sklearn API
+        Args:
+            X: features
+            y: label
+
+        X can also contain optional inputs https://github.com/scikit-learn/scikit-learn/issues/2879.
+        Which should be specified in the user code.
+        For example when the train function needs additional inputs:
+        ```python
+            curX = X.copy()
+            kwargs = {"S": curX["S"]}
+            del curX["S"]
+            self.train(curX, y, self.hps, **kwargs)
+        ```
+        """
+        self.train(X, y, self.hps)
+
+    def get_params(self, deep=True):
+        """
+        Get parameters for self.model and self.
+        Args:
+            deep : bool, default=True
+                If True, will return the parameters for this estimator and
+                contained subobjects that are estimators.
+
+        Returns:
+            params : dict
+                Parameter names mapped to their values.
+        """
+        out = dict()
+        # if self.hps is set return them and not the default values
+        for key in self.hps:
+            out[key] = self.hps[key]
+        for key in get_param_names(self):
+            value = getattr(self, key)
+            if deep and hasattr(value, "get_params") and not isinstance(value, type):
+                deep_items = value.get_params().items()
+                out.update((key + "__" + k, val) for k, val in deep_items)
+            out[key] = value
+        return out
+
+    def set_params(self, **params):
+        """
+        Set the parameters of this estimator.
+
+        We overwrite the sklearn BaseEstimator and set params to self.hps
+        Args:
+            **params : dict
+                Estimator parameters.
+
+        Returns:
+            self : estimator instance
+                Estimator instance.
+        """
+        self.hps = params
+
+        return self
+
+
+def get_param_names(cls):
+    """Get parameter names for the estimator"""
+    # fetch the constructor or the original constructor before
+    # deprecation wrapping if any
+    init = getattr(cls.__init__, "deprecated_original", cls.__init__)
+    if init is object.__init__:
+        # No explicit constructor to introspect
+        return []
+
+    # introspect the constructor arguments to find the model parameters
+    # to represent
+    init_signature = inspect.signature(init)
+    # Consider the constructor parameters excluding 'self'
+    parameters = [
+        p
+        for p in init_signature.parameters.values()
+        if p.name != "self" and p.kind != p.VAR_KEYWORD
+    ]
+    for p in parameters:
+        if p.kind == p.VAR_POSITIONAL:
+            raise RuntimeError(
+                "scikit-learn estimators should always "
+                "specify their parameters in the signature"
+                " of their __init__ (no varargs)."
+                " %s with constructor %s doesn't "
+                " follow this convention." % (cls, init_signature)
+            )
+    # Extract and sort argument names excluding 'self'
+    return sorted([p.name for p in parameters])

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/models/{{ cookiecutter.python_package }}_model.py

@@ -0,0 +1,16 @@
+import pandas as pd
+from .base_model import BaseModel
+from sklearn.dummy import DummyClassifier
+
+
+class {{ cookiecutter.python_package }}(BaseModel):
+    def train(self, X: pd.DataFrame, y: pd.DataFrame, **kwargs):
+        self.build()
+        self.history = self.model.fit(X, y)
+
+    def build(self):
+        """Build the test Model.
+        self.hps:
+            - 
+        """
+        self.model = DummyClassifier()

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipeline_registry.py

@@ -0,0 +1,17 @@
+"""Project pipelines."""
+
+from __future__ import annotations
+
+from kedro.framework.project import find_pipelines
+from kedro.pipeline import Pipeline
+
+
+def register_pipelines() -> dict[str, Pipeline]:
+    """Register the project's pipelines.
+
+    Returns:
+        A mapping from pipeline names to ``Pipeline`` objects.
+    """
+    pipelines = find_pipelines()
+    pipelines["__default__"] = sum(pipelines.values())
+    return pipelines

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/compile/__init__.py

@@ -0,0 +1,10 @@
+"""
+This is a boilerplate pipeline 'compile'
+generated using Kedro 1.0.0
+"""
+
+from .pipeline import create_pipeline
+
+__all__ = ["create_pipeline"]
+
+__version__ = "0.1"