triggerflow 0.1.12__py3-none-any.whl → 0.2__py3-none-any.whl

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/catalog.yml

@@ -0,0 +1,84 @@
+{{ cookiecutter.python_package }}_meta_data:
+  filepath: data/01_raw/samples_dummy.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data:
+  sample_info: data/01_raw/samples_dummy.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_dataset.{{ cookiecutter.project_name }}Dataset
+
+{{ cookiecutter.python_package }}_meta_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_meta_data.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_data.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_train:
+  filepath: data/03_preprocessed/event_ids_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_test:
+  filepath: data/03_preprocessed/event_ids_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+scaler:
+  filepath: data/03_preprocessed/scaler.pkl
+  type: pickle.PickleDataset
+
+train_model:
+  filepath: data/04_models/trained_model.pkl
+  type: pickle.PickleDataset
+
+training_history:
+  type: matplotlib.MatplotlibDataset
+  filepath: data/07_reporting/training_history.png
+  save_args:
+    format: png
+
+model_pred:
+  filepath: data/05_validation/model_pred.pkl
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_compile.yml

@@ -0,0 +1,14 @@
+# This is a boilerplate parameters config generated for pipeline 'compile'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+
+compile:
+  name: "munet"
+  ml_backend: "Keras"
+  compiler: "hls4ml"
+  mlflow_url: "https://mlflow-deploy-mflow.app.cern.ch"
+  compiler_config:
+    test: 123

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_data_processing.yml

@@ -0,0 +1,8 @@
+# This is a boilerplate parameters config generated for pipeline 'data_processing'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+random_state: 42
+test_size: 0.5

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_load_data.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'load_data'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_model_training.yml

@@ -0,0 +1,9 @@
+# This is a boilerplate parameters config generated for pipeline 'model_training'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+{{ cookiecutter.python_package }}_model:
+  hps:
+    name: "{{ cookiecutter.python_package }}"

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base/parameters_model_validation.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'model_validation'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/catalog.yml

@@ -0,0 +1,84 @@
+{{ cookiecutter.python_package }}_meta_data:
+  filepath: data/01_raw/samples_dummy.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data:
+  sample_info: data/01_raw/samples_dummy.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_dataset.{{ cookiecutter.project_name }}Dataset
+
+{{ cookiecutter.python_package }}_meta_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_meta_data.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
+
+{{ cookiecutter.python_package }}_data_loaded:
+  filepath: data/02_loaded/{{ cookiecutter.python_package }}_data.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_X_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_X_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_train:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+processed_{{ cookiecutter.python_package }}_y_test:
+  filepath: data/03_preprocessed/processed_{{ cookiecutter.python_package }}_y_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_train:
+  filepath: data/03_preprocessed/event_ids_train.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+event_ids_test:
+  filepath: data/03_preprocessed/event_ids_test.csv
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset
+
+scaler:
+  filepath: data/03_preprocessed/scaler.pkl
+  type: pickle.PickleDataset
+
+train_model:
+  filepath: data/04_models/trained_model.pkl
+  type: pickle.PickleDataset
+
+training_history:
+  type: matplotlib.MatplotlibDataset
+  filepath: data/07_reporting/training_history.png
+  save_args:
+    format: png
+
+model_pred:
+  filepath: data/05_validation/model_pred.pkl
+  save_args:
+    index: False
+    sep: ','
+  type: pandas.CSVDataset

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_compile.yml

@@ -0,0 +1,14 @@
+# This is a boilerplate parameters config generated for pipeline 'compile'
+# using Kedro 0.19.14.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html
+
+
+compile:
+  name: "munet"
+  ml_backend: "Keras"
+  compiler: "hls4ml"
+  mlflow_url: "https://mlflow-deploy-mflow.app.cern.ch"
+  compiler_config:
+    test: 123

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_data_processing.yml

@@ -0,0 +1,8 @@
+# This is a boilerplate parameters config generated for pipeline 'data_processing'
+# using Kedro 0.19.14.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html
+
+random_state: 42
+test_size: 0.5

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_load_data.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'load_data'
+# using Kedro 0.19.14.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_training.yml

@@ -0,0 +1,9 @@
+# This is a boilerplate parameters config generated for pipeline 'model_training'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/1.0.0/configuration/parameters.html
+
+{{ cookiecutter.python_package }}_model:
+  hps:
+    name: "{{ cookiecutter.python_package }}"

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_validation.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'model_validation'
+# using Kedro 1.0.0
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.14/configuration/parameters.html

triggerflow/starter/{{ cookiecutter.repo_name }}/conf/logging.yml

@@ -0,0 +1,43 @@
+# To enable this custom logging configuration, set KEDRO_LOGGING_CONFIG to the path of this file.
+# More information available at https://docs.kedro.org/en/stable/logging/logging.html
+version: 1
+
+disable_existing_loggers: False
+
+formatters:
+  simple:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: simple
+    stream: ext://sys.stdout
+
+  info_file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: INFO
+    formatter: simple
+    filename: info.log
+    maxBytes: 10485760 # 10MB
+    backupCount: 20
+    encoding: utf8
+    delay: True
+
+  rich:
+    class: kedro.logging.RichHandler
+    rich_tracebacks: True
+    # Advance options for customisation.
+    # See https://docs.kedro.org/en/stable/logging/logging.html#project-side-logging-configuration
+    # tracebacks_show_locals: False
+
+loggers:
+  kedro:
+    level: INFO
+
+  {{ cookiecutter.python_package }}:
+    level: INFO
+
+root:
+  handlers: [rich, info_file_handler]

triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples.json

@@ -0,0 +1,15 @@
+{
+    "samples" : {
+        "testSample" : {
+            "all_file_path":"data/01_raw/test.root",
+            "path":"data/01_raw/test.root",
+            "file_pattern":["test.root"],
+            "DAS" : "Blabla",
+            "type" : "123",
+            "data" : false,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": true
+        }
+    }
+}

triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples_dummy.json

@@ -0,0 +1,26 @@
+{
+    "samples" : {
+        "muon24I" : {
+            "path": "data/01_raw/samples_dummy.json",
+            "folder":"data/01_raw/",
+            "file_pattern":["samples_dummy.json"],
+            "DAS" : "-",
+            "type" : "-",
+            "data" : true,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": true
+        },
+        "zb24I" : {
+            "path": "data/01_raw/samples_dummy.json",
+            "folder":"data/01_raw/",
+            "file_pattern":["samples_dummy.json"],
+            "DAS" : "-",
+            "type" : "-",
+            "data" : true,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": false
+        }
+    }
+}

triggerflow/starter/{{ cookiecutter.repo_name }}/dvc.yaml

@@ -0,0 +1,7 @@
+stages:
+  process_samples:
+    cmd: 
+    deps:
+      - 
+    outs:
+      - 

triggerflow/starter/{{ cookiecutter.repo_name }}/environment.yml

@@ -0,0 +1,21 @@
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.11
+  - pip
+  - pip:
+    - kedro
+    - kedro-viz
+    - kedro-datasets
+    - matplotlib
+    - mplhep
+    - shap
+    - scikit-learn
+    - pandas
+    - dvc
+    - shap
+    - ruff
+    - uproot
+    - awkward
+    - triggerflow

triggerflow/starter/{{ cookiecutter.repo_name }}/pyproject.toml

@@ -0,0 +1,50 @@
+[build-system]
+requires = [ "setuptools",]
+build-backend = "setuptools.build_meta"
+
+[project]
+requires-python = ">=3.10"
+name = "{{ cookiecutter.python_package }}"
+readme = "README.md"
+dynamic = [ "version",]
+dependencies = [ "ipython>=8.10", "jupyterlab>=3.0", "notebook", "kedro~=1.0.0",]
+
+[project.scripts]
+{{ cookiecutter.project_name }} = "{{ cookiecutter.python_package }}.__main__:main"
+
+[project.optional-dependencies]
+dev = [ "pytest-cov~=3.0", "pytest-mock>=1.7.1, <2.0", "pytest~=7.2", "ruff~=0.1.8",]
+
+[tool.kedro]
+package_name = "{{ cookiecutter.python_package }}"
+project_name = "{{ cookiecutter.project_name }}"
+kedro_init_version = "1.0.0"
+tools = "['Linting', 'Testing', 'Custom Logging', 'Data Structure']"
+example_pipeline = "False"
+source_dir = "src"
+
+[tool.ruff]
+line-length = 88
+show-fixes = true
+select = [ "F", "W", "E", "I", "UP", "PL", "T201",]
+ignore = [ "E501",]
+
+[project.entry-points."kedro.hooks"]
+
+[tool.pytest.ini_options]
+addopts = "--cov-report term-missing --cov src/{{ cookiecutter.python_package }} -ra"
+
+[tool.coverage.report]
+fail_under = 0
+show_missing = true
+exclude_lines = [ "pragma: no cover", "raise NotImplementedError",]
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.setuptools.dynamic.version]
+attr = "{{ cookiecutter.python_package }}.__version__"
+
+[tool.setuptools.packages.find]
+where = [ "src",]
+namespaces = false

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/__init__.py

@@ -0,0 +1,3 @@
+"""{{ cookiecutter.project_name }}"""
+
+__version__ = "0.1"

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/__main__.py

@@ -0,0 +1,25 @@
+"""{{ cookiecutter.project_name }} file for ensuring the package is executable
+as `{{ cookiecutter.project_name }}` and `python -m {{ cookiecutter.python_package }}`
+"""
+
+import sys
+from pathlib import Path
+from typing import Any
+
+from kedro.framework.cli.utils import find_run_command
+from kedro.framework.project import configure_project
+
+
+def main(*args, **kwargs) -> Any:
+    package_name = Path(__file__).parent.name
+    configure_project(package_name)
+
+    interactive = hasattr(sys, "ps1")
+    kwargs["standalone_mode"] = not interactive
+
+    run = find_run_command(package_name)
+    return run(*args, **kwargs)
+
+
+if __name__ == "__main__":
+    main()

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/any_object.py

@@ -0,0 +1,20 @@
+from kedro.io import AbstractDataset
+from typing import Any
+
+
+class AnyObject(AbstractDataset):
+    """
+    Abstract class which can be used for passing "Any" object
+    """
+
+    def __init__(self):
+        pass
+
+    def _load(self) -> None:
+        pass
+
+    def _save(self, data: Any) -> Any:
+        return data
+
+    def _describe(self) -> dict:
+        return {}

triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/base_dataset.py

@@ -0,0 +1,137 @@
+import logging, uproot, json, os
+import pandas as pd
+import numpy as np
+from abc import abstractmethod
+from fnmatch import filter as fnmatch_filter
+from kedro.io import AbstractDataset
+
+
+class BaseDataset(AbstractDataset):
+    """
+    Abstract Base Class for loading data from ROOT files.
+
+    Users must inherit from this class and implement the abstract methods.
+    The core processing logic in `_load` is fixed and cannot be overridden.
+    """
+
+    def __init__(self, sample_info: str, sample_key: str):
+        with open(sample_info, "r") as f:
+            data = json.load(f)
+        self._sample_info = data[sample_key]
+        self._sample_key = sample_key
+
+        # get logger for reporting
+        self.logger = logging.getLogger(__name__)
+        self.logger.info(f"Initializing dataset: {self.__class__.__name__}")
+
+    @abstractmethod
+    def get_branches_to_keep(self) -> list[str]:
+        """
+        USER MUST IMPLEMENT: Return a list of branch names or patterns (with wildcards)
+        to keep from the ROOT file.
+
+        Example:
+            return ["Jet_*", "PuppiMET_pt", "nJet"]
+        """
+        pass
+
+    @abstractmethod
+    def get_cut(self) -> str | None:
+        """
+        USER MUST IMPLEMENT: Return a string representing the cuts to apply to the data.
+        """
+        pass
+
+    @abstractmethod
+    def convert_to_pandas(self, data: dict) -> pd.DataFrame:
+        """
+        USER MUST IMPLEMENT: Convert the loaded data from a dictionary format to a pandas DataFrame.
+        """
+        pass
+
+    def get_tree_name(self) -> str:
+        return "Events"
+
+    def _resolve_branches(self, all_branches: list) -> list[str]:
+        """Internal method to resolve wildcard patterns."""
+        selected = []
+        for pattern in self.get_branches_to_keep():
+            matched = fnmatch_filter(all_branches, pattern)
+            if not matched:
+                self.logger.warning(f"Pattern '{pattern}' did not match any branches.")
+            selected.extend(matched)
+        return sorted(list(set(selected)))
+
+    def _load(self) -> pd.DataFrame:
+        """
+        CORE LOGIC (NOT OVERRIDABLE): Loads and processes a single ROOT file.
+        """
+
+        # Process all files in sample
+        df = pd.DataFrame()
+
+        all_root_files = []
+        for key in self._sample_info.keys():
+            files = os.listdir(self._sample_info[key]["folder"])
+            cur_files = []
+            for file_pattern in self._sample_info[key]["file_pattern"]:
+                for f in fnmatch_filter(files, file_pattern):
+                    cur_files.append(os.path.join(self._sample_info[key]["folder"], f))
+            all_root_files.append(cur_files)
+
+        is_signals = [
+            self._sample_info[key]["is_signal"] for key in self._sample_info.keys()
+        ]
+        self.logger.info("Processing files")
+        for root_files, is_signal in zip(all_root_files, is_signals):
+            self.logger.info(f"Processing files: {root_files}")
+            for root_file in root_files:
+                if f"{root_file}" == "data/01_raw/samples_dummy.json":
+                    n = 100
+                    # generate dummy features
+                    dummy_data = {}
+                    for branch in self.get_branches_to_keep():
+                        dummy_data[branch] = np.random.randn(n)
+                    if is_signal:
+                        dummy_data["is_signal"] = np.ones(n)
+                    else:
+                        dummy_data["is_signal"] = np.zeros(n)
+
+                    cur_df = pd.DataFrame(dummy_data)
+
+                    # generate a binary target (0/1)
+                    cur_df["y"] = np.random.choice([0, 1], size=n)
+
+                    df = pd.concat([df, cur_df])
+
+                else:
+                    with uproot.open(f"{root_file}") as f:
+                        tree = f[self.get_tree_name()]
+                        all_branches = tree.keys()
+                        branches_to_load = self._resolve_branches(all_branches)
+
+                        if not branches_to_load:
+                            self.logger.warning(
+                                f"No valid branches to load for {root_file}. Skipping."
+                            )
+                            continue
+
+                        data = tree.arrays(branches_to_load, cut=self.get_cut())
+
+                        cur_df = self.convert_to_pandas(data)
+
+                        # set background or signal
+                        if is_signal:
+                            cur_df["is_signal"] = [1 for _ in range(len(cur_df))]
+                        else:
+                            cur_df["is_signal"] = [0 for _ in range(len(cur_df))]
+
+                        df = pd.concat([df, cur_df])
+
+        return df
+
+    def _save(self, data: pd.DataFrame) -> pd.DataFrame:
+        return data
+
+    def _describe(self) -> dict:
+        return {"output_sample_info": self._sample_info, "sample_key": self._sample_key}