triggerflow 0.2.9__tar.gz → 0.3.1__tar.gz

{triggerflow-0.2.9/src/triggerflow.egg-info → triggerflow-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: triggerflow
-Version: 0.2.9
+Version: 0.3.1
 Summary: Utilities for ML models targeting hardware triggers
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License

{triggerflow-0.2.9 → triggerflow-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "triggerflow"
-version = "0.2.9"
+version = "0.3.1"
 description = "Utilities for ML models targeting hardware triggers"
 readme = "README.md"
 requires-python = ">=3.11"

{triggerflow-0.2.9 → triggerflow-0.3.1}/src/trigger_loader/loader.py

@@ -12,8 +12,6 @@ from coffea.nanoevents import NanoAODSchema
 from .cluster_manager import ClusterManager
 from .processor import TriggerProcessor
 
-logger = logging.getLogger(__name__)
-
 
 class TriggerLoader:
     def __init__(self,
@@ -21,8 +19,9 @@ class TriggerLoader:
         transform: callable,
         output_path: str,
     ):
+        self.logger = logging.getLogger(__name__)
         self.transform = transform
-        self.fileset = self._load_sample_json(sample_json)
+        self.fileset, self.meta_data = self._load_sample_json(sample_json)
         self.output_path = output_path
         self.run_uuid = str(uuid.uuid4())
 
@@ -62,41 +61,38 @@ class TriggerLoader:
             full_data = json.load(f)
             dataset_metadata = full_data.get("samples", full_data)
 
-        fileset = {}
+        fileset, meta_data = {}, {}
         for ds_name, ds_info in dataset_metadata.items():
             files = []
-            
             if "files" in ds_info:
                 file_info = ds_info["files"]
-                
                 if isinstance(file_info, list):
                     files = file_info
-                    
                 elif isinstance(file_info, str):
                     if os.path.isdir(file_info):
                         path_glob = os.path.join(file_info, "*.root")
                         files = glob.glob(path_glob)
-                        logger.info(f"Resolved {len(files)} files from directory {file_info}.")
+                        self.logger.info(f"Resolved {len(files)} files from directory {file_info}.")
                     else:
                         files = [file_info]
-                
                 if files:
-                    logger.info(f"Using {len(files)} local/explicit files for {ds_name}.")
- 
+                    self.logger.info(f"Using {len(files)} local/explicit files for {ds_name}.")
+
             if not files and "DAS" in ds_info:
                 try:
                     files = _fetch_files_from_das(ds_info["DAS"]) 
-                    logger.info(f"Resolved {len(files)} files via DAS for {ds_name}.")
+                    self.logger.info(f"Resolved {len(files)} files via DAS for {ds_name}.")
                 except NameError:
-                    logger.error("DAS fetching skipped: _fetch_files_from_das is not defined.")
+                    self.logger.info("DAS fetching skipped: _fetch_files_from_das is not defined.")
 
             if not files:
-                logger.warning(f"No files found for dataset: {ds_name}. Skipping.")
+                self.logger.warning(f"No files found for dataset: {ds_name}. Skipping.")
                 continue
 
             fileset[ds_name] = files
+            meta_data[ds_name] = {"files": files, "is_signal": ds_info["is_signal"]}
 
-        return fileset
+        return fileset, meta_data
 
     def _write_run_metadata_file(self, path: str, duration_s: float | None = None):
         meta_path = f"{path}/run_metadata.json"
@@ -108,11 +104,10 @@ class TriggerLoader:
             json.dump(data, f, indent=2)
 
     def _run(self, runner: processor.Runner, label: str):
-        logger.log(logging.INFO, f"Starting processing ({label})...")
+        self.logger.info(f"Starting processing ({label})...")
         start = time.time()
         proc = self._build_processor()
-        print(self.fileset)
-        
+
         acc = runner(
             self.fileset,
             treename="Events",
@@ -120,7 +115,7 @@ class TriggerLoader:
         )
         elapsed = time.time() - start
         self._write_run_metadata_file(self.output_path, elapsed)
-        logger.log(logging.INFO, f"Finished in {elapsed:.2f}s (run_uuid={self.run_uuid})")
+        self.logger.info(f"Finished in {elapsed:.2f}s (run_uuid={self.run_uuid})")
         return acc
 
     def run_distributed(self, cluster_type: str, cluster_config: dict,

{triggerflow-0.2.9 → triggerflow-0.3.1}/src/trigger_loader/processor.py

@@ -32,7 +32,7 @@ class TriggerProcessor(processor.ProcessorABC):
         filename_template: str = "{dataset}_{fileuuid}_{start}-{stop}.parquet",
         add_uuid: bool = False,
         write_manifest: bool = True,
-        manifest_name: str = "manifest.jsonl",
+        manifest_name: str = "manifest.json",
         run_uuid: str | None = None,
         run_metadata: dict | None = None,
         preserve_event_metadata: bool = True,
@@ -64,6 +64,7 @@ class TriggerProcessor(processor.ProcessorABC):
         self.run_uuid = run_uuid or str(uuid.uuid4())
         self.run_metadata = run_metadata or {}
         self.preserve_event_metadata = preserve_event_metadata
+        self.output_file = ""
 
         # Initialize output directory and paths
         os.makedirs(self.output_path, exist_ok=True)
@@ -90,7 +91,7 @@ class TriggerProcessor(processor.ProcessorABC):
 
         # Extract event metadata
         event_meta = self._extract_event_metadata(events)
-        output_file = self._generate_output_filename(event_meta)
+        self.output_file = self._generate_output_filename(event_meta)
 
         # Convert to Arrow table
         table = ak.to_arrow_table(data)
@@ -100,11 +101,11 @@ class TriggerProcessor(processor.ProcessorABC):
         table = self._embed_metadata_in_schema(table, file_meta, events.metadata)
 
         # Write Parquet file
-        self._write_parquet(table, output_file)
+        self._write_parquet(table, self.output_file)
 
         # Write manifest entry
         if self.write_manifest:
-            self._write_manifest_entry(output_file, file_meta)
+            self._write_manifest_entry(self.output_file, file_meta)
 
         return {}
 

{triggerflow-0.2.9 → triggerflow-0.3.1}/src/triggerflow/core.py

@@ -232,9 +232,10 @@ class ConverterFactory:
     @staticmethod
     def create_converter(ml_backend: str, compiler: str) -> ModelConverter:
         if ml_backend.lower() == "keras" and compiler.lower() == "hls4ml":
-            return KerasToQONNXConverter()
-        else:
-            return NoOpConverter()
+            import keras
+            if not keras.__version__.startswith("3"):
+                return KerasToQONNXConverter()
+        return NoOpConverter()
 
 
 class CompilerFactory:
@@ -284,7 +285,7 @@ class WorkspaceManager:
     def save_native_model(self, model, ml_backend: str):
         """Save the native model to workspace"""
         if ml_backend.lower() == "keras":
-            model.save(self.workspace / "keras_model")
+            model.save(self.workspace / "keras_model.h5")
         elif ml_backend.lower() == "xgboost":
             model.save_model(str(self.workspace / "xgb_model.json"))
     
@@ -609,4 +610,4 @@ class TriggerModel:
         obj.converter = ConverterFactory.create_converter(obj.ml_backend, obj.compiler)
         obj.dataset_object = None 
         
-        return obj
+        return obj

{triggerflow-0.2.9/src/triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local → triggerflow-0.3.1/src/triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base}/catalog.yml

@@ -1,18 +1,18 @@
 {{ cookiecutter.python_package }}_meta_data:
-  filepath: data/01_raw/samples_dummy.json
+  filepath: data/01_raw/samples.json
   sample_key: samples
   type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
 
-{{ cookiecutter.python_package }}_data:
-  sample_info: data/01_raw/samples_dummy.json
-  sample_key: samples
-  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_dataset.{{ cookiecutter.project_name }}Dataset
-
 {{ cookiecutter.python_package }}_meta_data_loaded:
   filepath: data/02_loaded/{{ cookiecutter.python_package }}_meta_data.json
   sample_key: samples
   type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
 
+{{ cookiecutter.python_package }}_data:
+  sample_info: data/01_raw/samples.json
+  sample_key: samples
+  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_dataset.{{ cookiecutter.project_name }}Dataset
+
 {{ cookiecutter.python_package }}_data_loaded:
   filepath: data/02_loaded/{{ cookiecutter.python_package }}_data.csv
   save_args:

{triggerflow-0.2.9/src/triggerflow/starter/{{ cookiecutter.repo_name }}/conf/base → triggerflow-0.3.1/src/triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local}/catalog.yml

@@ -1,10 +1,10 @@
 {{ cookiecutter.python_package }}_meta_data:
-  filepath: data/01_raw/samples_dummy.json
+  filepath: data/01_raw/samples.json
   sample_key: samples
   type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
 
 {{ cookiecutter.python_package }}_data:
-  sample_info: data/01_raw/samples_dummy.json
+  sample_info: data/01_raw/samples.json
   sample_key: samples
   type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_dataset.{{ cookiecutter.project_name }}Dataset
 
@@ -13,6 +13,12 @@
   sample_key: samples
   type: {{ cookiecutter.python_package }}.datasets.meta_dataset.MetaDataset
 
+{{ cookiecutter.python_package }}_loader:
+  sample_json: data/01_raw/samples.json
+  settings: data/01_raw/settings.json
+  config: data/01_raw/condor_config.json
+  type: {{ cookiecutter.python_package }}.datasets.{{ cookiecutter.python_package }}_loader.{{ cookiecutter.project_name }}Loader
+
 {{ cookiecutter.python_package }}_data_loaded:
   filepath: data/02_loaded/{{ cookiecutter.python_package }}_data.csv
   save_args:

triggerflow-0.3.1/src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/condor_config.json

@@ -0,0 +1,11 @@
+{
+    "log_directory": "./condor_logs",
+    "cores": 2,
+    "memory": "2GB",
+    "disk": "2GB",
+    "job_extra_directives": {
+        "+JobFlavour": "longlunch",
+        "getenv": true
+    },
+    "death_timeout": "60"
+}

triggerflow-0.3.1/src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/cuda_config.json

@@ -0,0 +1,4 @@
+{
+    "n_workers": 2,
+    "device_memory_limit": "16GB"
+}

triggerflow-0.3.1/src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples.json

@@ -0,0 +1,28 @@
+{
+    "samples" : {
+        "signal" : {
+            "all_file_path":"data/01_raw/test.root",
+            "files":"data/01_raw/test.root",
+            "path":"data/01_raw/test.root",
+            "file_pattern":["test.root"],
+            "DAS" : "Blabla",
+            "type" : "123",
+            "data" : false,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": true
+        },
+        "background" : {
+            "all_file_path":"data/01_raw/test.root",
+            "files":"data/01_raw/test.root",
+            "path":"data/01_raw/test.root",
+            "file_pattern":["test.root"],
+            "DAS" : "Blabla",
+            "type" : "123",
+            "data" : false,
+            "era" : "phase1",
+            "run" : "run3",
+            "is_signal": false
+        }
+    }
+}

triggerflow-0.3.1/src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/settings.json

@@ -0,0 +1,8 @@
+{
+    "output_dir": "./data/02_loaded/",
+    "run_local": true,
+    "num_workers": 4,
+    "chunksize": 50000,
+    "cluster_type": "local",
+    "jobs": 4
+}

{triggerflow-0.2.9 → triggerflow-0.3.1}/src/triggerflow/starter/{{ cookiecutter.repo_name }}/environment.yml

@@ -19,3 +19,5 @@ dependencies:
     - uproot
     - awkward
     - triggerflow
+    - coffea
+    - dask[distributed]

triggerflow-0.3.1/src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/base_loader.py

@@ -0,0 +1,101 @@
+import logging, json
+from abc import abstractmethod
+from kedro.io import AbstractDataset
+from trigger_loader.loader import TriggerLoader
+import pandas as pd
+import numpy as np
+from pathlib import Path
+
+
+class BaseLoader(AbstractDataset):
+    """
+    Abstract Base Class for using the TriggerLoader.
+
+    Users must inherit from this class and implement the abstract methods.
+    The core processing logic in `_load` is fixed and cannot be overridden.
+    """
+
+    def __init__(self, sample_json: str, settings: str, config: str):
+        self.sample_json = sample_json
+        with open(settings, "r") as f:
+            self.settings = json.load(f)
+        with open(config, "r") as f:
+            self.config = json.load(f)
+
+        # get logger for reporting
+        self.logger = logging.getLogger(__name__)
+        self.logger.info(f"Initializing loader: {self.__class__.__name__}")
+
+    @abstractmethod
+    def transform(self, events):
+        """
+        USER MUST IMPLEMENT.
+        """
+        pass
+
+    def _load(self) -> pd.DataFrame:
+        """
+        CORE LOGIC (NOT OVERRIDABLE): Loads and processes a single ROOT file.
+        """
+
+        self.logger.info(f"Start Loading...")
+        loader = TriggerLoader(
+            sample_json=self.sample_json,
+            transform=self.transform,
+            output_path=self.settings["output_dir"]
+        )
+
+        if self.settings["run_local"]:
+            loader.run_local(
+                num_workers=self.settings["num_workers"],
+                chunksize=self.settings["chunksize"]
+            )
+        else:
+            loader.run_distributed(
+                cluster_type=self.settings["cluster_type"],
+                cluster_config=self.config,
+                chunksize=self.settings["chunksize"],
+                jobs=self.settings["jobs"]
+            )
+
+        # load last parquet file from manifest file for each dataset key
+        # from the meta_data
+        dataset_keys = set(loader.meta_data.keys())
+        manifest_path = Path(self.settings["output_dir"]) / "manifest.json"
+
+        last_records = {key: None for key in dataset_keys}
+
+        with manifest_path.open() as f:
+            for line in f:
+                record = json.loads(line)
+                dataset = record.get("dataset")
+
+                if dataset in last_records:
+                    last_records[dataset] = record
+
+        # sanity check
+        missing = [k for k, v in last_records.items() if v is None]
+        if missing:
+            raise ValueError(f"No manifest entry found for datasets: {missing}")
+
+        final_dfs = []
+        for dataset_key, record in last_records.items():
+            file_path = record["parquet_file"]
+            df = pd.read_parquet(file_path)
+
+            if loader.meta_data[dataset_key]["is_signal"]:
+                df["is_signal"] = np.ones(len(df), dtype=int)
+                df["y"] = np.ones(len(df), dtype=int)
+            else:
+                df["is_signal"] = np.zeros(len(df), dtype=int)
+                df["y"] = np.zeros(len(df), dtype=int)
+
+            final_dfs.append(df)
+
+        return pd.concat(final_dfs, ignore_index=True)
+
+    def _save(self, data: pd.DataFrame) -> pd.DataFrame:
+        return data
+
+    def _describe(self) -> dict:
+        return {"sample_json": self.sample_json, "settings": self.settings, "config": self.config}

triggerflow-0.3.1/src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/{{ cookiecutter.python_package }}_loader.py

@@ -0,0 +1,32 @@
+import pandas as pd
+import awkward as ak
+from .base_loader import BaseLoader
+
+
+class {{ cookiecutter.python_package }}Loader(BaseLoader):
+    """
+    A custom loader example.
+    """
+
+    def transform(self, events):
+
+        jets = events.Jet
+        pt  = ak.fill_none(ak.pad_none(jets.pt , 2, axis=1, clip=True), -9999.9)
+        eta = ak.fill_none(ak.pad_none(jets.eta, 2, axis=1, clip=True), -9999.9)
+        phi = ak.fill_none(ak.pad_none(jets.phi, 2, axis=1, clip=True), -9999.9)
+
+        met = events.MET
+
+        result = ak.zip({
+            "event": events.event,
+            "jet_pt_1":  pt[:, 0],
+            "jet_pt_2":  pt[:, 1],
+            "jet_eta_1": eta[:, 0],
+            "jet_eta_2": eta[:, 1],
+            "jet_phi_1": phi[:, 0],
+            "jet_phi_2": phi[:, 1],
+            "met_pt":    met.pt,
+            "met_phi":   met.phi,
+        })
+
+        return result

{triggerflow-0.2.9 → triggerflow-0.3.1}/src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/load_data/pipeline.py

@@ -12,7 +12,7 @@ def create_pipeline(**kwargs) -> Pipeline:
         [
             node(
                 func=load_data,
-                inputs=["{{ cookiecutter.python_package }}_data", "{{ cookiecutter.python_package }}_meta_data"],
+                inputs=["{{ cookiecutter.python_package }}_loader", "{{ cookiecutter.python_package }}_meta_data"],
                 outputs=["{{ cookiecutter.python_package }}_data_loaded", "{{ cookiecutter.python_package }}_meta_data_loaded"],
                 name="load_data",
             )

{triggerflow-0.2.9 → triggerflow-0.3.1/src/triggerflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: triggerflow
-Version: 0.2.9
+Version: 0.3.1
 Summary: Utilities for ML models targeting hardware triggers
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License

{triggerflow-0.2.9 → triggerflow-0.3.1}/src/triggerflow.egg-info/SOURCES.txt

@@ -47,8 +47,12 @@ src/triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_load_
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_training.yml
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/conf/local/parameters_model_validation.yml
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/.gitkeep
+src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/condor_config.json
+src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/cuda_config.json
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples.json
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples_dummy.json
+src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/settings.json
+src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/test.root
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/02_loaded/.gitkeep
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/03_preprocessed/.gitkeep
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/04_models/.gitkeep
@@ -61,8 +65,10 @@ src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/any_object.py
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/base_dataset.py
+src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/base_loader.py
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/meta_dataset.py
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/{{ cookiecutter.python_package }}_dataset.py
+src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/{{ cookiecutter.python_package }}_loader.py
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/models/__init__.py
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/models/base_model.py
 src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/models/{{ cookiecutter.python_package }}_model.py
@@ -105,4 +111,5 @@ src/triggerflow/templates/model-gt.cpp
 src/triggerflow/templates/model_template.cpp
 src/triggerflow/templates/scales.h
 tests/test.py
-tests/test_loader.py
+tests/test_loader.py
+tests/test_starter.py

{triggerflow-0.2.9 → triggerflow-0.3.1}/tests/test.py

@@ -1,4 +1,5 @@
 import numpy as np
+import keras
 from keras.layers import Input
 from keras.models import Model
 from qkeras.qlayers import QActivation, QDense

triggerflow-0.3.1/tests/test_starter.py

@@ -0,0 +1,31 @@
+# tests/test_starter_demo.py
+import subprocess
+import shutil
+from pathlib import Path
+
+
+def run(cmd, cwd=None):
+    """Run a shell command and stream output live."""
+    subprocess.run(
+        cmd,
+        cwd=cwd,
+        shell=True,
+        check=True,
+        stdout=None,   # inherit terminal stdout
+        stderr=None,   # inherit terminal stderr
+    )
+
+def test_triggerflow_starter_demo():
+    project_dir = Path("test_demo")
+
+    try:
+    # triggerflow new test_demo
+        run("triggerflow new test_demo")
+
+        # kedro run
+        run("kedro run", cwd=project_dir)
+
+    finally:
+        # cleanup (always)
+        if project_dir.exists():
+            shutil.rmtree(project_dir)

triggerflow-0.2.9/src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples.json

@@ -1,15 +0,0 @@
-{
-    "samples" : {
-        "testSample" : {
-            "all_file_path":"data/01_raw/test.root",
-            "path":"data/01_raw/test.root",
-            "file_pattern":["test.root"],
-            "DAS" : "Blabla",
-            "type" : "123",
-            "data" : false,
-            "era" : "phase1",
-            "run" : "run3",
-            "is_signal": true
-        }
-    }
-}