triggerflow 0.2.1__tar.gz → 0.2.3__tar.gz

{triggerflow-0.2.1/src/triggerflow.egg-info → triggerflow-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: triggerflow
-Version: 0.2.1
+Version: 0.2.3
 Summary: Utilities for ML models targeting hardware triggers
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
@@ -35,22 +35,91 @@ pip install triggerflow
 
 from triggerflow.core import TriggerModel
 
-triggerflow = TriggerModel(name="my-trigger-model", ml_backend="Keras", compiler="hls4ml", model, compiler_config or None)
-triggerflow() # call the constructor
+
+scales = {'offsets': np.array([18, 0, 72, 7, 0, 73, 4, 0, 73, 4, 0, 72, 3, 0, 72, 6, -0, 286, 3, -2, 285, 3, -2, 282, 3, -2, 286, 29, 0, 72, 22, 0, 72, 18, 0, 72, 14, 0, 72, 11, 0, 72, 10, 0, 72, 10, 0, 73, 9, 0], dtype='int'),
+'shifts': np.array([3, 0, 6, 2, 5, 6, 0, 5, 6, 0, 5, 6, -1, 5, 6, 2, 7, 8, 0, 7, 8, 0, 7, 8, 0, 7, 8, 4, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6], dtype='int')}
+
+
+trigger_model = TriggerModel(
+    config="triggermodel_config.yaml",
+    native_model=model, #Native XGboost/Keras model
+    scales=scales
+)
+
+trigger_model() #Vivado requird on $PATH for Firmware build.
 
 # then:
-output_software = triggerflow.software_predict(input_data)
-output_firmware = triggerflow.firmware_predict(input_data)
-output_qonnx = triggerflow.qonnx_predict(input_data)
+output_software = trigger_model.software_predict(input_data)
+output_firmware = trigger_model.firmware_predict(input_data)
+output_qonnx = trigger_model.qonnx_predict(input_data)
 
 # save and load trigger models:
-triggerflow.save("triggerflow.tar.xz")
+trigger_model.save("triggerflow.tar.xz")
 
 # in a separate session:
 from triggerflow.core import TriggerModel
 triggerflow = TriggerModel.load("triggerflow.tar.xz")
 ```
 
+## The Config file:
+
+Use this `.yaml` template and change as needed. 
+
+```yaml
+compiler:
+  name: "AXO"
+  ml_backend: "keras"
+  compiler: "hls4ml"
+  fpga_part: "xc7vx690t-ffg1927-2"
+  clock_period: 25
+  n_outputs: 1
+  project_name: "AXO_project"
+  namespace: "AXO"
+  io_type: "io_parallel"
+  backend: "Vitis"
+  write_weights_txt: false
+
+subsystem:
+  name: "uGT"
+  n_inputs: 50
+  offset_type: "ap_fixed<10,10>"
+  shift_type: "ap_fixed<10,10>"
+
+  objects:
+    muons:
+      size: 4
+      features: [pt, eta_extrapolated, phi_extrapolated]
+
+    jets:
+      size: 4
+      features: [et, eta, phi]
+
+    egammas:
+      size: 4
+      features: [et, eta, phi]
+
+    taus:
+      size: 4
+      features: [et, eta, phi]
+
+  global_features:
+    #- et.et
+    #- ht.et
+    - etmiss.et
+    - etmiss.phi
+    #- htmiss.et
+    #- htmiss.phi
+    #- ethfmiss.et
+    #- ethfmiss.phi
+    #- hthfmiss.et
+    #- hthfmiss.phi
+
+  muon_size: 4
+  jet_size: 4
+  egamma_size: 4
+  tau_size: 4
+```
+
 ## Logging with MLflow
 
 ```python

{triggerflow-0.2.1 → triggerflow-0.2.3}/README.md

@@ -15,22 +15,91 @@ pip install triggerflow
 
 from triggerflow.core import TriggerModel
 
-triggerflow = TriggerModel(name="my-trigger-model", ml_backend="Keras", compiler="hls4ml", model, compiler_config or None)
-triggerflow() # call the constructor
+
+scales = {'offsets': np.array([18, 0, 72, 7, 0, 73, 4, 0, 73, 4, 0, 72, 3, 0, 72, 6, -0, 286, 3, -2, 285, 3, -2, 282, 3, -2, 286, 29, 0, 72, 22, 0, 72, 18, 0, 72, 14, 0, 72, 11, 0, 72, 10, 0, 72, 10, 0, 73, 9, 0], dtype='int'),
+'shifts': np.array([3, 0, 6, 2, 5, 6, 0, 5, 6, 0, 5, 6, -1, 5, 6, 2, 7, 8, 0, 7, 8, 0, 7, 8, 0, 7, 8, 4, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6, 6, 3, 6], dtype='int')}
+
+
+trigger_model = TriggerModel(
+    config="triggermodel_config.yaml",
+    native_model=model, #Native XGboost/Keras model
+    scales=scales
+)
+
+trigger_model() #Vivado requird on $PATH for Firmware build.
 
 # then:
-output_software = triggerflow.software_predict(input_data)
-output_firmware = triggerflow.firmware_predict(input_data)
-output_qonnx = triggerflow.qonnx_predict(input_data)
+output_software = trigger_model.software_predict(input_data)
+output_firmware = trigger_model.firmware_predict(input_data)
+output_qonnx = trigger_model.qonnx_predict(input_data)
 
 # save and load trigger models:
-triggerflow.save("triggerflow.tar.xz")
+trigger_model.save("triggerflow.tar.xz")
 
 # in a separate session:
 from triggerflow.core import TriggerModel
 triggerflow = TriggerModel.load("triggerflow.tar.xz")
 ```
 
+## The Config file:
+
+Use this `.yaml` template and change as needed. 
+
+```yaml
+compiler:
+  name: "AXO"
+  ml_backend: "keras"
+  compiler: "hls4ml"
+  fpga_part: "xc7vx690t-ffg1927-2"
+  clock_period: 25
+  n_outputs: 1
+  project_name: "AXO_project"
+  namespace: "AXO"
+  io_type: "io_parallel"
+  backend: "Vitis"
+  write_weights_txt: false
+
+subsystem:
+  name: "uGT"
+  n_inputs: 50
+  offset_type: "ap_fixed<10,10>"
+  shift_type: "ap_fixed<10,10>"
+
+  objects:
+    muons:
+      size: 4
+      features: [pt, eta_extrapolated, phi_extrapolated]
+
+    jets:
+      size: 4
+      features: [et, eta, phi]
+
+    egammas:
+      size: 4
+      features: [et, eta, phi]
+
+    taus:
+      size: 4
+      features: [et, eta, phi]
+
+  global_features:
+    #- et.et
+    #- ht.et
+    - etmiss.et
+    - etmiss.phi
+    #- htmiss.et
+    #- htmiss.phi
+    #- ethfmiss.et
+    #- ethfmiss.phi
+    #- hthfmiss.et
+    #- hthfmiss.phi
+
+  muon_size: 4
+  jet_size: 4
+  egamma_size: 4
+  tau_size: 4
+```
+
 ## Logging with MLflow
 
 ```python

{triggerflow-0.2.1 → triggerflow-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "triggerflow"
-version = "0.2.1"
+version = "0.2.3"
 description = "Utilities for ML models targeting hardware triggers"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -13,7 +13,7 @@ dependencies = [
     "PyYAML>=6",
     "Jinja2>=3",
     "mlflow>=2.0",
-    "kedro==1.0.0",
+    "kedro==1.0.0"
 ]
 classifiers = [
     "Programming Language :: Python :: 3",

{triggerflow-0.2.1 → triggerflow-0.2.3}/src/trigger_loader/cluster_manager.py

@@ -4,9 +4,6 @@ import logging
 from typing import Any
 
 from dask.distributed import Client, LocalCluster
-from dask_cuda import LocalCUDACluster
-from dask_jobqueue import HTCondorCluster
-from dask_kubernetes import KubeCluster
 
 logger = logging.getLogger(__name__)
 
@@ -63,15 +60,18 @@ class ClusterManager:
             self.cluster = LocalCluster(**self.cluster_config)
 
         elif ct == "condor":
+            from dask_jobqueue import HTCondorCluster
             self.cluster = HTCondorCluster(**self.cluster_config)
             if self.jobs and self.jobs > 0:
                 # Scale to the requested number of jobs
                 self.cluster.scale(jobs=self.jobs)
 
         elif ct == "cuda":
+            from dask_cuda import LocalCUDACluster
             self.cluster = LocalCUDACluster(**self.cluster_config)
 
         elif ct == "kubernetes":
+            from dask_kubernetes import KubeCluster
             self.cluster = KubeCluster(**self.cluster_config)
             if self.jobs and self.jobs > 0:
                 try:

{triggerflow-0.2.1 → triggerflow-0.2.3}/src/trigger_loader/loader.py

@@ -45,8 +45,58 @@ class TriggerLoader:
         )
 
     def _load_sample_json(self, sample_json: str) -> dict:
+        """
+        Loads the JSON and resolves file paths using the priority:
+        1. Explicit 'files' list or directory path (Local/Explicit)
+        2. 'DAS' query (Remote Fallback)
+
+        Returns the canonical coffea fileset format: {dataset_name: [file_path_list]}.
+        """
+        import glob
+        import os
+        
+        # Helper function definition needed here if it's not imported:
+        # def _fetch_files_from_das(das_query: str) -> list[str]: ... (placeholder or actual implementation)
+
         with open(sample_json) as f:
-            return json.load(f)
+            full_data = json.load(f)
+            dataset_metadata = full_data.get("samples", full_data)
+
+        fileset = {}
+        for ds_name, ds_info in dataset_metadata.items():
+            files = []
+            
+            if "files" in ds_info:
+                file_info = ds_info["files"]
+                
+                if isinstance(file_info, list):
+                    files = file_info
+                    
+                elif isinstance(file_info, str):
+                    if os.path.isdir(file_info):
+                        path_glob = os.path.join(file_info, "*.root")
+                        files = glob.glob(path_glob)
+                        logger.info(f"Resolved {len(files)} files from directory {file_info}.")
+                    else:
+                        files = [file_info]
+                
+                if files:
+                    logger.info(f"Using {len(files)} local/explicit files for {ds_name}.")
+ 
+            if not files and "DAS" in ds_info:
+                try:
+                    files = _fetch_files_from_das(ds_info["DAS"]) 
+                    logger.info(f"Resolved {len(files)} files via DAS for {ds_name}.")
+                except NameError:
+                    logger.error("DAS fetching skipped: _fetch_files_from_das is not defined.")
+
+            if not files:
+                logger.warning(f"No files found for dataset: {ds_name}. Skipping.")
+                continue
+
+            fileset[ds_name] = files
+
+        return fileset
 
     def _write_run_metadata_file(self, path: str, duration_s: float | None = None):
         meta_path = f"{path}/run_metadata.json"
@@ -58,9 +108,11 @@ class TriggerLoader:
             json.dump(data, f, indent=2)
 
     def _run(self, runner: processor.Runner, label: str):
-        logger.log(f"Starting processing ({label})...")
+        logger.log(logging.INFO, f"Starting processing ({label})...")
         start = time.time()
         proc = self._build_processor()
+        print(self.fileset)
+        
         acc = runner(
             self.fileset,
             treename="Events",
@@ -68,7 +120,7 @@ class TriggerLoader:
         )
         elapsed = time.time() - start
         self._write_run_metadata_file(self.output_path, elapsed)
-        logger.log(f"Finished in {elapsed:.2f}s (run_uuid={self.run_uuid})")
+        logger.log(logging.INFO, f"Finished in {elapsed:.2f}s (run_uuid={self.run_uuid})")
         return acc
 
     def run_distributed(self, cluster_type: str, cluster_config: dict,

{triggerflow-0.2.1 → triggerflow-0.2.3}/src/triggerflow/core.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 import json
+import yaml
 import numpy as np
 import tarfile
 import importlib
@@ -8,6 +9,7 @@ from typing import Optional, Dict, Any, Union
 import shutil, warnings
 import importlib.resources as pkg_resources
 import triggerflow.templates 
+from triggerflow.interfaces.uGT import build_ugt_model
 
 
 class ModelConverter(ABC):
@@ -73,37 +75,31 @@ class NoOpConverter(ModelConverter):
 
 
 class HLS4MLStrategy(CompilerStrategy):
-    """HLS4ML compilation strategy for Keras models"""
-    
-    def compile(self, model, workspace: Path, config: Optional[Dict] = None, **kwargs) -> Any:
+    def compile(self, model, workspace: Path, config: Optional[Dict] = None) -> Any:
         import hls4ml
-        
+
         firmware_dir = workspace / "firmware"
         firmware_dir.mkdir(exist_ok=True)
-        
-        cfg = config or hls4ml.utils.config_from_keras_model(model, granularity="name")
 
-        hls_kwargs = {
-            "hls_config": cfg,
-            "output_dir": str(firmware_dir),
-            "io_type": "io_stream",
-            "backend": "Vitis"
-        }
-        hls_kwargs.update(kwargs)
+        hls_config = hls4ml.utils.config_from_keras_model(model, granularity="name")
+        hls_kwargs = {}
+
+        for key in ["project_name", "namespace", "io_type", "backend", "write_weights_txt"]:
+            if key in config:
+                hls_kwargs[key] = config[key]
 
         firmware_model = hls4ml.converters.convert_from_keras_model(
             model,
+            hls_config=hls_config,
+            output_dir=str(firmware_dir),
             **hls_kwargs
         )
 
         firmware_model.compile()
-        if shutil.which("vivado") is not None:
-             firmware_model.build()
-        else:
-            warnings.warn("Vivado not found in PATH. Firmware build failed.", UserWarning)
         firmware_model.save(workspace / "firmware_model.fml")
         return firmware_model
-    
+
+        
     def load_compiled_model(self, workspace: Path) -> Any:
         from hls4ml.converters import link_existing_project
         
@@ -113,46 +109,31 @@ class HLS4MLStrategy(CompilerStrategy):
 
 
 class ConiferStrategy(CompilerStrategy):
-    """Conifer compilation strategy for XGBoost models"""
+    """Conifer compilation strategy for XGBoost models, unified config/workspace handling."""
 
-    def compile(self, model, workspace: Path, config: Optional[Dict] = None, **kwargs) -> Any:
+    def compile(self, model, workspace: Path, config: Optional[Dict] = None) -> Any:
         import conifer
-        import shutil
-        import warnings
         import os
 
         firmware_dir = workspace / "firmware"
         firmware_dir.mkdir(exist_ok=True)
-        os.environ['JSON_ROOT'] = '/eos/user/m/maglowac/TriggerModel/json'
-        os.environ['XILINX_AP_INCLUDE'] = '/eos/user/m/maglowac/TriggerModel/HLS_arbitrary_Precision_Types/include'
 
-        
-        cfg = conifer.backends.xilinxhls.auto_config()#config or conifer.backends.cpp.auto_config()
+        cfg = conifer.backends.xilinxhls.auto_config()
         cfg['OutputDir'] = str(firmware_dir)
-        
-        for key, value in kwargs.items():
-            cfg[key] = value
+        cfg['ProjectName'] = config['project_name']
+        cfg['XilinxPart'] = config['fpga_part']
+        cfg['ClockPeriod'] = config['clock_period']
 
-        print(cfg)
-        firmware_model = conifer.converters.convert_from_xgboost(
-            model,
-            config=cfg
-        )
+        if config:
+            for key, value in config.items():
+                cfg[key] = value
 
-        firmware_model.write()
-        proj_name = cfg.get('ProjectName', 'my_prj')
-        bridge_file = firmware_dir / "bridge.cpp"
-        text = bridge_file.read_text()
-        text = text.replace("my_prj.h", f"{proj_name}.h")
-        bridge_file.write_text(text)
+        firmware_model = conifer.converters.convert_from_xgboost(model, config=cfg)
         firmware_model.compile()
-        if shutil.which("vivado") is not None:
-            firmware_model.build()
-        else:
-            warnings.warn("Vivado not found in PATH. Firmware build failed.", UserWarning)
-
         firmware_model.save(firmware_dir / "firmware_model.fml")
+
         return firmware_model
+
     
     def load_compiled_model(self, workspace: Path) -> Any:
         from conifer import load_model
@@ -384,33 +365,45 @@ class ModelSerializer:
             return model, input_name
         return None, None
 
+
 class TriggerModel:
-    """Main facade class that orchestrates model conversion, compilation, and inference"""
-    
-    def __init__(self, name: str, ml_backend: str, n_outputs:int, compiler: str,
-                 native_model: object, compiler_config: dict = None, scales: dict = None, unscaled_type: str = "ap_fixed<16,6>"):
-        
-        if ml_backend.lower() not in ("keras", "xgboost"):
-            raise ValueError("Only Keras or XGBoost backends are currently supported.")
-        
-        self.name = name
-        self.ml_backend = ml_backend.lower()
-        self.scales = scales
-        self.unscaled_type =  unscaled_type
-        self.n_outputs = n_outputs
-        self.compiler = compiler.lower()
+    def __init__(self, config: Union[str, Path, Dict], native_model, scales):
+        if isinstance(config, (str, Path)):
+            with open(config, "r") as f:
+                config = yaml.safe_load(f)
+        elif not isinstance(config, dict):
+            raise TypeError("config must be a dict or path to a YAML file")
+
         self.native_model = native_model
-        self.compiler_conifg = compiler_config
-        
+        self.scales = scales
+
+        self.compiler_cfg  = config.get("compiler", {})
+        self.subsystem_cfg = config.get("subsystem", {})
+
+        self.name = self.compiler_cfg.get("name", "model")
+        self.ml_backend = self.compiler_cfg.get("ml_backend", "").lower()
+        self.compiler = self.compiler_cfg.get("compiler", "").lower()
+
+        self.n_outputs = self.compiler_cfg.get("n_outputs")
+        self.unscaled_type = self.subsystem_cfg.get("unscaled_type", "ap_fixed<16,6>")
+
+        if self.ml_backend not in ("keras", "xgboost"):
+            raise ValueError("Unsupported backend")
+
         self.workspace_manager = WorkspaceManager()
-        self.converter = ConverterFactory.create_converter(ml_backend, compiler)
-        self.compiler_strategy = CompilerFactory.create_compiler(ml_backend, compiler)
-        
+        self.converter = ConverterFactory.create_converter(self.ml_backend, self.compiler)
+        self.compiler_strategy = CompilerFactory.create_compiler(self.ml_backend, self.compiler)
+
         self.firmware_model = None
         self.model_qonnx = None
         self.input_name = None
         
-        self.workspace_manager.setup_workspace(name, self.ml_backend, self.compiler)
+        
+        self.workspace_manager.setup_workspace(
+            self.name, 
+            self.ml_backend, 
+            self.compiler
+        )
     
     @property
     def workspace(self) -> Path:
@@ -427,8 +420,8 @@ class TriggerModel:
         """Get metadata dictionary"""
         return self.workspace_manager.metadata
     
-    def __call__(self, **compiler_kwargs):
-        """Execute the full model conversion and compilation pipeline"""
+    def __call__(self):
+        """Execute full model conversion and compilation pipeline using YAML config"""
         self.parse_dataset_object()
         
         # Save native model
@@ -445,22 +438,57 @@ class TriggerModel:
             self.input_name = self.model_qonnx.graph.input[0].name
             self.workspace_manager.add_artifact("qonnx", qonnx_path)
             self.workspace_manager.add_version({"qonnx": str(qonnx_path)})
+
         
         # Compile model
         self.firmware_model = self.compiler_strategy.compile(
             self.native_model,
             self.workspace_manager.workspace,
-            self.compiler_conifg, 
-            **compiler_kwargs
+            self.compiler_cfg,
+            **self.compiler_cfg.get("kwargs", {})
         )
         
         self.workspace_manager.add_artifact("firmware", self.workspace_manager.workspace / "firmware")
+        if self.compiler != "conifer" and self.scales is not None:
+            self.build_emulator(
+                self.scales['shifts'],
+                self.scales['offsets'],
+                self.n_outputs,
+                self.unscaled_type
+            )
 
-        if self.compiler is not "conifer" and self.scales is not None:
-           self.build_emulator(self.scales['shifts'], self.scales['offsets'], self.n_outputs, self.unscaled_type)
         
+        if shutil.which("vivado") is not None:
+            build_ugt_model(
+                templates_dir=self.subsystem_cfg.get("templates_dir", Path("templates")),
+                firmware_dir=self.workspace_manager.workspace / "firmware",
+                compiler = self.compiler,
+                model_name=self.name,
+                n_inputs=self.subsystem_cfg["n_inputs"],
+                n_outputs=self.subsystem_cfg.get("n_outputs", self.n_outputs),
+                nn_offsets=self.scales["offsets"],
+                nn_shifts=self.scales["shifts"],
+                muon_size=self.subsystem_cfg.get("muon_size", 0),
+                jet_size=self.subsystem_cfg.get("jet_size", 0),
+                egamma_size=self.subsystem_cfg.get("egamma_size", 0),
+                tau_size=self.subsystem_cfg.get("tau_size", 0),
+                output_type=self.subsystem_cfg.get("output_type", "result_t"),
+                offset_type=self.subsystem_cfg.get("offset_type", "ap_fixed<10,10>"),
+                shift_type=self.subsystem_cfg.get("shift_type", "ap_fixed<10,10>"),
+                object_features=self.subsystem_cfg.get("object_features"),
+                global_features=self.subsystem_cfg.get("global_features")
+            )
+        else:
+            warnings.warn(
+                "Vivado executable not found on the system PATH. "
+                "Skipping FW build. ",
+                UserWarning
+            )
+
+
         self.workspace_manager.add_artifact("firmware", self.workspace_manager.workspace / "firmware")
         self.workspace_manager.save_metadata()
+
     
     @staticmethod
     def parse_dataset_object():
@@ -482,10 +510,16 @@ class TriggerModel:
         predictor = SoftwarePredictor(self.native_model, self.ml_backend)
         return predictor.predict(input_data)
     
-    def qonnx_predict(self, input_data: np.ndarray) -> np.ndarray:
+    def qonnx_predict(self, input_data: np.ndarray) -> np.ndarray | None:
         """Make predictions using QONNX model"""
+        
         if self.model_qonnx is None:
-            raise RuntimeError("QONNX model not available")
+            warnings.warn(
+                "QONNX model is not available. Prediction skipped.",
+                UserWarning
+            )
+            return None 
+            
         predictor = QONNXPredictor(self.model_qonnx, self.input_name)
         return predictor.predict(input_data)