triggerflow 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

trigger_loader/cluster_manager.py

@@ -4,9 +4,6 @@ import logging
 from typing import Any
 
 from dask.distributed import Client, LocalCluster
-from dask_cuda import LocalCUDACluster
-from dask_jobqueue import HTCondorCluster
-from dask_kubernetes import KubeCluster
 
 logger = logging.getLogger(__name__)
 
@@ -63,15 +60,18 @@ class ClusterManager:
             self.cluster = LocalCluster(**self.cluster_config)
 
         elif ct == "condor":
+            from dask_jobqueue import HTCondorCluster
             self.cluster = HTCondorCluster(**self.cluster_config)
             if self.jobs and self.jobs > 0:
                 # Scale to the requested number of jobs
                 self.cluster.scale(jobs=self.jobs)
 
         elif ct == "cuda":
+            from dask_cuda import LocalCUDACluster
             self.cluster = LocalCUDACluster(**self.cluster_config)
 
         elif ct == "kubernetes":
+            from dask_kubernetes import KubeCluster
             self.cluster = KubeCluster(**self.cluster_config)
             if self.jobs and self.jobs > 0:
                 try:

trigger_loader/loader.py

@@ -45,8 +45,58 @@ class TriggerLoader:
         )
 
     def _load_sample_json(self, sample_json: str) -> dict:
+        """
+        Loads the JSON and resolves file paths using the priority:
+        1. Explicit 'files' list or directory path (Local/Explicit)
+        2. 'DAS' query (Remote Fallback)
+
+        Returns the canonical coffea fileset format: {dataset_name: [file_path_list]}.
+        """
+        import glob
+        import os
+        
+        # Helper function definition needed here if it's not imported:
+        # def _fetch_files_from_das(das_query: str) -> list[str]: ... (placeholder or actual implementation)
+
         with open(sample_json) as f:
-            return json.load(f)
+            full_data = json.load(f)
+            dataset_metadata = full_data.get("samples", full_data)
+
+        fileset = {}
+        for ds_name, ds_info in dataset_metadata.items():
+            files = []
+            
+            if "files" in ds_info:
+                file_info = ds_info["files"]
+                
+                if isinstance(file_info, list):
+                    files = file_info
+                    
+                elif isinstance(file_info, str):
+                    if os.path.isdir(file_info):
+                        path_glob = os.path.join(file_info, "*.root")
+                        files = glob.glob(path_glob)
+                        logger.info(f"Resolved {len(files)} files from directory {file_info}.")
+                    else:
+                        files = [file_info]
+                
+                if files:
+                    logger.info(f"Using {len(files)} local/explicit files for {ds_name}.")
+ 
+            if not files and "DAS" in ds_info:
+                try:
+                    files = _fetch_files_from_das(ds_info["DAS"]) 
+                    logger.info(f"Resolved {len(files)} files via DAS for {ds_name}.")
+                except NameError:
+                    logger.error("DAS fetching skipped: _fetch_files_from_das is not defined.")
+
+            if not files:
+                logger.warning(f"No files found for dataset: {ds_name}. Skipping.")
+                continue
+
+            fileset[ds_name] = files
+
+        return fileset
 
     def _write_run_metadata_file(self, path: str, duration_s: float | None = None):
         meta_path = f"{path}/run_metadata.json"
@@ -58,9 +108,11 @@ class TriggerLoader:
             json.dump(data, f, indent=2)
 
     def _run(self, runner: processor.Runner, label: str):
-        logger.log(f"Starting processing ({label})...")
+        logger.log(logging.INFO, f"Starting processing ({label})...")
         start = time.time()
         proc = self._build_processor()
+        print(self.fileset)
+        
         acc = runner(
             self.fileset,
             treename="Events",
@@ -68,7 +120,7 @@ class TriggerLoader:
         )
         elapsed = time.time() - start
         self._write_run_metadata_file(self.output_path, elapsed)
-        logger.log(f"Finished in {elapsed:.2f}s (run_uuid={self.run_uuid})")
+        logger.log(logging.INFO, f"Finished in {elapsed:.2f}s (run_uuid={self.run_uuid})")
         return acc
 
     def run_distributed(self, cluster_type: str, cluster_config: dict,

{triggerflow-0.2.1.dist-info → triggerflow-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: triggerflow
-Version: 0.2.1
+Version: 0.2.2
 Summary: Utilities for ML models targeting hardware triggers
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License

{triggerflow-0.2.1.dist-info → triggerflow-0.2.2.dist-info}/RECORD

@@ -1,8 +1,8 @@
 trigger_dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trigger_dataset/core.py,sha256=ZX96U6rWxxfCatDQbst6IRZvtlyDj1_2JA7stPydGTQ,2645
 trigger_loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trigger_loader/cluster_manager.py,sha256=n_HgphyNrNzSsCXOnnV02QMd1mR_HLEdlaBy4ClhIF8,3581
-trigger_loader/loader.py,sha256=pLk4KENBGaviZcRuA-XdMQQiNdNU93KBitXx2YbqM8Q,3112
+trigger_loader/cluster_manager.py,sha256=XgmY1xeW8zrpQDJqssKamWzjn6TQ60NGNzpcdZwL6NE,3617
+trigger_loader/loader.py,sha256=wMkeZ3k36wpxt-B8OpKOa6j7z0-fnJUqQ-5AbVjNpBM,5158
 trigger_loader/processor.py,sha256=cvBfYmvcr4FLzOHgGE50oy7EkFzFaV80Z_66amqfsEY,7724
 triggerflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 triggerflow/cli.py,sha256=ZNQb3XQN8Ir6Hp6KX_ugec9bm2kqxLNZ0KdVGJmnqFA,4498
@@ -90,8 +90,8 @@ triggerflow/templates/makefile,sha256=A-aetsLC51Bop0T_-yPY8Z8Hg29ApN4YPvKx_jjPuH
 triggerflow/templates/makefile_version,sha256=6kFc_u2oiM9l2rH7RK_BLzdZu1ZEK8PQTQKGBLRY0v4,328
 triggerflow/templates/model_template.cpp,sha256=jMNRcO7NgC6I9Wd2BV3Bim-P1qPsAl_oeVQ8KofQGEw,1807
 triggerflow/templates/scales.h,sha256=MFcB5S0DEvfzHuUhyZqILR0O4ktugOG-fLnuCDUUewM,373
-triggerflow-0.2.1.dist-info/METADATA,sha256=U4Sh7vfO-SWMUN0a3Yer08Le8R8U7XHLa4vxPMRK7t8,2921
-triggerflow-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-triggerflow-0.2.1.dist-info/entry_points.txt,sha256=5QSV9YDseB_FqgVh9q10BdL4b1I6t68rGwPLXgVL60g,53
-triggerflow-0.2.1.dist-info/top_level.txt,sha256=cX0jkuM9tfxGp002ZBQ1AYgx-6D_NgBtomgPL0WA9bE,43
-triggerflow-0.2.1.dist-info/RECORD,,
+triggerflow-0.2.2.dist-info/METADATA,sha256=08dT-O69f0YlTIWRjJHgYg5bE6jTPBovQmcq6vidK9M,2921
+triggerflow-0.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+triggerflow-0.2.2.dist-info/entry_points.txt,sha256=5QSV9YDseB_FqgVh9q10BdL4b1I6t68rGwPLXgVL60g,53
+triggerflow-0.2.2.dist-info/top_level.txt,sha256=cX0jkuM9tfxGp002ZBQ1AYgx-6D_NgBtomgPL0WA9bE,43
+triggerflow-0.2.2.dist-info/RECORD,,