triggerflow 0.3.3__tar.gz → 0.3.6__tar.gz

{triggerflow-0.3.3 → triggerflow-0.3.6}/PKG-INFO

@@ -1,48 +1,62 @@
 Metadata-Version: 2.4
 Name: triggerflow
-Version: 0.3.3
+Version: 0.3.6
 Summary: Utilities for ML models targeting hardware triggers
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.11
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: cookiecutter>=2.3
 Requires-Dist: PyYAML>=6
 Requires-Dist: Jinja2>=3
-Requires-Dist: mlflow==2.21.0
 Requires-Dist: kedro==1.0.0
+Requires-Dist: kedro-datasets
+Requires-Dist: kedro-mlflow==2.0.1
+Requires-Dist: awkward<3,>=2.8
+Requires-Dist: dask==2025.3.0
+Requires-Dist: coffea>=2025.12
+Requires-Dist: distributed==2025.3.0
 Provides-Extra: dev
 Requires-Dist: pytest-cov~=3.0; extra == "dev"
 Requires-Dist: pytest-mock<2.0,>=1.7.1; extra == "dev"
 Requires-Dist: pytest~=7.2; extra == "dev"
 Requires-Dist: ruff~=0.1.8; extra == "dev"
-Provides-Extra: extended
-Requires-Dist: coffea; extra == "extended"
-Requires-Dist: dask; extra == "extended"
-Requires-Dist: pyarrow; extra == "extended"
-Requires-Dist: pandas==2.0.3; extra == "extended"
-Requires-Dist: uproot4==4.0.0; extra == "extended"
-Requires-Dist: pyarrow==17.0.0; extra == "extended"
-Requires-Dist: hist==2.7.3; extra == "extended"
-Requires-Dist: mlflow==2.21.0; extra == "extended"
-Requires-Dist: tensorflow==2.13.0; extra == "extended"
-Requires-Dist: QKeras==0.9.0; extra == "extended"
-Requires-Dist: keras==2.13.1; extra == "extended"
-Requires-Dist: numpy; extra == "extended"
-Requires-Dist: pyparsing; extra == "extended"
-Requires-Dist: PyYAML; extra == "extended"
-Requires-Dist: matplotlib; extra == "extended"
-Requires-Dist: mplhep; extra == "extended"
-Requires-Dist: h5py; extra == "extended"
-Requires-Dist: xxhash; extra == "extended"
-Requires-Dist: shap; extra == "extended"
-Requires-Dist: awkward-pandas; extra == "extended"
-Requires-Dist: qonnx==0.4.0; extra == "extended"
-Requires-Dist: tf_keras; extra == "extended"
-Requires-Dist: tf2onnx; extra == "extended"
-Requires-Dist: hls4ml; extra == "extended"
-Requires-Dist: conifer; extra == "extended"
+Provides-Extra: qkeras
+Requires-Dist: tensorflow==2.13.0; extra == "qkeras"
+Requires-Dist: QKeras==0.9.0; extra == "qkeras"
+Requires-Dist: keras==2.13.1; extra == "qkeras"
+Requires-Dist: qonnx==0.4.0; extra == "qkeras"
+Requires-Dist: tf_keras; extra == "qkeras"
+Requires-Dist: tf2onnx; extra == "qkeras"
+Requires-Dist: hls4ml; extra == "qkeras"
+Requires-Dist: conifer; extra == "qkeras"
+Requires-Dist: coffea; extra == "qkeras"
+Requires-Dist: dask; extra == "qkeras"
+Requires-Dist: pyarrow==17.0.0; extra == "qkeras"
+Requires-Dist: pandas==2.0.3; extra == "qkeras"
+Requires-Dist: uproot==4.0.0; extra == "qkeras"
+Requires-Dist: hist==2.7.3; extra == "qkeras"
+Requires-Dist: awkward-pandas; extra == "qkeras"
+Requires-Dist: matplotlib; extra == "qkeras"
+Requires-Dist: mplhep; extra == "qkeras"
+Requires-Dist: h5py; extra == "qkeras"
+Requires-Dist: xxhash; extra == "qkeras"
+Requires-Dist: shap; extra == "qkeras"
+Requires-Dist: numpy==1.24.3; extra == "qkeras"
+Provides-Extra: keras3
+Requires-Dist: keras==3.11.0; extra == "keras3"
+Requires-Dist: tensorflow==2.16.1; extra == "keras3"
+Requires-Dist: hgq2; extra == "keras3"
+Requires-Dist: hls4ml; extra == "keras3"
+Requires-Dist: conifer; extra == "keras3"
+Requires-Dist: pyarrow==17.0.0; extra == "keras3"
+Requires-Dist: pandas==2.0.3; extra == "keras3"
+Requires-Dist: uproot==4.0.0; extra == "keras3"
+Requires-Dist: hist==2.7.3; extra == "keras3"
+Requires-Dist: awkward-pandas; extra == "keras3"
+Requires-Dist: matplotlib; extra == "keras3"
+Requires-Dist: mplhep; extra == "keras3"
+Requires-Dist: h5py; extra == "keras3"
+Requires-Dist: xxhash; extra == "keras3"
+Requires-Dist: shap; extra == "keras3"
 
 # Machine Learning for Hardware Triggers
 

{triggerflow-0.3.3 → triggerflow-0.3.6}/pyproject.toml

@@ -4,63 +4,73 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "triggerflow"
-version = "0.3.3"
+version = "0.3.6"
 description = "Utilities for ML models targeting hardware triggers"
 readme = "README.md"
-requires-python = ">=3.11"
+requires-python = ">=3.10"
 dependencies = [
     "cookiecutter>=2.3",
     "PyYAML>=6",
     "Jinja2>=3",
-    "mlflow==2.21.0",
-    "kedro==1.0.0"
-]
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent"
+    "kedro==1.0.0",
+    "kedro-datasets",
+    "kedro-mlflow==2.0.1",
+    "awkward>=2.8,<3",
+    "dask==2025.3.0",
+    "coffea>=2025.12",
+    "distributed==2025.3.0"
 ]
 
 [project.optional-dependencies]
 dev = [
     "pytest-cov~=3.0",
     "pytest-mock>=1.7.1, <2.0",
-    "pytest~=7.2", "ruff~=0.1.8",
+    "pytest~=7.2",
+    "ruff~=0.1.8",
 ]
 
-extended = [
-    # Conda-equivalent packages (can often be installed via pip)
+qkeras = [
+    "tensorflow==2.13.0",
+    "QKeras==0.9.0",
+    "keras==2.13.1",
+    "qonnx==0.4.0",
+    "tf_keras",
+    "tf2onnx",
+    "hls4ml",
+    "conifer",
+    # Physics/Data tools included in this flow
     "coffea",
     "dask",
-    "pyarrow",
-    
-    # Explicitly Versioned Packages
-    "pandas==2.0.3",
-    "uproot4==4.0.0",
     "pyarrow==17.0.0",
+    "pandas==2.0.3",
+    "uproot==4.0.0",
     "hist==2.7.3",
-    "mlflow==2.21.0",
-    "tensorflow==2.13.0",
-    "QKeras==0.9.0",
-    "keras==2.13.1",
-    
-    # Packages with required versions/source links
-    "numpy",
-    "pyparsing",
-    "PyYAML", 
+    "awkward-pandas",
     "matplotlib",
     "mplhep",
     "h5py",
     "xxhash",
     "shap",
-    "awkward-pandas",
-    "qonnx==0.4.0",
-    "tf_keras",
-    "tf2onnx",
-    "hls4ml",
-    "conifer"
+    "numpy==1.24.3"
 ]
 
+keras3 = [
+    "keras==3.11.0",
+    "tensorflow==2.16.1",
+    "hgq2",
+    "hls4ml",
+    "conifer",
+    "pyarrow==17.0.0",
+    "pandas==2.0.3",
+    "uproot==4.0.0",
+    "hist==2.7.3",
+    "awkward-pandas",
+    "matplotlib",
+    "mplhep",
+    "h5py",
+    "xxhash",
+    "shap",
+]
 
 [tool.setuptools]
 include-package-data = true
@@ -78,6 +88,5 @@ select = [ "F", "W", "E", "I", "UP", "PL", "T201",]
 ignore = [ "E501",]
 extend-exclude = ["src/triggerflow/starter"]
 
-# expose CLI entrypoint
 [project.scripts]
 triggerflow = "triggerflow.cli:main"

{triggerflow-0.3.3 → triggerflow-0.3.6}/src/trigger_loader/loader.py

@@ -46,16 +46,16 @@ class TriggerLoader:
     def _load_sample_json(self, sample_json: str) -> dict:
         """
         Loads the JSON and resolves file paths using the priority:
-        1. Explicit 'files' list or directory path (Local/Explicit)
-        2. 'DAS' query (Remote Fallback)
+        1. 'files' list or path (local)
+        2. 'DAS' query (remote fallback)
 
-        Returns the canonical coffea fileset format: {dataset_name: [file_path_list]}.
+        Returns canonical coffea fileset: {dataset_name: [file_path_list]}.
         """
         import glob
         import os
-        
-        # Helper function definition needed here if it's not imported:
-        # def _fetch_files_from_das(das_query: str) -> list[str]: ... (placeholder or actual implementation)
+
+        # Placeholder for DAS fetching if needed
+        # def _fetch_files_from_das(das_query: str) -> list[str]: ...
 
         with open(sample_json) as f:
             full_data = json.load(f)
@@ -64,23 +64,31 @@ class TriggerLoader:
         fileset, meta_data = {}, {}
         for ds_name, ds_info in dataset_metadata.items():
             files = []
+
             if "files" in ds_info:
                 file_info = ds_info["files"]
+
                 if isinstance(file_info, list):
-                    files = file_info
+                    files = []
+                    for f in file_info:
+                        if os.path.isdir(f):
+                            files.extend(glob.glob(os.path.join(f, "*.root")))
+                        elif os.path.isfile(f):
+                            files.append(f)
+
                 elif isinstance(file_info, str):
                     if os.path.isdir(file_info):
-                        path_glob = os.path.join(file_info, "*.root")
-                        files = glob.glob(path_glob)
-                        self.logger.info(f"Resolved {len(files)} files from directory {file_info}.")
-                    else:
+                        files = glob.glob(os.path.join(file_info, "*.root"))
+                    elif os.path.isfile(file_info):
                         files = [file_info]
+
                 if files:
                     self.logger.info(f"Using {len(files)} local/explicit files for {ds_name}.")
 
+            # Fallback to DAS if no local files found
             if not files and "DAS" in ds_info:
                 try:
-                    files = _fetch_files_from_das(ds_info["DAS"]) 
+                    files = _fetch_files_from_das(ds_info["DAS"])
                     self.logger.info(f"Resolved {len(files)} files via DAS for {ds_name}.")
                 except NameError:
                     self.logger.info("DAS fetching skipped: _fetch_files_from_das is not defined.")
@@ -90,10 +98,14 @@ class TriggerLoader:
                 continue
 
             fileset[ds_name] = files
-            meta_data[ds_name] = {"files": files, "is_signal": ds_info["is_signal"]}
+            meta_data[ds_name] = {
+                "files": files,
+                "is_signal": ds_info.get("is_signal", False)
+            }
 
         return fileset, meta_data
 
+
     def _write_run_metadata_file(self, path: str, duration_s: float | None = None):
         meta_path = f"{path}/run_metadata.json"
         data = {

{triggerflow-0.3.3 → triggerflow-0.3.6}/src/trigger_loader/processor.py

@@ -164,7 +164,7 @@ class TriggerProcessor(processor.ProcessorABC):
             "entrystop": event_meta["stop"],
             "n_events_written": len(table),
             "columns": table.column_names,
-            "created_utc": dt.datetime.now(dt.UTC).isoformat(timespec="seconds") + "Z",
+            "created_utc": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") + "Z",
             "compression": self.compression,
             "processing_time_s": round(elapsed_s, 6),
         }

{triggerflow-0.3.3 → triggerflow-0.3.6}/src/triggerflow/core.py

@@ -9,7 +9,8 @@ from typing import Optional, Dict, Any, Union
 import shutil, warnings
 import importlib.resources as pkg_resources
 import triggerflow.templates 
-from triggerflow.interfaces.uGT import build_ugt_model
+from importlib import import_module
+from datetime import datetime
 
 
 class ModelConverter(ABC):
@@ -265,21 +266,19 @@ class WorkspaceManager:
     def __init__(self):
         self.workspace = self.BASE_WORKSPACE
         self.artifacts = {"firmware": None}
-        self.metadata = {
-            "name": None,
-            "ml_backend": None,
-            "compiler": None,
-            "versions": []
-        }
+        self.metadata = {}
     
-    def setup_workspace(self, name: str, ml_backend: str, compiler: str):
-        """Initialize workspace and metadata"""
+    def setup_workspace(self, config: Dict):
+        """Initialize workspace and save metadata"""
         self.workspace.mkdir(parents=True, exist_ok=True)
-        self.metadata.update({
-            "name": name,
-            "ml_backend": ml_backend,
-            "compiler": compiler,
-        })
+        
+        # Store config + timestamp as metadata
+        self.metadata = {
+            "config": config,
+            "created_at": datetime.now().isoformat()
+        }
+        
+        self.save_metadata()
     
     def save_native_model(self, model, ml_backend: str):
         """Save the native model to workspace"""
@@ -292,18 +291,10 @@ class WorkspaceManager:
         """Add artifact to tracking"""
         self.artifacts[key] = value
     
-    def add_version(self, version_info: Dict):
-        """Add version information"""
-        self.metadata["versions"].append(version_info)
-    
     def save_metadata(self):
         """Save metadata to file"""
         with open(self.workspace / "metadata.json", "w") as f:
-            json.dump({
-                "name": self.metadata["name"],
-                "ml_backend": self.metadata["ml_backend"],
-                "compiler": self.metadata["compiler"],
-            }, f, indent=2)
+            json.dump(self.metadata, f, indent=2)
 
 
 class ModelSerializer:
@@ -394,6 +385,7 @@ class TriggerModel:
 
         self.native_model = native_model
         self.scales = scales
+        self.config = config  # Store the full config
 
         self.compiler_cfg  = config.get("compiler", {})
         self.subsystem_cfg = config.get("subsystem", {})
@@ -404,6 +396,8 @@ class TriggerModel:
 
         self.n_outputs = self.compiler_cfg.get("n_outputs")
         self.unscaled_type = self.subsystem_cfg.get("unscaled_type", "ap_fixed<16,6>")
+        self.namespace = self.compiler_cfg.get("namespace", "triggerflow")
+        self.project_name = self.compiler_cfg.get("project_name", "triggerflow")
 
         if self.ml_backend not in ("keras", "xgboost"):
             raise ValueError("Unsupported backend")
@@ -416,13 +410,8 @@ class TriggerModel:
         self.model_qonnx = None
         self.input_name = None
         
-        
-        self.workspace_manager.setup_workspace(
-            self.name, 
-            self.ml_backend, 
-            self.compiler
-        )
-    
+        self.workspace_manager.setup_workspace(self.config)
+
     @property
     def workspace(self) -> Path:
         """Get workspace path"""
@@ -455,7 +444,6 @@ class TriggerModel:
             qonnx_path, self.model_qonnx = conversion_result
             self.input_name = self.model_qonnx.graph.input[0].name
             self.workspace_manager.add_artifact("qonnx", qonnx_path)
-            self.workspace_manager.add_version({"qonnx": str(qonnx_path)})
 
         
         # Compile model
@@ -475,27 +463,20 @@ class TriggerModel:
                 self.unscaled_type
             )
 
+        subsystem_name = self.subsystem_cfg.get("name", "uGT")
+        interface_module = import_module(f"triggerflow.interfaces.{subsystem_name}")
+        build_firmware = getattr(interface_module, f"build_{subsystem_name.lower()}_model")
+
+        build_firmware(
+            subsystem_cfg=self.subsystem_cfg,
+            compiler_cfg=self.compiler_cfg,
+            workspace_manager=self.workspace_manager,
+            compiler=self.compiler,
+            scales=self.scales,
+            name=self.name,
+            n_outputs=self.n_outputs
+        )
         
-        build_ugt_model(
-                templates_dir=self.subsystem_cfg.get("templates_dir", Path("templates")),
-                firmware_dir=self.workspace_manager.workspace / "firmware",
-                compiler = self.compiler,
-                model_name=self.name,
-                n_inputs=self.subsystem_cfg["n_inputs"],
-                n_outputs=self.subsystem_cfg.get("n_outputs", self.n_outputs),
-                nn_offsets=self.scales["offsets"],
-                nn_shifts=self.scales["shifts"],
-                muon_size=self.subsystem_cfg.get("muon_size", 0),
-                jet_size=self.subsystem_cfg.get("jet_size", 0),
-                egamma_size=self.subsystem_cfg.get("egamma_size", 0),
-                tau_size=self.subsystem_cfg.get("tau_size", 0),
-                output_type=self.subsystem_cfg.get("output_type", "result_t"),
-                offset_type=self.subsystem_cfg.get("offset_type", "ap_fixed<10,10>"),
-                shift_type=self.subsystem_cfg.get("shift_type", "ap_fixed<10,10>"),
-                object_features=self.subsystem_cfg.get("object_features"),
-                global_features=self.subsystem_cfg.get("global_features")
-            )
-  
 
 
         self.workspace_manager.add_artifact("firmware", self.workspace_manager.workspace / "firmware")
@@ -563,6 +544,8 @@ class TriggerModel:
                 "AD_SHIFT": ", ".join(map(str, ad_shift)),
                 "AD_OFFSETS": ", ".join(map(str, ad_offsets)),
                 "UNSCALED_TYPE": unscaled_type,  
+                "NAMESPACE": self.namespace, 
+                "PROJECT_NAME": self.project_name,  
             }
             self._render_template(scales_template_path, scales_out_path, context)
 

{triggerflow-0.3.3 → triggerflow-0.3.6}/src/triggerflow/interfaces/uGT.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-import shutil, warnings
+import shutil, warnings, subprocess
 import pkg_resources
 from jinja2 import Template
 import re
@@ -7,7 +7,7 @@ import re
 
 def _render_template(template_name: str, output_file: Path, context: dict):
     template_bytes = pkg_resources.resource_string(
-        "triggerflow" , template_name
+        "triggerflow", template_name
     )
     template_text = template_bytes.decode('utf-8')
 
@@ -17,11 +17,62 @@ def _render_template(template_name: str, output_file: Path, context: dict):
     with open(output_file, "w") as f:
         f.write(rendered)
 
+
 def build_ugt_model(
+    subsystem_cfg: dict,
+    compiler_cfg: dict,
+    workspace_manager,
+    compiler,
+    scales: dict,
+    name: str,
+    n_outputs: int
+):
+    
+    firmware_dir = workspace_manager.workspace / "firmware"
+    templates_dir = subsystem_cfg.get("templates_dir", Path("templates"))
+    
+    objects = subsystem_cfg.get("objects", {})
+
+    object_features = {}
+
+    for object_name, object_config in objects.items():
+        object_features[object_name] = object_config.get("features", [])
+    
+    muon_size = subsystem_cfg.get("muon_size", 0)
+    jet_size = subsystem_cfg.get("jet_size", 0)
+    egamma_size = subsystem_cfg.get("egamma_size", 0)
+    tau_size = subsystem_cfg.get("tau_size", 0)
+    
+    uGT(
+        templates_dir=templates_dir,
+        firmware_dir=firmware_dir,
+        compiler=compiler,
+        model_name=name,
+        namespace=compiler_cfg.get("namespace", "triggerflow"),
+        project_name=compiler_cfg.get("project_name", "triggerflow"),
+        n_inputs=subsystem_cfg["n_inputs"],
+        n_outputs=subsystem_cfg.get("n_outputs", n_outputs),
+        nn_offsets=scales["offsets"],
+        nn_shifts=scales["shifts"],
+        muon_size=muon_size,
+        jet_size=jet_size,
+        egamma_size=egamma_size,
+        tau_size=tau_size,
+        output_type=subsystem_cfg.get("output_type", "result_t"),
+        offset_type=subsystem_cfg.get("offset_type", "ap_fixed<10,10>"),
+        shift_type=subsystem_cfg.get("shift_type", "ap_fixed<10,10>"),
+        object_features=object_features,
+        global_features=subsystem_cfg.get("global_features")
+    )
+
+
+def uGT(
     templates_dir: Path,
     firmware_dir: Path,
     compiler: str,
     model_name: str,
+    namespace: str,
+    project_name: str,
     n_inputs: int,
     n_outputs: int,
     nn_offsets: list,
@@ -40,7 +91,6 @@ def build_ugt_model(
     Render uGT top func.
     """
 
-
     if object_features is None:
         object_features = {
             "muons": ["pt", "eta_extrapolated", "phi_extrapolated"],
@@ -84,29 +134,31 @@ def build_ugt_model(
 
         
     context = {
-    "MODEL_NAME": model_name,
-    "N_INPUTS": n_inputs,
-    "N_OUTPUTS": n_outputs,
-    "NN_OFFSETS": ", ".join(map(str, nn_offsets)),
-    "NN_SHIFTS": ", ".join(map(str, nn_shifts)),
-    "MUON_SIZE": muon_size,
-    "JET_SIZE": jet_size,
-    "EGAMMA_SIZE": egamma_size,
-    "TAU_SIZE": tau_size,
-    "OUTPUT_TYPE": output_type,
-    "OUTPUT_LAYER": output_layer,
-    "OUT": out,
-    "OFFSET_TYPE": offset_type,
-    "SHIFT_TYPE": shift_type,
-    "MUON_FEATURES": object_features["muons"],
-    "JET_FEATURES": object_features["jets"],
-    "EGAMMA_FEATURES": object_features["egammas"],
-    "TAU_FEATURES": object_features["taus"],
-    "GLOBAL_FEATURES": global_features
+        "MODEL_NAME": model_name,
+        "NAMESPACE": namespace,
+        "PROJECT_NAME": project_name,
+        "N_INPUTS": n_inputs,
+        "N_OUTPUTS": n_outputs,
+        "NN_OFFSETS": ", ".join(map(str, nn_offsets)),
+        "NN_SHIFTS": ", ".join(map(str, nn_shifts)),
+        "MUON_SIZE": muon_size,
+        "JET_SIZE": jet_size,
+        "EGAMMA_SIZE": egamma_size,
+        "TAU_SIZE": tau_size,
+        "OUTPUT_TYPE": output_type,
+        "OUTPUT_LAYER": output_layer,
+        "OUT": out,
+        "OFFSET_TYPE": offset_type,
+        "SHIFT_TYPE": shift_type,
+        "MUON_FEATURES": object_features["muons"],
+        "JET_FEATURES": object_features["jets"],
+        "EGAMMA_FEATURES": object_features["egammas"],
+        "TAU_FEATURES": object_features["taus"],
+        "GLOBAL_FEATURES": global_features
     }
 
     context_tcl = {
-    "MODEL_NAME": model_name,
+        "MODEL_NAME": model_name,
     }
 
     out_path = firmware_dir / "firmware/model-gt.cpp"
@@ -128,12 +180,8 @@ def build_ugt_model(
             check=True
         )
     else:
-            warnings.warn(
-                "Vivado executable not found on the system PATH. "
-                "Skipping FW build. ",
-                UserWarning
-            )
-
-
-
-
+        warnings.warn(
+            "Vivado executable not found on the system PATH. "
+            "Skipping FW build. ",
+            UserWarning
+        )

{triggerflow-0.3.3 → triggerflow-0.3.6}/src/triggerflow/starter/{{ cookiecutter.repo_name }}/data/01_raw/samples.json

@@ -1,9 +1,7 @@
 {
     "samples" : {
         "signal" : {
-            "all_file_path":"data/01_raw/test.root",
             "files":"data/01_raw/test.root",
-            "path":"data/01_raw/test.root",
             "file_pattern":["test.root"],
             "DAS" : "Blabla",
             "type" : "123",
@@ -13,9 +11,7 @@
             "is_signal": true
         },
         "background" : {
-            "all_file_path":"data/01_raw/test.root",
             "files":"data/01_raw/test.root",
-            "path":"data/01_raw/test.root",
             "file_pattern":["test.root"],
             "DAS" : "Blabla",
             "type" : "123",

triggerflow-0.3.6/src/triggerflow/starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/datasets/meta_dataset.py

@@ -0,0 +1,49 @@
+import logging, json
+from glob import glob
+from kedro.io import AbstractDataset
+
+METADATA_CONFIG = {"x": 0}
+
+class MetaDataset(AbstractDataset):
+    def __init__(self, filepath: str, sample_key: str):
+        self._filepath = filepath
+        self._sample_key = sample_key
+        self.logger = logging.getLogger(__name__)
+
+    def get_dasgoclient_metadata(self, das_name: str, config: dict) -> dict:
+        self.logger.info(f"Fetching DAS metadata for dataset: {das_name}")
+        return {"gridpack": "0.0.0"}
+
+    def _load(self) -> dict:
+        self.logger.info(f"Processing file: {self._filepath}")
+        with open(self._filepath, "r") as f:
+            data = json.load(f)
+        return data
+
+    def _save(self, samples: dict) -> None:
+        metadata = {}
+
+        dataset_content = samples.get(self._sample_key, {})
+        
+        for sample_name, sample_info in dataset_content.items():
+            self.logger.info(f"Processing sample: {sample_name}")
+
+            sample_files = sample_info.get("files", "")
+            
+            resolved_files = glob(sample_files) if isinstance(sample_files, str) else sample_files
+            sample_info["files"] = resolved_files
+            
+            self.logger.info(f"Found {len(resolved_files)} files for {sample_name}.")
+
+            metadata[sample_name] = self.get_dasgoclient_metadata(
+                sample_info.get("DAS", "Unknown"), 
+                METADATA_CONFIG
+            )
+
+        with open(self._filepath, "w") as f:
+            json.dump(metadata, f, indent=4)
+
+    def _describe(self) -> dict:
+        return {"filepath": self._filepath, "sample_key": self._sample_key}
+    
+