pynnlf 0.2.2__py3-none-any.whl

pynnlf/hyperparams.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from typing import Any
+from .yamlio import load_yaml
+
+def load_hyperparameters(path) -> dict[str, Any]:
+    """
+    Load hyperparameters from YAML.
+
+    Args:
+        path (str | Path): Path to models/hyperparameters.yaml.
+
+    Returns:
+        dict: {model_name: {hp_no: hyperparameter_dict}}
+    """
+    return load_yaml(path)
+
+def get_hp(hparams: dict[str, Any], model_name: str, hp_no: str) -> dict[str, Any]:
+    """
+    Get hyperparameter dict for a specific model and hp_no.
+
+    Args:
+        hparams (dict): Output of load_hyperparameters().
+        model_name (str): e.g., "m6_lr" (model file stem)
+        hp_no (str): e.g., "hp1"
+
+    Returns:
+        dict: Hyperparameter dict for the model/hp.
+    """
+    if model_name not in hparams:
+        raise KeyError(f"Model '{model_name}' not found in hyperparameters.yaml")
+    if hp_no not in hparams[model_name]:
+        raise KeyError(f"HP '{hp_no}' not found for model '{model_name}'")
+    hp = hparams[model_name][hp_no]
+    if not isinstance(hp, dict):
+        raise TypeError("Hyperparameter entry must be a mapping/dict")
+    return hp

pynnlf/model_utils.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from pathlib import Path
+import json
+
+from .hyperparams import load_hyperparameters, get_hp
+from .engine import run_experiment_engine
+
+def _load_json(path: Path) -> dict:
+    """
+    Load JSON file into dict (legacy JSON workflow).
+
+    Args:
+        path (Path): JSON file path.
+
+    Returns:
+        dict: Parsed JSON.
+    """
+    return json.loads(path.read_text(encoding="utf-8"))
+
+def _workspace_root_from_spec(spec_path: Path) -> Path:
+    """
+    Infer workspace root from a spec path (legacy JSON workflow).
+
+    Assumes structure:
+        <workspace>/specs/experiment.json
+        <workspace>/specs/batch.json
+
+    Args:
+        spec_path (Path): Path to spec JSON.
+
+    Returns:
+        Path: Workspace root directory.
+    """
+    return spec_path.parent.parent
+
+def run_single(spec_path: str | Path) -> None:
+    """
+    Run a single experiment from a 4-key JSON spec (legacy workflow).
+
+    Spec contains only:
+        dataset, forecast_horizon, model, hyperparameter
+
+    Args:
+        spec_path (str | Path): Path to <workspace>/specs/experiment.json
+
+    Returns:
+        None
+    """
+    spec_path = Path(spec_path)
+    ws = _workspace_root_from_spec(spec_path)
+
+    spec = _load_json(spec_path)
+    cfg = _load_json(ws / "specs" / "pynnlf_config.json")
+
+    ds_id = spec["dataset"]
+    fh_id = spec["forecast_horizon"]
+    m_id  = spec["model"]
+    hp_no = spec["hyperparameter"]
+
+    data_dir = ws / cfg["paths"]["data_dir"]
+    out_dir  = ws / cfg["paths"]["output_dir"]
+    hp_path  = ws / cfg["paths"]["hyperparameters_path"]  # models/hyperparameters.json
+
+    dataset_file = cfg["datasets"][ds_id]
+    fh_min       = int(cfg["forecast_horizons"][fh_id])
+    model_name   = cfg["models"][m_id]
+
+    dataset_path = data_dir / dataset_file
+    models_dir   = ws / "models"
+
+    hparams = load_hyperparameters(hp_path)
+    hp = get_hp(hparams, model_name, hp_no)
+
+    run_experiment_engine(
+        dataset_path=dataset_path,
+        forecast_horizon_min=fh_min,
+        model_name=model_name,
+        hyperparameter_no=hp_no,
+        hyperparameter=hp,
+        output_dir=out_dir,
+        models_dir=models_dir,
+        config=cfg,
+    )
+
+def run_batch(spec_path: str | Path) -> None:
+    """
+    Run batch experiments from a batch JSON spec (legacy workflow).
+
+    Batch spec contains:
+        datasets: [dsX...]
+        forecast_horizons: [fhX...]
+        model_and_hp: [[mX, hpY], ...]
+
+    Runs all combinations:
+        datasets × forecast_horizons × model_and_hp
+
+    Args:
+        spec_path (str | Path): Path to <workspace>/specs/batch.json
+
+    Returns:
+        None
+    """
+    spec_path = Path(spec_path)
+    ws = _workspace_root_from_spec(spec_path)
+
+    batch = _load_json(spec_path)
+    cfg = _load_json(ws / "specs" / "pynnlf_config.json")
+
+    data_dir = ws / cfg["paths"]["data_dir"]
+    out_dir  = ws / cfg["paths"]["output_dir"]
+    hp_path  = ws / cfg["paths"]["hyperparameters_path"]
+    models_dir = ws / "models"
+
+    hparams = load_hyperparameters(hp_path)
+
+    ds_files = [cfg["datasets"][d] for d in batch["datasets"]]
+    fh_mins  = [int(cfg["forecast_horizons"][h]) for h in batch["forecast_horizons"]]
+    model_and_hp = [(cfg["models"][m], hp) for (m, hp) in batch["model_and_hp"]]
+
+    for ds_file in ds_files:
+        for fh_min in fh_mins:
+            for model_name, hp_no in model_and_hp:
+                dataset_path = data_dir / ds_file
+                hp = get_hp(hparams, model_name, hp_no)
+
+                run_experiment_engine(
+                    dataset_path=dataset_path,
+                    forecast_horizon_min=fh_min,
+                    model_name=model_name,
+                    hyperparameter_no=hp_no,
+                    hyperparameter=hp,
+                    output_dir=out_dir,
+                    models_dir=models_dir,
+                    config=cfg,
+                )
+                
+# transform below scripts into function with input train_df_y and output train_df_y_updated
+def remove_jump_df(train_df_y):
+    #make docstring with the same format like other cells
+    """
+    Remove jump in the time series data
+    Parameters:
+        train_df_y (pd.Series): Time series data
+        
+    Returns:
+        train_df_y_updated (pd.Series): Time series data with jump removed
+    """
+    
+    time_diff = train_df_y.index.to_series().diff().dt.total_seconds()
+    initial_freq = time_diff.iloc[1]
+    jump_indices = time_diff[time_diff > initial_freq].index
+    if not jump_indices.empty:
+        jump_index = jump_indices[0]
+        jump_pos = train_df_y.index.get_loc(jump_index)
+        train_df_y_updated = train_df_y.iloc[:jump_pos]
+    else:
+        train_df_y_updated = train_df_y
+    return train_df_y_updated
+
+def separate_lag_and_exogenous_features(train_df_X, target_column='y', lag_prefix='y_lag'):
+    '''
+    This function separates the lag features and exogenous variables from the training dataframe.
+
+    Args:
+        train_df_X (pd.DataFrame): The dataframe containing both lag features and exogenous variables.
+        target_column (str): The name of the target column (e.g., 'y').
+        lag_prefix (str): The prefix used for lag columns (e.g., 'y_lag').
+
+    Returns:
+        X_lags (pd.DataFrame): DataFrame containing only the lag features.
+        X_exog (pd.DataFrame): DataFrame containing only the exogenous variables.
+    '''
+    
+    # Identify lag features (columns that start with 'y_lag')
+    lag_features = [col for col in train_df_X.columns if col.startswith(lag_prefix)]
+    
+    # Identify exogenous variables (everything except the target and lag features)
+    exog_features = [col for col in train_df_X.columns if col not in [target_column] + lag_features]
+    
+    # Create dataframes for lag features and exogenous features
+    X_lags = train_df_X[lag_features]
+    X_exog = train_df_X[exog_features]
+    
+    return X_lags, X_exog

pynnlf/runner.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from pathlib import Path
+
+from .yamlio import load_yaml
+from .hyperparams import load_hyperparameters, get_hp
+from .discovery import discover_model_name, discover_dataset_path
+from .engine import run_experiment_engine
+
+def _workspace_root_from_spec(spec_path: Path) -> Path:
+    """
+    Infer workspace root from a spec path.
+
+    Args:
+        spec_path (Path): <workspace>/specs/experiment.yaml or batch.yaml
+
+    Returns:
+        Path: workspace root
+    """
+    return spec_path.parent.parent
+
+def run_single(spec_path: str | Path) -> None:
+    """
+    Run a single experiment from a 4-key YAML spec.
+
+    Args:
+        spec_path (str | Path): <workspace>/specs/experiment.yaml
+
+    Returns:
+        None
+    """
+    spec_path = Path(spec_path)
+    ws = _workspace_root_from_spec(spec_path)
+
+    spec = load_yaml(spec_path)
+    cfg = load_yaml(ws / "specs" / "pynnlf_config.yaml")
+
+    ds_id = spec["dataset"]              # e.g. ds19
+    fh_id = spec["forecast_horizon"]     # e.g. fh1
+    m_id  = spec["model"]                # e.g. m19
+    hp_no = spec["hyperparameter"]       # e.g. hp1
+
+    data_dir = ws / cfg["paths"]["data_dir"]
+    out_dir  = ws / cfg["paths"]["output_dir"]
+    models_dir = ws / "models"
+    hp_path = ws / cfg["paths"]["hyperparameters_path"]  # models/hyperparameters.yaml
+
+    # auto-discovery (no config edits)
+    dataset_path = discover_dataset_path(data_dir, ds_id)
+    model_name = discover_model_name(models_dir, m_id)
+
+    fh_min = int(cfg["forecast_horizons"][fh_id])
+
+    hparams = load_hyperparameters(hp_path)
+    hp = get_hp(hparams, model_name, hp_no)
+
+    run_experiment_engine(
+        dataset_path=dataset_path,
+        forecast_horizon_min=fh_min,
+        model_name=model_name,
+        hyperparameter_no=hp_no,
+        hyperparameter=hp,
+        output_dir=out_dir,
+        models_dir=models_dir,
+        config=cfg,
+    )
+
+def run_batch(spec_path: str | Path) -> None:
+    """
+    Run batch experiments from YAML batch spec (cartesian product).
+
+    Args:
+        spec_path (str | Path): <workspace>/specs/batch.yaml
+
+    Returns:
+        None
+    """
+    spec_path = Path(spec_path)
+    ws = _workspace_root_from_spec(spec_path)
+
+    batch = load_yaml(spec_path)
+    cfg = load_yaml(ws / "specs" / "pynnlf_config.yaml")
+
+    data_dir = ws / cfg["paths"]["data_dir"]
+    out_dir  = ws / cfg["paths"]["output_dir"]
+    models_dir = ws / "models"
+    hp_path = ws / cfg["paths"]["hyperparameters_path"]
+
+    hparams = load_hyperparameters(hp_path)
+
+    for ds_id in batch["datasets"]:
+        dataset_path = discover_dataset_path(data_dir, ds_id)
+        for fh_id in batch["forecast_horizons"]:
+            fh_min = int(cfg["forecast_horizons"][fh_id])
+            for m_id, hp_no in batch["model_and_hp"]:
+                model_name = discover_model_name(models_dir, m_id)
+                hp = get_hp(hparams, model_name, hp_no)
+                run_experiment_engine(
+                    dataset_path=dataset_path,
+                    forecast_horizon_min=fh_min,
+                    model_name=model_name,
+                    hyperparameter_no=hp_no,
+                    hyperparameter=hp,
+                    output_dir=out_dir,
+                    models_dir=models_dir,
+                    config=cfg,
+                )

pynnlf/scaffold/data/README_data.md

@@ -0,0 +1,40 @@
+# Data folder (auto-discovery)
+
+This folder is **user-editable**. You can add new datasets here without editing any package code or config mappings.
+
+## Add a new dataset (no config edits)
+
+1) Drop your CSV into this folder.
+
+2) The file name **must start** with the dataset ID prefix, e.g.:
+
+- `ds19_mydata.csv` (example)
+
+3) Auto-discovery uses the prefix (`ds19`) to find the dataset file.
+
+## Dataset requirements (minimum)
+
+- CSV must have a **datetime index** in the first column (same style as `ds0_test.csv`).
+- CSV must contain a target column named `netload_kW` (the engine renames it to `y`).
+
+## Run the dataset (4-key spec only)
+
+Edit `../specs/experiment.yaml`:
+
+```yaml
+dataset: ds19
+forecast_horizon: fh1
+model: m6
+hyperparameter: hp1
+```
+
+Then run:
+
+```bash
+python -c "import pynnlf; pynnlf.run_experiment('PATH_TO_WORKSPACE/specs/experiment.yaml')"
+```
+
+## Gotchas
+
+- Auto-discovery requires **exactly one** match for a dataset ID. If you have `ds19_a.csv` and `ds19_b.csv`, the runner will raise an error asking you to rename.
+- If your file is huge, keep it out of Git history (use `.gitignore`).