ber-equalization-studio 0.1.0__py3-none-any.whl

ber_equalization_studio/config.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+def _default_device() -> str:
+    try:
+        import torch
+
+        if torch.cuda.is_available():
+            return "cuda"
+        if torch.backends.mps.is_available():
+            return "mps"
+    except Exception:
+        pass
+    return "cpu"
+
+
+@dataclass(slots=True)
+class DataConfig:
+    data_dirs: list[Path] = field(
+        default_factory=lambda: [Path("symbols_new"), Path("Symbols_1m_1ch_PR"), Path(".")]
+    )
+    max_files: int = 64
+    train_portion: float = 0.97
+    val_portion_within_train: float = 0.10
+    min_val_files: int = 1
+    randomize_file_split: bool = False
+    split_seed: int = 42
+    context_k: int = 32
+    input_dim: int = 2
+    power_normalize: bool = True
+    max_test_files: int | None = None
+
+    def __post_init__(self) -> None:
+        self.data_dirs = [Path(path) for path in self.data_dirs]
+
+    @property
+    def seq_len(self) -> int:
+        return 2 * self.context_k + 1
+
+
+@dataclass(slots=True)
+class ModelConfig:
+    name: str = "complex_fastkan"
+    hidden_dim: int = 96
+    dropout: float = 0.2
+    mlp_layers: int = 3
+    tcn_hidden_dim: int = 96
+    tcn_layers: int = 5
+    tcn_kernel_size: int = 5
+    fastkan_hidden_dim: int = 96
+    fastkan_layers: int = 2
+    fastkan_num_grids: int = 8
+    fastkan_grid_min: float = -2.5
+    fastkan_grid_max: float = 2.5
+    efficient_kan_hidden_dim: int = 128
+    efficient_kan_layers: int = 2
+    efficient_kan_grid_size: int = 8
+    efficient_kan_spline_order: int = 3
+    efficient_kan_grid_range: tuple[float, float] = (-3.0, 3.0)
+    complex_light_channels: int = 48
+    complex_light_dilations: tuple[int, ...] = (1, 2, 4)
+    complex_light_kernel_size: int = 3
+    kan_prune_l1: float = 1e-5
+    kan_prune_threshold: float = 0.02
+
+
+@dataclass(slots=True)
+class TrainingConfig:
+    epochs: int = 250
+    learning_rate: float = 1e-3
+    weight_decay: float = 0.0
+    optimizer: str = "adam"
+    loss: str = "mse"
+    train_block_size: int = 8192
+    min_block_size: int = 1024
+    use_amp: bool = True
+    use_torch_compile: bool = False
+    grad_clip_norm: float = 1.0
+    lr_scheduler: str = "notebook_decay"
+    scheduler_factor: float = 0.5
+    scheduler_threshold: float = 1e-6
+    decay_steps: int = 24
+    min_lr: float = 1e-5
+    early_stopping: bool = True
+    early_stopping_patience: int = 72
+    early_stopping_min_epochs: int = 40
+    early_stopping_threshold: float = 0.0
+    save_best_by: str = "val_ber"
+    save_best: bool = True
+    log_every: int = 1
+    seed: int = 42
+
+
+@dataclass(slots=True)
+class EvaluationConfig:
+    eval_batch_size: int = 65536
+    eval_test_during_training: bool = False
+    test_ber_every: int = 10
+    compute_per_file_metrics: bool = True
+    ber_scale_search: bool = True
+    ber_scale_min: float = 0.5
+    ber_scale_max: float = 1.5
+    ber_scale_steps: int = 10
+    ber_scale_offset: int = 10000
+    ber_scale_samples: int = 1 << 20
+    efficiency_batch_size: int = 16000
+    efficiency_timing_warmup: int = 5
+    efficiency_timing_repeats: int = 20
+
+
+@dataclass(slots=True)
+class OutputConfig:
+    out_dir: Path = Path("studio_outputs")
+    experiment_name: str = "experiment"
+    save_checkpoint: bool = True
+    save_history: bool = True
+    save_plots: bool = True
+    plot_backend: str = "plotly"
+
+    def __post_init__(self) -> None:
+        self.out_dir = Path(self.out_dir)
+
+    @property
+    def run_dir(self) -> Path:
+        return self.out_dir / self.experiment_name
+
+
+@dataclass(slots=True)
+class ExperimentConfig:
+    models: list[str] = field(default_factory=lambda: ["complex_fastkan"])
+    tags: dict[str, str] = field(default_factory=dict)
+    notes: str = ""
+
+
+@dataclass(slots=True)
+class StudioConfig:
+    data: DataConfig = field(default_factory=DataConfig)
+    model: ModelConfig = field(default_factory=ModelConfig)
+    training: TrainingConfig = field(default_factory=TrainingConfig)
+    evaluation: EvaluationConfig = field(default_factory=EvaluationConfig)
+    output: OutputConfig = field(default_factory=OutputConfig)
+    experiment: ExperimentConfig = field(default_factory=ExperimentConfig)
+    device: str = field(default_factory=_default_device)
+
+    def to_dict(self) -> dict[str, Any]:
+        raw = asdict(self)
+        raw["data"]["data_dirs"] = [str(path) for path in self.data.data_dirs]
+        raw["output"]["out_dir"] = str(self.output.out_dir)
+        return raw
+
+    def with_updates(self, **updates: Any) -> "StudioConfig":
+        values = self.to_dict()
+        for dotted_key, value in updates.items():
+            section, key = dotted_key.split(".", 1)
+            values[section][key] = value
+        return StudioConfig(
+            data=DataConfig(**values["data"]),
+            model=ModelConfig(**values["model"]),
+            training=TrainingConfig(**values["training"]),
+            evaluation=EvaluationConfig(**values["evaluation"]),
+            output=OutputConfig(**values["output"]),
+            experiment=ExperimentConfig(**values["experiment"]),
+            device=values["device"],
+        )

ber_equalization_studio/data.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from ber_equalization_studio.config import StudioConfig
+from ber_equalization_studio.legacy import configured_legacy
+
+
+def prepare_dataset(config: StudioConfig, legacy_path: Path | None = None) -> dict[str, Any]:
+    with configured_legacy(config, legacy_path) as module:
+        return module.prepare_data(max_test_files=config.data.max_test_files)
+
+
+def dataset_summary(data: dict[str, Any]) -> dict[str, Any]:
+    summary: dict[str, Any] = {}
+    for split in ("train", "val", "test"):
+        x_key = f"{split}_x"
+        y_key = f"{split}_y"
+        if x_key in data:
+            summary[f"{split}_samples"] = int(data[x_key].size(0))
+        if y_key in data:
+            summary[f"{split}_targets"] = int(data[y_key].size(0))
+        spans = data.get(f"{split}_file_spans", [])
+        if spans:
+            summary[f"{split}_files"] = [int(item["file_idx"]) for item in spans]
+    for key in ("tx_scale", "rx_scale"):
+        if key in data:
+            value = data[key]
+            summary[key] = float(value.item() if hasattr(value, "item") else value)
+    return summary

ber_equalization_studio/experiment.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import copy
+import json
+import random
+from dataclasses import replace
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import torch
+
+from ber_equalization_studio.config import StudioConfig
+from ber_equalization_studio.data import dataset_summary, prepare_dataset
+from ber_equalization_studio.legacy import configured_legacy
+from ber_equalization_studio.models import canonical_model_name
+from ber_equalization_studio.results import save_json, write_result_table
+from ber_equalization_studio.visualization import plot_comparison, plot_history
+
+
+def set_reproducible_seed(seed: int) -> None:
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+
+
+class ExperimentRunner:
+    def __init__(self, config: StudioConfig, legacy_path: Path | None = None):
+        self.config = config
+        self.legacy_path = legacy_path
+        self.run_dir = config.output.run_dir
+
+    def _model_config(self, model_name: str) -> StudioConfig:
+        model = replace(self.config.model, name=canonical_model_name(model_name))
+        return replace(self.config, model=model)
+
+    def run(self) -> dict[str, Any]:
+        set_reproducible_seed(self.config.training.seed)
+        self.run_dir.mkdir(parents=True, exist_ok=True)
+        save_json(self.run_dir / "config.json", self.config.to_dict())
+
+        data = prepare_dataset(self.config, self.legacy_path)
+        save_json(self.run_dir / "dataset_summary.json", dataset_summary(data))
+
+        rows: list[dict[str, Any]] = []
+        histories: dict[str, dict[str, Any]] = {}
+        models = self.config.experiment.models or [self.config.model.name]
+
+        for requested_model_name in models:
+            model_name = canonical_model_name(requested_model_name)
+            model_config = self._model_config(model_name)
+            model_run_dir = self.run_dir / model_name
+            model_run_dir.mkdir(parents=True, exist_ok=True)
+
+            with configured_legacy(model_config, self.legacy_path) as legacy:
+                legacy.Config.OUT_DIR = model_run_dir
+                legacy.Config.OUT_DIR.mkdir(parents=True, exist_ok=True)
+                model, history, metrics = legacy.train_one_model(model_name, data)
+
+                if model_config.output.save_checkpoint:
+                    torch.save(model.state_dict(), model_run_dir / "final_state_dict.pt")
+
+            metrics = copy.deepcopy(metrics)
+            metrics["model_type"] = model_name
+            metrics["experiment_name"] = self.config.output.experiment_name
+            rows.append(metrics)
+            histories[model_name] = history
+
+            if self.config.output.save_history:
+                save_json(model_run_dir / "history.json", history)
+            save_json(model_run_dir / "metrics.json", metrics)
+            if self.config.output.save_plots:
+                plot_history(history, model_name, model_run_dir)
+
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+
+        df = write_result_table(self.run_dir / "results.csv", rows)
+        if self.config.output.save_plots:
+            plot_comparison(df, self.run_dir)
+        return {
+            "run_dir": self.run_dir,
+            "results": rows,
+            "histories": histories,
+            "results_csv": self.run_dir / "results.csv",
+        }
+
+
+def run_experiment(config: StudioConfig, legacy_path: Path | None = None) -> dict[str, Any]:
+    return ExperimentRunner(config=config, legacy_path=legacy_path).run()

ber_equalization_studio/legacy.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import importlib.util
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+from types import ModuleType
+from typing import Any, Iterator
+
+import torch
+
+from ber_equalization_studio.config import StudioConfig
+
+_LEGACY_MODULE: ModuleType | None = None
+
+
+def default_legacy_path() -> Path:
+    package_root = Path(__file__).resolve().parents[3]
+    package_dir = Path(__file__).resolve().parent
+    candidates = [
+        package_dir / "_legacy_backend" / "ber_equalization.py",
+        package_root.parent / "BER_minimization_survey" / "ber_equalization.py",
+        package_root / "BER_minimization_survey" / "ber_equalization.py",
+        Path.cwd() / "BER_minimization_survey" / "ber_equalization.py",
+        Path.cwd() / "ber_equalization.py",
+    ]
+    for path in candidates:
+        if path.exists():
+            return path
+    raise FileNotFoundError(
+        "Could not find legacy ber_equalization.py. Run from the repository root or pass legacy_path explicitly."
+    )
+
+
+def load_legacy_module(legacy_path: Path | None = None) -> ModuleType:
+    global _LEGACY_MODULE
+    if _LEGACY_MODULE is not None:
+        return _LEGACY_MODULE
+
+    path = (legacy_path or default_legacy_path()).resolve()
+    sys.path.insert(0, str(path.parent))
+    spec = importlib.util.spec_from_file_location("_ber_equalization_legacy", path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot import legacy module from {path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    _LEGACY_MODULE = module
+    return module
+
+
+def _legacy_overrides(config: StudioConfig) -> dict[str, Any]:
+    data = config.data
+    model = config.model
+    training = config.training
+    evaluation = config.evaluation
+    output = config.output
+    return {
+        "DEVICE": torch.device(config.device),
+        "DATA_DIR_CANDIDATES": [Path(path) for path in data.data_dirs],
+        "MAX_FILES": data.max_files,
+        "TRAIN_PORTION": data.train_portion,
+        "VAL_PORTION_WITHIN_TRAIN": data.val_portion_within_train,
+        "MIN_VAL_FILES": data.min_val_files,
+        "RANDOMIZE_FILE_SPLIT": data.randomize_file_split,
+        "SPLIT_SEED": data.split_seed,
+        "CONTEXT_K": data.context_k,
+        "SEQ_LEN": data.seq_len,
+        "INPUT_DIM": data.input_dim,
+        "POWER_NORMALIZE": data.power_normalize,
+        "HIDDEN_DIM": model.hidden_dim,
+        "DROPOUT": model.dropout,
+        "MLP_LAYERS": model.mlp_layers,
+        "TCN_HIDDEN_DIM": model.tcn_hidden_dim,
+        "TCN_LAYERS": model.tcn_layers,
+        "TCN_KERNEL_SIZE": model.tcn_kernel_size,
+        "FASTKAN_HIDDEN_DIM": model.fastkan_hidden_dim,
+        "FASTKAN_LAYERS": model.fastkan_layers,
+        "FASTKAN_NUM_GRIDS": model.fastkan_num_grids,
+        "FASTKAN_GRID_MIN": model.fastkan_grid_min,
+        "FASTKAN_GRID_MAX": model.fastkan_grid_max,
+        "EFFICIENT_KAN_HIDDEN_DIM": model.efficient_kan_hidden_dim,
+        "EFFICIENT_KAN_LAYERS": model.efficient_kan_layers,
+        "EFFICIENT_KAN_GRID_SIZE": model.efficient_kan_grid_size,
+        "EFFICIENT_KAN_SPLINE_ORDER": model.efficient_kan_spline_order,
+        "EFFICIENT_KAN_GRID_RANGE": list(model.efficient_kan_grid_range),
+        "COMPLEX_LIGHT_CHANNELS": model.complex_light_channels,
+        "COMPLEX_LIGHT_DILATIONS": list(model.complex_light_dilations),
+        "COMPLEX_LIGHT_KERNEL_SIZE": model.complex_light_kernel_size,
+        "KAN_PRUNE_L1": model.kan_prune_l1,
+        "KAN_PRUNE_THRESHOLD": model.kan_prune_threshold,
+        "EPOCHS": training.epochs,
+        "LEARNING_RATE": training.learning_rate,
+        "WEIGHT_DECAY": training.weight_decay,
+        "OPTIMIZER": training.optimizer,
+        "LOSS": training.loss,
+        "TRAIN_BLOCK_SIZE": training.train_block_size,
+        "MIN_BLOCK_SIZE": training.min_block_size,
+        "USE_AMP": training.use_amp,
+        "USE_TORCH_COMPILE": training.use_torch_compile,
+        "GRAD_CLIP_NORM": training.grad_clip_norm,
+        "LR_SCHEDULER": training.lr_scheduler,
+        "SCHEDULER_FACTOR": training.scheduler_factor,
+        "SCHEDULER_THRESHOLD": training.scheduler_threshold,
+        "DECAY_STEPS": training.decay_steps,
+        "MIN_LR": training.min_lr,
+        "EARLY_STOPPING": training.early_stopping,
+        "EARLY_STOPPING_PATIENCE": training.early_stopping_patience,
+        "EARLY_STOPPING_MIN_EPOCHS": training.early_stopping_min_epochs,
+        "EARLY_STOPPING_THRESHOLD": training.early_stopping_threshold,
+        "SAVE_BEST_BY": training.save_best_by,
+        "SAVE_BEST": training.save_best,
+        "LOG_EVERY": training.log_every,
+        "EVAL_BATCH_SIZE": evaluation.eval_batch_size,
+        "EVAL_TEST_DURING_TRAINING": evaluation.eval_test_during_training,
+        "TEST_BER_EVERY": evaluation.test_ber_every,
+        "COMPUTE_PER_FILE_METRICS": evaluation.compute_per_file_metrics,
+        "BER_SCALE_SEARCH": evaluation.ber_scale_search,
+        "BER_SCALE_MIN": evaluation.ber_scale_min,
+        "BER_SCALE_MAX": evaluation.ber_scale_max,
+        "BER_SCALE_STEPS": evaluation.ber_scale_steps,
+        "BER_SCALE_OFFSET": evaluation.ber_scale_offset,
+        "BER_SCALE_SAMPLES": evaluation.ber_scale_samples,
+        "EFFICIENCY_BATCH_SIZE": evaluation.efficiency_batch_size,
+        "EFFICIENCY_TIMING_WARMUP": evaluation.efficiency_timing_warmup,
+        "EFFICIENCY_TIMING_REPEATS": evaluation.efficiency_timing_repeats,
+        "OUT_DIR": output.run_dir,
+        "MODEL_TYPES": list(config.experiment.models),
+        "RUN_MAIN_EXPERIMENTS": True,
+        "RUN_SWEEP_EXPERIMENTS": False,
+        "RUN_EFFICIENT_KAN_SWEEP": False,
+        "RUN_KAN_EXPERIMENT_SUITE": False,
+        "RUN_FASTKAN_CLASSIFIER_SWEEP": False,
+    }
+
+
+@contextmanager
+def configured_legacy(config: StudioConfig, legacy_path: Path | None = None) -> Iterator[ModuleType]:
+    module = load_legacy_module(legacy_path)
+    legacy_config = module.Config
+    overrides = _legacy_overrides(config)
+    previous = {key: getattr(legacy_config, key, None) for key in overrides}
+    for key, value in overrides.items():
+        setattr(legacy_config, key, value)
+    try:
+        yield module
+    finally:
+        for key, value in previous.items():
+            setattr(legacy_config, key, value)

ber_equalization_studio/models.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import torch.nn as nn
+
+from ber_equalization_studio.config import StudioConfig
+
+
+MODEL_ALIASES: dict[str, str] = {
+    "ekan": "efficient_kan_baseline",
+    "efficient_kan": "efficient_kan_baseline",
+    "kan": "complex_fastkan",
+    "fastkan": "complex_fastkan",
+    "rbf_kan": "complex_fastkan",
+    "efficient_kan_classifier": "kan_classifier",
+    "rbf_kan_classifier": "fastkan_classifier",
+    "complex_rbf_kan_classifier": "complex_fastkan_classifier",
+}
+
+
+MODEL_NOTES: dict[str, str] = {
+    "efficient_kan_baseline": "flat IQ window -> B-spline EfficientKAN -> corrected I/Q",
+    "efficient_kan_residual": "flat IQ window -> KAN correction -> rx center + correction",
+    "efficient_kan_features": "handcrafted IQ statistics -> EfficientKAN -> corrected I/Q",
+    "cnn_kan": "temporal CNN encoder -> EfficientKAN head -> corrected I/Q",
+    "kan_classifier": "flat IQ window -> EfficientKAN -> 16 constellation logits",
+    "fastkan_classifier": "flat IQ window -> Gaussian RBF/FastKAN -> 16 logits",
+    "complex_fastkan": "lightweight complex temporal encoder -> RBF/FastKAN -> corrected I/Q",
+    "complex_fastkan_classifier": "lightweight complex temporal encoder -> RBF/FastKAN -> 16 logits",
+    "mlp": "flat IQ window -> residual MLP -> corrected I/Q",
+    "cnn": "temporal CNN encoder -> MLP head -> corrected I/Q",
+    "lstm": "LSTM over IQ window -> corrected I/Q",
+    "hybrid": "CNN front-end + LSTM -> corrected I/Q",
+    "complex_cnn": "complex feature encoder + temporal CNN -> corrected I/Q",
+    "complex_lstm": "complex feature encoder + LSTM -> corrected I/Q",
+    "complex_cnn_lstm": "complex temporal CNN + LSTM -> corrected I/Q",
+    "complex_dbp_seqstat": "learnable DBP-inspired front-end + sequence statistics",
+    "transformer": "local transformer encoder over IQ window -> corrected I/Q",
+    "tcn": "dilated temporal convolutional network -> corrected I/Q",
+    "mamba": "Mamba sequence blocks over IQ window -> corrected I/Q",
+}
+
+
+def canonical_model_name(name: str) -> str:
+    normalized = name.strip().lower()
+    return MODEL_ALIASES.get(normalized, normalized)
+
+
+def available_models() -> dict[str, str]:
+    return dict(sorted(MODEL_NOTES.items()))
+
+
+def build_model(
+    name: str,
+    config: StudioConfig | None = None,
+    legacy_path: Path | None = None,
+) -> "nn.Module":
+    """Build a model from the studio registry.
+
+    The current backend reuses the verified model implementations from
+    `ber_equalization.py`, while the public API is driven by StudioConfig.
+    """
+
+    from ber_equalization_studio.legacy import configured_legacy, load_legacy_module
+
+    model_name = canonical_model_name(name)
+    if config is None:
+        module = load_legacy_module(legacy_path)
+        return module.make_model(model_name)
+    with configured_legacy(config, legacy_path) as module:
+        return module.make_model(model_name)
+
+
+def count_parameters(model: "nn.Module") -> int:
+    return sum(parameter.numel() for parameter in model.parameters() if parameter.requires_grad)
+
+
+def describe_model(name: str) -> dict[str, Any]:
+    model_name = canonical_model_name(name)
+    return {
+        "name": model_name,
+        "description": MODEL_NOTES.get(model_name, "custom or legacy model"),
+    }

ber_equalization_studio/results.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+
+PREFERRED_COLUMNS = [
+    "model_type",
+    "equalized_ber",
+    "baseline_ber",
+    "improvement_rel",
+    "improvement_db",
+    "accuracy",
+    "ser",
+    "best_val_ber",
+    "trainable_params",
+    "batch_time_sec",
+    "samples_per_sec",
+    "train_samples_per_sec",
+    "epochs_ran",
+    "stop_reason",
+]
+
+
+def flatten_metrics(metrics: dict[str, Any]) -> dict[str, Any]:
+    flat: dict[str, Any] = {}
+    for key, value in metrics.items():
+        if isinstance(value, (str, int, float, bool)) or value is None:
+            flat[key] = value
+        elif hasattr(value, "item"):
+            flat[key] = value.item()
+        else:
+            flat[key] = json.dumps(value, ensure_ascii=True)
+    return flat
+
+
+def save_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, ensure_ascii=True), encoding="utf-8")
+
+
+def load_result_table(path: Path) -> pd.DataFrame:
+    if path.is_dir():
+        candidates = sorted(path.glob("**/results.csv"))
+        if not candidates:
+            candidates = sorted(path.glob("**/*summary*.csv"))
+        if not candidates:
+            raise FileNotFoundError(f"No result CSV files found below {path}")
+        frames = [pd.read_csv(candidate) for candidate in candidates]
+        return pd.concat(frames, ignore_index=True)
+    return pd.read_csv(path)
+
+
+def compare_results(path: Path, sort_by: str = "equalized_ber") -> pd.DataFrame:
+    df = load_result_table(path)
+    if sort_by in df.columns:
+        ascending = sort_by not in {"improvement_rel", "improvement_db", "accuracy", "samples_per_sec"}
+        df = df.sort_values(sort_by, ascending=ascending)
+    ordered = [column for column in PREFERRED_COLUMNS if column in df.columns]
+    rest = [column for column in df.columns if column not in ordered]
+    return df[ordered + rest]
+
+
+def write_result_table(path: Path, rows: list[dict[str, Any]]) -> pd.DataFrame:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    df = pd.DataFrame([flatten_metrics(row) for row in rows])
+    ordered = [column for column in PREFERRED_COLUMNS if column in df.columns]
+    rest = [column for column in df.columns if column not in ordered]
+    df = df[ordered + rest]
+    df.to_csv(path, index=False)
+    return df