datatunner 3.2.0__tar.gz → 3.2.2__tar.gz

{datatunner-3.2.0/datatunner.egg-info → datatunner-3.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatunner
-Version: 3.2.0
+Version: 3.2.2
 Summary: DataTunner: Scientific Platform for Optimal Artificial Data Proportion in Deep Learning
 Author-email: Leandro Costa Rocha <leandrocrx@gmail.com>
 License-Expression: MIT

{datatunner-3.2.0 → datatunner-3.2.2}/datatunner/domain/experiment.py

@@ -63,6 +63,7 @@ class ExperimentReport:
     quality_results: List[Any] = field(default_factory=list)
     optimization_results: List[ExperimentResult] = field(default_factory=list)
     baseline_results: List[ExperimentResult] = field(default_factory=list)
+    paired_comparisons: List[Dict] = field(default_factory=list)
     creation_timestamp: float = field(default_factory=time.time)
 
     @property
@@ -97,6 +98,7 @@ class ExperimentReport:
             'best_result': self.best_result.to_dict() if self.best_result else None,
             'optimization_results': [r.to_dict() for r in self.optimization_results],
             'baseline_results': [r.to_dict() for r in self.baseline_results],
+            'paired_comparisons': self.paired_comparisons,
             'n_optimization_results': len(self.optimization_results),
             'n_baseline_results': len(self.baseline_results),
             'creation_timestamp': self.creation_timestamp,

{datatunner-3.2.0 → datatunner-3.2.2}/datatunner/orchestrator.py

@@ -168,21 +168,56 @@ class DataTunner:
             seed_manager=self.seed_manager
         )
 
-        # Phase 4: Run baselines
+        # Phase 4: Run baselines with a dedicated seed block (paired design)
         baseline_results = []
+        paired_comparisons = []
         if baselines:
-            print(f"[DataTunner] Running {len(baselines)} baseline comparisons...")
+            print(f"[DataTunner] Running {len(baselines)} baseline comparisons (paired, n={self.config.n_repetitions})...")
+
+            # Best point from the search grid — used for the paired re-run
+            best_alpha = None
+            if optimization_results:
+                target = self.config.target_metric
+                best_alpha = max(
+                    optimization_results,
+                    key=lambda r: r.performance.to_dict().get(target, -float('inf'))
+                ).alpha
+
             for name, alpha, base_gen in baselines:
-                base_result = self._run_baseline(name, alpha, base_gen, real_data, test_data)
-                if base_result:
-                    baseline_results.append(base_result)
+                # ONE seed block per baseline; the same seeds are reused for
+                # alpha* so each pair shares the same random state (thesis
+                # Section 4.7 — "blocagem por semente").
+                seed_block = self.seed_manager.generate_seeds(self.config.n_repetitions)
+
+                base_results = self._run_baseline(name, alpha, base_gen, real_data, test_data,
+                                                  seed_block=seed_block)
+                baseline_results.extend(base_results)
+
+                if best_alpha is not None and base_results:
+                    star_results = self._run_paired_alpha_star(best_alpha, seed_block)
+                    if star_results:
+                        paired_comparisons.append({
+                            'baseline_name': name,
+                            'baseline_alpha': alpha,
+                            'alpha_star': best_alpha,
+                            'target_metric': self.config.target_metric,
+                            'pairs': [
+                                {
+                                    'seed': b.seed,
+                                    'baseline_metric': b.performance.to_dict().get(self.config.target_metric),
+                                    'alpha_star_metric': s.performance.to_dict().get(self.config.target_metric),
+                                }
+                                for b, s in zip(base_results, star_results)
+                            ],
+                        })
 
         # Phase 5: Compile report
         report = ExperimentReport(
             config=self.config,
             quality_results=quality_results,
             optimization_results=optimization_results,
-            baseline_results=baseline_results
+            baseline_results=baseline_results,
+            paired_comparisons=paired_comparisons,
         )
 
         # Save artifacts
@@ -201,6 +236,18 @@ class DataTunner:
 
         return report
 
+    @staticmethod
+    def _count_samples(data: Any) -> int:
+        """Number of samples in a dataset, handling (images, labels) tuples.
+
+        Image pipelines pass real_data as a (X, y) tuple; ``len()`` on the
+        tuple returns the number of elements (2), not the number of samples.
+        Without this, n_synthetic = int(alpha * 2) ≈ 0 for every alpha.
+        """
+        if isinstance(data, tuple) and len(data) == 2:
+            return len(data[0])
+        return len(data)
+
     def _run_single_experiment(self, alpha: float, seed: int) -> ExperimentResult:
         """Execute ONE complete experiment: generate -> mix -> train -> evaluate.
 
@@ -210,7 +257,7 @@ class DataTunner:
 
         with self.environment.isolate(seed, experiment_id):
             # 1. Generate synthetic data
-            n_real = len(self._real_data)
+            n_real = self._count_samples(self._real_data)
             n_synthetic = int(alpha * n_real)
 
             # Bind the generator RNG to this trial's seed so repetitions
@@ -311,7 +358,7 @@ class DataTunner:
     ) -> List[Dict]:
         """Pre-evaluate generator by generating a sample and measuring fidelity."""
         # Generate a fixed-size sample for quality assessment
-        sample_size = min(1000, len(real_data))
+        sample_size = min(1000, self._count_samples(real_data))
         generator.set_seed(generator.spec.random_state)
         synthetic_sample = generator.generate(sample_size)
 
@@ -329,25 +376,70 @@ class DataTunner:
         alpha: float,
         generator: AbstractGenerator,
         real_data: Any,
-        test_data: Any
-    ) -> Optional[ExperimentResult]:
-        """Run a single baseline configuration."""
+        test_data: Any,
+        seed_block: Optional[List[int]] = None
+    ) -> List[ExperimentResult]:
+        """Run baseline trials over a dedicated seed block.
+
+        The block is generated ONCE per baseline (Section 4.7 of the thesis:
+        "blocagem por semente") and the SAME seeds are later reused to
+        re-run alpha* — that pairing is what legitimizes the Wilcoxon
+        signed-rank test of P1/H1. Earlier versions ran a single trial with
+        one fresh seed, which made a paired comparison impossible.
+
+        Args:
+            name: Baseline label.
+            alpha: Proportion to evaluate (usually 0.0).
+            generator: Generator instance for this baseline.
+            real_data: Training data.
+            test_data: Hold-out test data.
+            seed_block: Optional pre-generated block of seeds. If None, a
+                        fresh block of ``config.n_repetitions`` seeds is
+                        drawn from the counter.
+        """
         old_generator = self._generator
+        results: List[ExperimentResult] = []
         try:
-            # generate_seeds advances a counter, so each baseline gets a seed
-            # that is fresh (never reused from the main search).
-            seed = self.seed_manager.generate_seeds(1)[0]
+            if seed_block is None:
+                seed_block = self.seed_manager.generate_seeds(self.config.n_repetitions)
 
             # Temporarily swap generator
             self._generator = generator
             generator.fit(real_data)
 
-            result = self._run_single_experiment(alpha, seed)
-
-            return result
+            for seed in seed_block:
+                try:
+                    result = self._run_single_experiment(alpha, seed)
+                    if result is not None:
+                        results.append(result)
+                except Exception as e:
+                    print(f"[DataTunner] Baseline '{name}' trial seed={seed} failed: {e}")
+            return results
         except Exception as e:
             print(f"[DataTunner] Baseline '{name}' failed: {e}")
-            return None
+            return results
         finally:
             # Always restore the main generator, even on failure.
             self._generator = old_generator
+
+    def _run_paired_alpha_star(
+        self,
+        alpha_star: float,
+        seed_block: List[int]
+    ) -> List[ExperimentResult]:
+        """Re-run alpha* with the SAME seed block used by a baseline.
+
+        This is the second half of the paired design: for each seed in the
+        block, one trial at alpha* is executed with the main generator. The
+        caller zips these results with the baseline results (same seed
+        order) to build the paired comparison for the Wilcoxon test.
+        """
+        results: List[ExperimentResult] = []
+        for seed in seed_block:
+            try:
+                result = self._run_single_experiment(alpha_star, seed)
+                if result is not None:
+                    results.append(result)
+            except Exception as e:
+                print(f"[DataTunner] Paired alpha* trial seed={seed} failed: {e}")
+        return results

{datatunner-3.2.0 → datatunner-3.2.2}/datatunner/reporting/tables.py

@@ -1,7 +1,8 @@
 """Publication-ready table generation for experiment reports."""
 
-from typing import List
+from typing import Dict, List, Tuple
 
+import numpy as np
 import pandas as pd
 
 from datatunner.domain.experiment import ExperimentReport, ExperimentResult
@@ -59,20 +60,29 @@ class TableGenerator:
                 "type": "optimization",
             })
 
+        # Baselines: aggregate the n_r trials of the paired block into one
+        # row per (method, alpha) so the comparison table stays readable.
+        baseline_agg: Dict[Tuple[str, float], List[float]] = {}
         for r in report.baseline_results:
             perf = r.performance.to_dict()
             value = perf.get(target)
+            method = (
+                r.generator_spec.name
+                if hasattr(r.generator_spec, "name")
+                else str(r.generator_spec)
+            )
+            if value is not None:
+                baseline_agg.setdefault((method, r.alpha), []).append(value)
+
+        for (method, alpha), values in baseline_agg.items():
+            mean = float(np.mean(values))
             improvement = None
-            if best_value is not None and value is not None and best_value != 0:
-                improvement = ((value - best_value) / best_value) * 100.0
+            if best_value is not None and best_value != 0:
+                improvement = ((mean - best_value) / best_value) * 100.0
             rows.append({
-                "method": (
-                    r.generator_spec.name
-                    if hasattr(r.generator_spec, "name")
-                    else str(r.generator_spec)
-                ),
-                "alpha": r.alpha,
-                target: f"{value:.4f}" if value is not None else "N/A",
+                "method": method,
+                "alpha": alpha,
+                target: f"{mean:.4f}",
                 "improvement_%": (
                     f"{improvement:+.2f}%" if improvement is not None else "N/A"
                 ),

{datatunner-3.2.0 → datatunner-3.2.2/datatunner.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatunner
-Version: 3.2.0
+Version: 3.2.2
 Summary: DataTunner: Scientific Platform for Optimal Artificial Data Proportion in Deep Learning
 Author-email: Leandro Costa Rocha <leandrocrx@gmail.com>
 License-Expression: MIT

{datatunner-3.2.0 → datatunner-3.2.2}/datatunner.egg-info/SOURCES.txt

@@ -64,6 +64,7 @@ tests/__init__.py
 tests/test_aggregation.py
 tests/test_domain.py
 tests/test_generators.py
+tests/test_image_support.py
 tests/test_infrastructure.py
 tests/test_metrics_formatting.py
 tests/test_mixing.py

{datatunner-3.2.0 → datatunner-3.2.2}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatunner"
-# Bump on release: the current published version on PyPI is 3.1.1.
-version = "3.2.0"
+# Bump on release: the current published version on PyPI is 3.2.1.
+version = "3.2.2"
 description = "DataTunner: Scientific Platform for Optimal Artificial Data Proportion in Deep Learning"
 readme = "README.md"
 license = "MIT"

datatunner-3.2.2/tests/test_image_support.py

@@ -0,0 +1,39 @@
+"""Regression tests for image pipeline support (X, y) tuples.
+
+Bug fix (2026-08): `_run_single_experiment` computed n_synthetic with
+``len(real_data)``. For image pipelines real_data is a (X, y) tuple, so
+len() returned 2 → n_synthetic = int(alpha * 2) ≈ 0 for every alpha:
+the ImageAugmentation pipeline generated (almost) no synthetic data.
+
+Fix: DataTunner._count_samples() counts rows of data[0] for 2-tuples.
+Used in _run_single_experiment and _evaluate_generator_quality.
+
+Run where deps are installed (CI / Colab):  pytest tests/test_image_support.py
+"""
+import pytest
+
+from datatunner.orchestrator import DataTunner
+
+
+class TestCountSamples:
+    @pytest.mark.parametrize("n", [0, 1, 2, 10, 1000])
+    def test_tuple_counts_samples_not_elements(self, n):
+        X = [0] * n
+        y = [0] * n
+        assert DataTunner._count_samples((X, y)) == n
+
+    def test_nested_2d_like_image_array(self):
+        # X shaped (N, H, W): must count N, not dimensions
+        X = [[[0] * 4 for _ in range(4)] for _ in range(250)]
+        y = [0] * 250
+        assert DataTunner._count_samples((X, y)) == 250
+
+    def test_plain_list_counts_len(self):
+        assert DataTunner._count_samples([1, 2, 3, 4]) == 4
+
+    def test_single_element_list(self):
+        assert DataTunner._count_samples(["only"]) == 1
+
+    def test_non_tuple_pair_not_misclassified(self):
+        # A 2-element tuple is ambiguous; contract: (X, y) tuples mean images
+        assert DataTunner._count_samples((1, 2)) == 1  # len(data[0]) = 1

{datatunner-3.2.0 → datatunner-3.2.2}/tests/test_orchestrator_regressions.py

@@ -116,7 +116,7 @@ class TestBaselineIsolation:
 
     def test_baseline_uses_fresh_seed(self):
         spec = GeneratorSpec(name="stub", mechanism="smote", hyperparameters={}, random_state=42)
-        tunner = _make_tunner()
+        tunner = _make_tunner()  # n_repetitions=1
         baseline_gen = _StubGenerator(spec)
 
         report = _simple_run(tunner, _StubGenerator(spec), baselines=[("base", 0.0, baseline_gen)])
@@ -125,3 +125,32 @@ class TestBaselineIsolation:
         baseline_result = report.baseline_results[0]
         # Regression: baselines reused the first grid seed
         assert baseline_result.seed != grid_seed
+
+    def test_baseline_paired_block_shares_seed_with_alpha_star(self):
+        """Paired design (thesis 4.7): baseline and alpha* share a seed block."""
+        config = ExperimentConfig(
+            data_type="tabular",
+            search_strategy=GridSearch(),
+            alpha_bounds=(0.0, 1.0),
+            n_repetitions=3,
+            search_budget=3,
+            target_metric="f1_macro",
+        )
+        spec = GeneratorSpec(name="stub", mechanism="smote", hyperparameters={}, random_state=42)
+        tunner = _make_tunner(config)
+        baseline_gen = _StubGenerator(spec)
+
+        report = _simple_run(tunner, _StubGenerator(spec), baselines=[("base", 0.0, baseline_gen)])
+
+        # Baseline now runs n_repetitions trials (one per seed in the block)
+        assert len(report.baseline_results) == config.n_repetitions
+        # Paired comparison: baseline seeds == alpha* re-run seeds
+        assert report.paired_comparisons, "paired_comparisons should be populated"
+        pc = report.paired_comparisons[0]
+        assert len(pc["pairs"]) == config.n_repetitions
+        seeds = [p["seed"] for p in pc["pairs"]]
+        assert len(set(seeds)) == config.n_repetitions
+        assert pc["alpha_star"] == report.best_result.alpha
+        # Every pair has both metrics
+        for p in pc["pairs"]:
+            assert "baseline_metric" in p and "alpha_star_metric" in p