flipcosmo 1.0.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

scripts/flip_launch_minuit_full_fit.py

@@ -0,0 +1,117 @@
+import os
+
+import numpy as np
+import pandas as pd
+from flip.covariance import covariance, fitter
+from pkg_resources import resource_filename
+
+from flip import data_vector, utils
+
+flip_base = resource_filename("flip", ".")
+data_path = os.path.join(flip_base, "data")
+
+### Load data
+grid = pd.read_parquet(os.path.join(data_path, "data_density.parquet"))
+grid_window = pd.read_parquet(os.path.join(data_path, "data_window_density.parquet"))
+coordinates_density = np.array([grid["ra"], grid["dec"], grid["rcom_zobs"]])
+data_density = {
+    "density": np.array(grid["density"]),
+    "density_error": np.array(grid["density_error"]),
+}
+
+
+sn_data = pd.read_parquet(os.path.join(data_path, "data_velocity.parquet"))
+
+coordinates_velocity = np.array([sn_data["ra"], sn_data["dec"], sn_data["como_dist"]])
+data_velocity = sn_data.to_dict("list")
+for key in data_velocity.keys():
+    data_velocity[key] = np.array(data_velocity[key])
+data_velocity["velocity"] = data_velocity.pop("vpec")
+data_velocity["velocity_error"] = np.zeros_like(data_velocity["velocity"])
+
+data_velocity_object = data_vector.DirectVel(data_velocity)
+data_density_object = data_vector.Dens(data_density)
+
+data_density_velocity_object = data_vector.DensVel(
+    data_density_object, data_velocity_object
+)
+
+sigmau_fiducial = 15.0
+sigmag_fiducial = 3.0
+
+ktt, ptt = np.loadtxt(os.path.join(data_path, "power_spectrum_tt.txt"))
+kmt, pmt = np.loadtxt(os.path.join(data_path, "power_spectrum_mt.txt"))
+kmm, pmm = np.loadtxt(os.path.join(data_path, "power_spectrum_mm.txt"))
+power_spectrum_dict_bias = {
+    "gg": [[kmm, pmm * np.array(grid_window["window_mm"]) ** 2]]
+}
+power_spectrum_dict = {
+    "gg": [
+        [kmm, pmm * np.array(grid_window["window_mm"]) ** 2],
+        [kmt, pmt * np.array(grid_window["window_mt"])],
+        [ktt, ptt],
+    ],
+    "gv": [
+        [
+            kmt,
+            pmt * np.array(grid_window["window_mt"]) * utils.Du(kmt, sigmau_fiducial),
+        ],
+        [ktt, ptt * utils.Du(kmt, sigmau_fiducial)],
+    ],
+    "vv": [[ktt, ptt * utils.Du(ktt, sigmau_fiducial) ** 2]],
+}
+
+### Compute covariance
+size_batch = 500_000
+number_worker = 8
+
+covariance_fit = covariance.CovMatrix.init_from_flip(
+    "adamsblake20",
+    "full",
+    power_spectrum_dict,
+    coordinates_density=coordinates_density,
+    coordinates_velocity=coordinates_velocity,
+    size_batch=size_batch,
+    number_worker=number_worker,
+    additional_parameters_values=(sigmag_fiducial,),
+    variant="nobeta",
+)
+
+### Load fitter
+likelihood_type = "multivariate_gaussian"
+likelihood_properties = {"inversion_method": "cholesky_inverse"}
+
+
+parameter_dict = {
+    "bs8": {
+        "value": 1.0,
+        "limit_low": 0.0,
+        "limit_up": 3.0,
+        "fixed": False,
+    },
+    "fs8": {
+        "value": 0.4,
+        "limit_low": 0.0,
+        "limit_up": 1.0,
+        "fixed": False,
+    },
+    "sigv": {
+        "value": 100,
+        "limit_low": None,
+        "limit_up": None,
+        "fixed": False,
+    },
+}
+
+
+minuit_fitter = fitter.FitMinuit.init_from_covariance(
+    covariance_fit,
+    data_density_velocity_object,
+    parameter_dict,
+    likelihood_type=likelihood_type,
+    likelihood_properties=likelihood_properties,
+)
+
+
+### Fit
+minuit_fitter.run()

scripts/flip_launch_minuit_velocity_fit.py

@@ -0,0 +1,78 @@
+import os
+
+import numpy as np
+import pandas as pd
+from flip.covariance import covariance, fitter
+from pkg_resources import resource_filename
+
+from flip import data_vector, utils
+
+flip_base = resource_filename("flip", ".")
+data_path = os.path.join(flip_base, "data")
+
+### Load data
+sn_data = pd.read_parquet(os.path.join(data_path, "data_velocity.parquet"))
+
+
+coordinates_velocity = np.array([sn_data["ra"], sn_data["dec"], sn_data["rcom_zobs"]])
+
+data_velocity = sn_data.to_dict("list")
+for key in data_velocity.keys():
+    data_velocity[key] = np.array(data_velocity[key])
+data_velocity["velocity"] = data_velocity.pop("vpec")
+data_velocity["velocity_error"] = np.zeros_like(data_velocity["velocity"])
+
+data_velocity_object = data_vector.DirectVel(data_velocity)
+
+ktt, ptt = np.loadtxt(os.path.join(data_path, "power_spectrum_tt.txt"))
+kmt, pmt = np.loadtxt(os.path.join(data_path, "power_spectrum_mt.txt"))
+kmm, pmm = np.loadtxt(os.path.join(data_path, "power_spectrum_mm.txt"))
+
+sigmau_fiducial = 15
+
+power_spectrum_dict = {"vv": [[ktt, ptt * utils.Du(ktt, sigmau_fiducial) ** 2]]}
+
+### Compute covariance
+size_batch = 10_000
+number_worker = 16
+
+covariance_fit = covariance.CovMatrix.init_from_flip(
+    "carreres23",
+    "velocity",
+    power_spectrum_dict,
+    coordinates_velocity=coordinates_velocity,
+    size_batch=size_batch,
+    number_worker=number_worker,
+)
+
+
+### Load fitter
+likelihood_type = "multivariate_gaussian"
+likelihood_properties = {"inversion_method": "cholesky_inverse"}
+
+
+parameter_dict = {
+    "fs8": {
+        "value": 0.4,
+        "limit_low": 0.0,
+        "fixed": False,
+    },
+    "sigv": {
+        "value": 200,
+        "limit_low": 0.0,
+        "fixed": False,
+    },
+}
+
+
+minuit_fitter = fitter.FitMinuit.init_from_covariance(
+    covariance_fit,
+    data_velocity_object,
+    parameter_dict,
+    likelihood_type=likelihood_type,
+    likelihood_properties=likelihood_properties,
+)
+
+
+### Fit
+minuit_fitter.run()

scripts/flip_launch_minuit_velocity_fit_full.py

@@ -0,0 +1,107 @@
+import os
+
+import numpy as np
+import pandas as pd
+from flip.covariance import covariance, fitter
+from pkg_resources import resource_filename
+
+from flip import data_vector, utils
+
+flip_base = resource_filename("flip", ".")
+data_path = os.path.join(flip_base, "data")
+
+### Load data
+sn_data = pd.read_parquet(os.path.join(data_path, "data_velocity.parquet"))
+
+coordinates_velocity = np.array([sn_data["ra"], sn_data["dec"], sn_data["rcom_zobs"]])
+
+data_velocity = sn_data.to_dict("list")
+for key in data_velocity.keys():
+    data_velocity[key] = np.array(data_velocity[key])
+data_velocity["velocity"] = data_velocity.pop("vpec")
+data_velocity["velocity_error"] = np.zeros_like(data_velocity["velocity"])
+
+data_velocity_object = data_vector.snia_vectors.VelFromSALTfit(
+    data_velocity, velocity_estimator="full", h=0.7
+)
+
+
+ktt, ptt = np.loadtxt(os.path.join(data_path, "power_spectrum_tt.txt"))
+kmt, pmt = np.loadtxt(os.path.join(data_path, "power_spectrum_mt.txt"))
+kmm, pmm = np.loadtxt(os.path.join(data_path, "power_spectrum_mm.txt"))
+
+sigmau_fiducial = 15
+
+power_spectrum_dict = {"vv": [[ktt, ptt * utils.Du(ktt, sigmau_fiducial) ** 2]]}
+
+### Compute covariance
+size_batch = 10_000
+number_worker = 16
+
+covariance_fit = covariance.CovMatrix.init_from_flip(
+    "carreres23",
+    "velocity",
+    power_spectrum_dict,
+    coordinates_velocity=coordinates_velocity,
+    size_batch=size_batch,
+    number_worker=number_worker,
+)
+
+
+### Load fitter
+likelihood_type = "multivariate_gaussian"
+likelihood_properties = {
+    "inversion_method": "cholesky_inverse",
+}
+
+
+parameter_dict = {
+    "fs8": {
+        "value": 0.4,
+        "limit_low": 0.0,
+        "limit_up": 1.0,
+        "fixed": False,
+    },
+    "sigv": {
+        "value": 200,
+        "limit_low": 0.0,
+        "limit_up": 300,
+        "fixed": False,
+    },
+    "alpha": {
+        "value": 0.1,
+        "limit_low": 0.05,
+        "limit_up": 0.15,
+        "fixed": False,
+    },
+    "beta": {
+        "value": 3.0,
+        "limit_low": 1.5,
+        "limit_up": 4.5,
+        "fixed": False,
+    },
+    "M_0": {
+        "value": -19,
+        "limit_low": -21,
+        "limit_up": -18,
+        "fixed": False,
+    },
+    "sigma_M": {
+        "value": 0.1,
+        "limit_low": 0.0,
+        "limit_up": 1.0,
+        "fixed": False,
+    },
+}
+
+minuit_fitter = fitter.FitMinuit.init_from_covariance(
+    covariance_fit,
+    data_velocity_object,
+    parameter_dict,
+    likelihood_type=likelihood_type,
+    likelihood_properties=likelihood_properties,
+)
+
+
+### Fit
+minuit_fitter.run()

scripts/flip_launch_minuit_velocity_fit_interpolation.py

@@ -0,0 +1,93 @@
+import os
+
+import numpy as np
+import pandas as pd
+from flip.covariance import covariance, fitter
+from pkg_resources import resource_filename
+
+from flip import data_vector, utils
+
+flip_base = resource_filename("flip", ".")
+data_path = os.path.join(flip_base, "data")
+
+### Load data
+sn_data = pd.read_parquet(os.path.join(data_path, "data_velocity.parquet"))
+
+coordinates_velocity = np.array([sn_data["ra"], sn_data["dec"], sn_data["rcom_zobs"]])
+
+data_velocity = sn_data.to_dict("list")
+for key in data_velocity.keys():
+    data_velocity[key] = np.array(data_velocity[key])
+data_velocity["velocity"] = data_velocity.pop("vpec")
+data_velocity["velocity_error"] = np.zeros_like(data_velocity["velocity"])
+
+data_velocity_object = data_vector.DirectVel(data_velocity)
+
+
+ktt, ptt = np.loadtxt(os.path.join(data_path, "power_spectrum_tt.txt"))
+kmt, pmt = np.loadtxt(os.path.join(data_path, "power_spectrum_mt.txt"))
+kmm, pmm = np.loadtxt(os.path.join(data_path, "power_spectrum_mm.txt"))
+
+sigmau_fiducial = 15
+
+power_spectrum_dict = {"vv": [[ktt, ptt * utils.Du(ktt, sigmau_fiducial) ** 2]]}
+
+### Compute covariance
+sigmau_list = np.linspace(10.0, 20.0, 10)
+covariance_list = []
+size_batch = 10_000
+number_worker = 16
+
+
+for sigu in sigmau_list:
+    power_spectrum_dict = {"vv": [[ktt, ptt * utils.Du(ktt, sigu) ** 2]]}
+
+    covariance_list.append(
+        covariance.CovMatrix.init_from_flip(
+            "carreres23",
+            "velocity",
+            power_spectrum_dict,
+            coordinates_velocity=coordinates_velocity,
+            size_batch=size_batch,
+            number_worker=number_worker,
+        )
+    )
+
+
+### Load fitter
+likelihood_type = "multivariate_gaussian_interp1d"
+likelihood_properties = {"inversion_method": "cholesky_inverse"}
+
+parameter_dict = {
+    "fs8": {
+        "value": 0.4,
+        "limit_low": 0.0,
+        "fixed": False,
+    },
+    "sigv": {
+        "value": 200,
+        "limit_low": 0.0,
+        "fixed": False,
+    },
+    "sigu": {
+        "value": 15.0,
+        "limit_low": 13.0,
+        "limit_up": 17.0,
+        "fixed": False,
+    },
+}
+
+
+minuit_fitter = fitter.FitMinuit.init_from_covariance(
+    covariance_list,
+    data_velocity_object,
+    parameter_dict,
+    likelihood_type=likelihood_type,
+    likelihood_properties=likelihood_properties,
+    interpolation_value_name="sigu",
+    interpolation_value_range=sigmau_list,
+)
+
+
+### Fit
+minuit_fitter.run()

test/refresh_reference_values.py

@@ -0,0 +1,43 @@
+import json
+
+import test_covariance_reference_values
+import test_e2e_density
+import test_e2e_joint
+import test_e2e_velocity
+
+
+def refresh_covariance_reference_values():
+    ref: dict = {}
+    for m in test_covariance_reference_values.model_to_test:
+        model, kind = m[0], m[1]
+        key = f"{model}:{kind}"
+        ref[key] = test_covariance_reference_values.test_covariance_reference_metrics(
+            model,
+            kind,
+            debug_return=True,
+        )
+
+    out_main = "test_covariance_reference_values.json"
+    with open(out_main, "w") as f:
+        json.dump(ref, f, indent=2)
+
+    return ref
+
+
+def refresh_e2e_reference_values():
+    ref = {
+        "e2e_density": test_e2e_density.test_e2e_density(debug_return=True),
+        "e2e_velocity": test_e2e_velocity.test_e2e_velocity(debug_return=True),
+        "e2e_joint": test_e2e_joint.test_e2e_joint(debug_return=True),
+    }
+
+    out_main = "test_e2e_reference_values.json"
+    with open(out_main, "w") as f:
+        json.dump(ref, f, indent=2)
+
+    return ref
+
+
+if __name__ == "__main__":
+    refresh_covariance_reference_values()
+    refresh_e2e_reference_values()

test/test_covariance_assembly.py

@@ -0,0 +1,102 @@
+import numpy as np
+from flip.data import load_data_test
+
+from flip import covariance, data_vector
+
+
+def test_covariance_assembly_density_velocity():
+
+    coordinates_density, density_data = load_data_test.load_density_data(subsample=50)
+    density_data_vector = data_vector.Dens(density_data)
+
+    coordinates_velocity, velocity_data = load_data_test.load_velocity_data(
+        subsample=50
+    )
+    velocity_data_vector = data_vector.DirectVel(velocity_data)
+
+    density_velocity_data_vector = data_vector.DensVel(
+        density_data_vector, velocity_data_vector
+    )
+
+    power_spectrum_dict = load_data_test.load_power_spectrum_dict()
+
+    model_name = "ravouxcarreres"
+    model_type = "density_velocity"
+    sigmag_fiducial = 5.0
+
+    covariance_object = covariance.CovMatrix.init_from_flip(
+        model_name,
+        model_type,
+        power_spectrum_dict,
+        coordinates_density=coordinates_density,
+        coordinates_velocity=coordinates_velocity,
+        size_batch=50_000,
+        number_worker=1,
+        additional_parameters_values=(sigmag_fiducial,),
+        variant="nobeta",
+    )
+
+    # Assemble total covariance with simple coefficients
+    coefficients = {"bs8": 1.0, "fs8": 1.0, "sigv": 0.0}
+    _, data_variance = density_velocity_data_vector.give_data_and_variance()
+    covariance_matrix = covariance_object.compute_covariance_sum(
+        coefficients, data_variance
+    )
+
+    # Check shapes and that cross-block is exactly zero (no gv provided)
+    n_density = len(density_data_vector.data["density"])
+    n_velocity = len(velocity_data_vector.data["velocity"])
+    covariance_density_density = covariance_matrix[:n_density, :n_density]
+    covariance_velocity_velocity = covariance_matrix[n_density:, n_density:]
+    covariance_density_velocity = covariance_matrix[:n_density, n_density:]
+
+    np.testing.assert_allclose(covariance_density_velocity, 0.0, atol=0.0)
+    assert covariance_matrix.shape == (n_density + n_velocity, n_density + n_velocity)
+    assert np.all(np.diag(covariance_density_density) > 0)
+    assert np.all(np.diag(covariance_velocity_velocity) > 0)
+
+
+def test_covariance_assembly_full():
+
+    coordinates_density, density_data = load_data_test.load_density_data(subsample=50)
+    density_data_vector = data_vector.Dens(density_data)
+
+    coordinates_velocity, velocity_data = load_data_test.load_velocity_data(
+        subsample=50
+    )
+    velocity_data_vector = data_vector.DirectVel(velocity_data)
+
+    density_velocity_data_vector = data_vector.DensVel(
+        density_data_vector, velocity_data_vector
+    )
+
+    power_spectrum_dict = load_data_test.load_power_spectrum_dict()
+
+    model_name = "ravouxcarreres"
+    model_type = "full"
+    sigmag_fiducial = 5.0
+
+    covariance_object = covariance.CovMatrix.init_from_flip(
+        model_name,
+        model_type,
+        power_spectrum_dict,
+        coordinates_density=coordinates_density,
+        coordinates_velocity=coordinates_velocity,
+        size_batch=50_000,
+        number_worker=1,
+        additional_parameters_values=(sigmag_fiducial,),
+        variant="nobeta",
+    )
+
+    coefficients = {"bs8": 1.0, "fs8": 1.0, "sigv": 0.0}
+    _, data_variance = density_velocity_data_vector.give_data_and_variance()
+    covariance_matrix = covariance_object.compute_covariance_sum(
+        coefficients, data_variance
+    )
+
+    n_density = len(density_data_vector.data["density"])
+    covariance_density_velocity = covariance_matrix[:n_density, n_density:]
+
+    print(np.min(np.abs(covariance_density_velocity)))
+
+    assert np.any(np.abs(covariance_density_velocity) > 0)

test/test_covariance_reference_values.py

@@ -0,0 +1,125 @@
+import numpy as np
+import pytest
+from flip.data import load_data_test
+
+from flip import covariance, data_vector
+
+model_to_test = [
+    ("carreres23", "velocity"),
+    ("adamsblake17plane", "density"),
+    ("adamsblake17plane", "velocity"),
+    ("adamsblake17plane", "full"),
+    ("adamsblake20", "density"),
+    ("adamsblake20", "velocity"),
+    ("adamsblake20", "full"),
+    ("ravouxcarreres", "velocity"),
+    ("ravouxcarreres", "density"),
+    ("ravouxcarreres", "full"),
+    ("lai22", "density"),
+    ("lai22", "velocity"),
+    ("lai22", "full"),
+]
+
+
+def compute_covariance_metrics(
+    model,
+    model_type,
+):
+
+    if model_type == "density":
+        coordinates_density, density_data = load_data_test.load_density_data(
+            subsample=50
+        )
+        data_vector_obj = data_vector.Dens(density_data)
+        coordinates_velocity = None
+
+    elif model_type == "velocity":
+        coordinates_velocity, velocity_data = load_data_test.load_velocity_data(
+            subsample=50
+        )
+        data_vector_obj = data_vector.DirectVel(velocity_data)
+        coordinates_density = None
+    else:
+        coordinates_velocity, velocity_data = load_data_test.load_velocity_data(
+            subsample=50
+        )
+        coordinates_density, density_data = load_data_test.load_density_data(
+            subsample=50
+        )
+        data_vector_obj = data_vector.DensVel(
+            data_vector.Dens(density_data),
+            data_vector.DirectVel(velocity_data),
+        )
+
+    power_spectrum_dict = load_data_test.load_power_spectrum_dict()
+
+    additional_parameters = ()
+    variant = None
+    parameters = {"bs8": 1.0, "fs8": 0.5, "sigv": 0.0}
+    if model in {"adamsblake20", "ravouxcarreres"}:
+        parameters["beta_f"] = parameters["fs8"] / max(parameters["bs8"], 1e-6)
+        additional_parameters = (15.0,)
+        variant = "nobeta"
+    if model == "lai22":
+        parameters["sigg"] = 1.0
+        parameters["beta_f"] = parameters["fs8"] / max(parameters["bs8"], 1e-6)
+        variant = "nobeta"
+
+    covariance_object = covariance.CovMatrix.init_from_flip(
+        model,
+        model_type,
+        power_spectrum_dict,
+        coordinates_density=coordinates_density,
+        coordinates_velocity=coordinates_velocity,
+        size_batch=50_000,
+        number_worker=1,
+        variant=variant,
+        additional_parameters_values=additional_parameters,
+    )
+
+    _, data_variance = data_vector_obj.give_data_and_variance(
+        parameters if "M_0" in data_vector_obj.free_par else {}
+    )
+    covariance_matrix = covariance_object.compute_covariance_sum(
+        parameters, data_variance
+    )
+    return {
+        "shape": [int(covariance_matrix.shape[0]), int(covariance_matrix.shape[1])],
+        "trace": float(np.trace(covariance_matrix)),
+        "diag_mean": float(np.mean(np.diag(covariance_matrix))),
+        "entry_0_0": float(covariance_matrix[0, 0]),
+        "entry_0_-1": float(covariance_matrix[0, -1]),
+        "entry_mid_mid": float(
+            covariance_matrix[
+                covariance_matrix.shape[0] // 2, covariance_matrix.shape[1] // 2
+            ]
+        ),
+    }
+
+
+@pytest.mark.parametrize(
+    "model,kind",
+    model_to_test,
+)
+def test_covariance_reference_metrics(model, kind, debug_return=False):
+
+    got = compute_covariance_metrics(model, kind)
+
+    if debug_return:
+        return got
+
+    reference_values = load_data_test.load_covariance_test_reference_values()
+
+    key = f"{model}:{kind}"
+    assert key in reference_values, f"Missing reference metrics for {key}"
+    ref = reference_values[key]
+    assert got["shape"] == ref["shape"]
+    np.testing.assert_allclose(got["trace"], ref["trace"], rtol=1e-6, atol=1e-8)
+    np.testing.assert_allclose(got["diag_mean"], ref["diag_mean"], rtol=1e-6, atol=1e-8)
+    np.testing.assert_allclose(got["entry_0_0"], ref["entry_0_0"], rtol=1e-6, atol=1e-8)
+    np.testing.assert_allclose(
+        got["entry_0_-1"], ref["entry_0_-1"], rtol=1e-6, atol=1e-8
+    )
+    np.testing.assert_allclose(
+        got["entry_mid_mid"], ref["entry_mid_mid"], rtol=1e-6, atol=1e-8
+    )

test/test_covariance_utils.py

@@ -0,0 +1,34 @@
+import numpy as np
+from flip.covariance import cov_utils
+
+
+def test_flat_matrix_roundtrip_small():
+    # Covariance flattening stores a single variance value (shared diagonal)
+    # followed by upper-triangular off-diagonals. Build a matrix consistent with this contract.
+    rng = np.random.default_rng(0)
+    n = 5
+    off = rng.standard_normal((n, n))
+    off = np.triu(off, k=1)
+    off = off + off.T
+    var = 2.3
+    cov = off.copy()
+    np.fill_diagonal(cov, var)
+
+    flat = cov_utils.return_flat_covariance(cov)
+    cov_back = cov_utils.return_matrix_covariance(flat)
+
+    assert cov_back.shape == cov.shape
+    np.testing.assert_allclose(cov_back, cov, rtol=0, atol=1e-12)
+
+
+def test_flat_cross_roundtrip_shape():
+    # Cross-covariance flattening/reshaping
+    g, v = 3, 4
+    rng = np.random.default_rng(1)
+    cross = rng.standard_normal((g, v))
+
+    flat = cov_utils.return_flat_cross_cov(cross)
+    back = cov_utils.return_matrix_covariance_cross(flat, g, v)
+
+    assert back.shape == (g, v)
+    np.testing.assert_allclose(back, cross)