diffsky 0.2.0__tar.gz → 0.2.1__tar.gz

{diffsky-0.2.0 → diffsky-0.2.1}/.github/workflows/monthly-warning-test.yaml

@@ -54,7 +54,7 @@ jobs:
           pip install --no-deps git+https://github.com/ArgonneCPAC/diffmah.git
           pip install --no-deps git+https://github.com/ArgonneCPAC/diffstar.git
           pip install --no-deps git+https://github.com/ArgonneCPAC/dsps.git
-          pip install --no-deps git+https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/ArgonneCPAC/diffstarpop.git
+          pip install --no-deps git+https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/ArgonneCPAC/diffstarpop.git@main
           python -m pip install --no-build-isolation --no-deps -e .
 
       - name: test that no warnings are raised

{diffsky-0.2.0 → diffsky-0.2.1}/.github/workflows/test_latest_releases.yaml

@@ -47,9 +47,6 @@ jobs:
             matplotlib \
             astropy \
             python-build
-          pip uninstall dsps --yes
-          pip install --no-deps git+https://github.com/ArgonneCPAC/dsps.git
-          pip install --no-deps git+https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/ArgonneCPAC/diffstarpop.git
           python -m pip install --no-build-isolation --no-deps -e .
 
       - name: test

{diffsky-0.2.0 → diffsky-0.2.1}/.github/workflows/test_main_branch.yaml

@@ -53,7 +53,7 @@ jobs:
           pip install --no-deps git+https://github.com/ArgonneCPAC/diffmah.git
           pip install --no-deps git+https://github.com/ArgonneCPAC/diffstar.git
           pip install --no-deps git+https://github.com/ArgonneCPAC/dsps.git
-          pip install --no-deps git+https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/ArgonneCPAC/diffstarpop.git
+          pip install --no-deps git+https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/ArgonneCPAC/diffstarpop.git@main
           python -m pip install --no-build-isolation --no-deps -e .
 
       - name: test

{diffsky-0.2.0 → diffsky-0.2.1}/.github/workflows/tests_cron.yaml

@@ -57,7 +57,7 @@ jobs:
           pip install --no-deps git+https://github.com/ArgonneCPAC/diffmah.git
           pip install --no-deps git+https://github.com/ArgonneCPAC/diffstar.git
           pip install --no-deps git+https://github.com/ArgonneCPAC/dsps.git
-          pip install --no-deps git+https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/ArgonneCPAC/diffstarpop.git
+          pip install --no-deps git+https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/ArgonneCPAC/diffstarpop.git@main
           python -m pip install --no-build-isolation --no-deps -e .
 
       - name: test

{diffsky-0.2.0 → diffsky-0.2.1}/.gitignore

@@ -6,6 +6,9 @@ Untitled*.ipynb
 *.npy
 *.png
 
+scripts/*.cfg
+scripts/*.cfg~
+
 .DS_Store
 
 # Byte-compiled / optimized / DLL files

diffsky-0.2.1/CHANGES.rst

@@ -0,0 +1,26 @@
+0.2.1 (2025-06-30)
+-------------------
+- Incorporate synthetic lightcone into mock production pipeline (https://github.com/ArgonneCPAC/diffsky/pull/165)
+- Add pipeline to generate lightcone mocks (https://github.com/ArgonneCPAC/diffsky/pull/154)
+- Improve accuracy of young-star contribution stellar age PDF (https://github.com/ArgonneCPAC/diffsky/pull/151)
+- Implement synthetic lightcones to extend resolution limits (https://github.com/ArgonneCPAC/diffsky/pull/143)
+
+
+0.2.0 (2025-03-24)
+-------------------
+- Update population-level photometry models of dust and burstiness
+
+
+0.1.2 (2024-10-25)
+-------------------
+- Update calls to diffmah v0.6.1 and diffstar v0.3.2
+
+
+0.1.1 (2023-10-04)
+-------------------
+- Update calls to dsps v0.3.4
+
+
+0.1.0 (2023-10-03)
+-------------------
+- First release. Compatible with diffstar v0.2.1 and dsps v0.3.3.

{diffsky-0.2.0 → diffsky-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: diffsky
-Version: 0.2.0
+Version: 0.2.1
 Summary: Library for differentiable generation of synthetic skies
 Author-email: Andrew Hearin <ahearin@anl.gov>
 License: BSD 3-Clause License
@@ -40,9 +40,10 @@ Description-Content-Type: text/x-rst
 License-File: LICENSE.rst
 Requires-Dist: numpy
 Requires-Dist: jax
-Requires-Dist: diffmah>=0.7.0
-Requires-Dist: diffstar>=0.3.3
-Requires-Dist: dsps>=0.4.0
+Requires-Dist: diffmah>=0.7.2
+Requires-Dist: diffstar>=0.3.4
+Requires-Dist: diffstarpop>=0.0.2
+Requires-Dist: dsps>=0.4.5
 Requires-Dist: h5py
 Dynamic: license-file
 

{diffsky-0.2.0 → diffsky-0.2.1}/diffsky/__init__.py

@@ -1,7 +1,7 @@
-"""
-"""
+""" """
 
 # flake8: noqa
 
 from ._version import __version__
 from .mass_functions.mc_diffmah_tpeak import mc_subhalos
+from .utils import tw_utils

diffsky-0.2.1/diffsky/_version.py

@@ -0,0 +1 @@
+__version__ = '0.2.1'

diffsky-0.2.1/diffsky/burstpop/diffqburstpop_mono_noise.py

@@ -0,0 +1,136 @@
+""" """
+
+from collections import namedtuple
+
+from dsps.sfh.diffburst import BurstParams, calc_bursty_age_weights
+from jax import jit as jjit
+from jax import numpy as jnp
+
+from .fburstpop_mono import (
+    DEFAULT_FBURSTPOP_PARAMS,
+    ZEROBURST_FBURSTPOP_PARAMS,
+    get_bounded_fburstpop_params,
+    get_fburst_from_fburstpop_params,
+    get_unbounded_fburstpop_params,
+    _get_bounded_fburstpop_param,
+    _get_unbounded_fburstpop_param
+)
+from .freqburst_mono import (
+    DEFAULT_FREQBURST_PARAMS,
+    ZEROBURST_FREQBURST_PARAMS,
+    get_bounded_freqburst_params,
+    get_unbounded_freqburst_params,
+)
+from .tburstpop import (
+    DEFAULT_TBURSTPOP_PARAMS,
+    get_bounded_tburstpop_params,
+    get_tburst_params_from_tburstpop_params,
+    get_unbounded_tburstpop_params,
+)
+
+from ..utils.utility_funcs import _inverse_sigmoid
+
+DiffburstPopParams = namedtuple(
+    "DiffburstPopParams", ["freqburst_params", "fburstpop_params", "tburstpop_params"]
+)
+DEFAULT_DIFFBURSTPOP_PARAMS = DiffburstPopParams(
+    DEFAULT_FREQBURST_PARAMS, DEFAULT_FBURSTPOP_PARAMS, DEFAULT_TBURSTPOP_PARAMS
+)
+_BURSTPOP_UPNAMES = [
+    key.replace("params", "u_params") for key in DEFAULT_DIFFBURSTPOP_PARAMS._fields
+]
+DiffburstPopUParams = namedtuple("DiffburstPopUParams", _BURSTPOP_UPNAMES)
+
+ZERO_DIFFBURSTPOP_PARAMS = DiffburstPopParams(
+    ZEROBURST_FREQBURST_PARAMS, ZEROBURST_FBURSTPOP_PARAMS, DEFAULT_TBURSTPOP_PARAMS
+)
+
+FBURST_BOUNDS = (0.0, 1.0)
+
+
+@jjit
+def get_bounded_diffburstpop_params(u_params):
+    u_freqburst_params, u_fburstpop_params, u_tburstpop_params = u_params
+    bounded_freqburst_params = get_bounded_freqburst_params(u_freqburst_params)
+    bounded_tburstpop_params = get_bounded_tburstpop_params(u_tburstpop_params)
+    bounded_fburstpop_params = get_bounded_fburstpop_params(u_fburstpop_params)
+    diffburstpop_params = DiffburstPopParams(
+        bounded_freqburst_params, bounded_fburstpop_params, bounded_tburstpop_params
+    )
+    return diffburstpop_params
+
+
+@jjit
+def get_unbounded_diffburstpop_params(params):
+    freqburst_params, fburstpop_params, tburstpop_params = params
+    unbounded_freqburst_params = get_unbounded_freqburst_params(freqburst_params)
+    unbounded_fburstpop_params = get_unbounded_fburstpop_params(fburstpop_params)
+    unbounded_tburstpop_params = get_unbounded_tburstpop_params(tburstpop_params)
+    diffburstpop_u_params = DiffburstPopUParams(
+        unbounded_freqburst_params,
+        unbounded_fburstpop_params,
+        unbounded_tburstpop_params,
+    )
+    return diffburstpop_u_params
+
+
+@jjit
+def calc_bursty_age_weights_from_diffburstpop_params(
+    diffburstpop_params,
+    logsm,
+    logssfr,
+    ssp_lg_age_gyr,
+    smooth_age_weights,
+    random_draw_burst,
+    scatter_params,
+):
+    f_burst = get_fburst_from_fburstpop_params(
+        diffburstpop_params.fburstpop_params, logsm, logssfr
+    )
+    ufburst = _get_unbounded_fburstpop_param(f_burst, FBURST_BOUNDS)
+    noisy_ufburst = _inverse_sigmoid(random_draw_burst, ufburst, scatter_params.fburst_scatter, 0.0, 1.0)
+    noisy_fburst = _get_bounded_fburstpop_param(noisy_ufburst, FBURST_BOUNDS)
+    lgfburst = jnp.log10(noisy_fburst)
+
+    tburst_params = get_tburst_params_from_tburstpop_params(
+        diffburstpop_params.tburstpop_params, logsm, logssfr
+    )
+    lgyr_peak, lgyr_max = tburst_params
+    burst_params = BurstParams(lgfburst, lgyr_peak, lgyr_max)
+
+    age_weights = calc_bursty_age_weights(
+        burst_params, smooth_age_weights, ssp_lg_age_gyr
+    )
+
+    return age_weights, burst_params
+
+
+def calc_bursty_age_weights_from_diffburstpop_u_params(
+    diffburstpop_u_params,
+    logsm,
+    logssfr,
+    ssp_lg_age_gyr,
+    smooth_age_weights,
+    random_draw_burst,
+    scatter_params,
+
+):
+    diffburstpop_params = get_bounded_diffburstpop_params(diffburstpop_u_params)
+    args = (
+        diffburstpop_params,
+        logsm,
+        logssfr,
+        ssp_lg_age_gyr,
+        smooth_age_weights,
+        random_draw_burst,
+        scatter_params)
+    age_weights, burst_params = calc_bursty_age_weights_from_diffburstpop_params(*args)
+    return age_weights, burst_params
+
+
+DEFAULT_DIFFBURSTPOP_U_PARAMS = DiffburstPopUParams(
+    *get_unbounded_diffburstpop_params(DEFAULT_DIFFBURSTPOP_PARAMS)
+)
+ZERO_DIFFBURSTPOP_U_PARAMS = get_unbounded_diffburstpop_params(
+    DEFAULT_DIFFBURSTPOP_PARAMS
+)

{diffsky-0.2.0 → diffsky-0.2.1}/diffsky/burstpop/fburstpop_mono.py

@@ -1,5 +1,4 @@
-"""
-"""
+""" """
 
 from collections import OrderedDict, namedtuple
 from copy import deepcopy
@@ -23,9 +22,9 @@ DEFAULT_FBURSTPOP_PDICT = OrderedDict(
     sufb_logsm_yhi_ms=-10.0,
 )
 
-LGSM_X0_BOUNDS = (8.0, 11.0)
+LGSM_X0_BOUNDS = (8.0, 12.0)
 LGSSFR_X0_BOUNDS = (-12.0, -8.0)
-SUFB_BOUNDS = (-15.0, -5.0)
+SUFB_BOUNDS = (-15.0, -2.0)
 
 FBURSTPOP_PBOUNDS_PDICT = OrderedDict(
     sufb_logsm_x0=LGSM_X0_BOUNDS,

{diffsky-0.2.0 → diffsky-0.2.1}/diffsky/burstpop/freqburst_mono.py

@@ -1,5 +1,4 @@
-"""
-"""
+""" """
 
 from collections import OrderedDict, namedtuple
 from copy import deepcopy
@@ -17,15 +16,15 @@ LGSSFR_K = 5.0
 DEFAULT_FREQBURST_PDICT = OrderedDict(
     sufqb_logsm_x0=10.0,
     sufqb_logssfr_x0=-10.25,
-    sufqb_logsm_ylo_q=-10.0,
-    sufqb_logsm_ylo_ms=-10.0,
-    sufqb_logsm_yhi_q=-10.0,
-    sufqb_logsm_yhi_ms=-10.0,
+    sufqb_logsm_ylo_q=-5.0,
+    sufqb_logsm_ylo_ms=-0.2,
+    sufqb_logsm_yhi_q=-5.0,
+    sufqb_logsm_yhi_ms=-0.5,
 )
 
 LGSM_X0_BOUNDS = (9.0, 11.0)
 LGSSFR_X0_BOUNDS = (-12.0, -8.0)
-SUFQB_BOUNDS = (-15.0, -5.0)
+SUFQB_BOUNDS = (-10.0, 0.5)
 
 FREQBURST_PBOUNDS_PDICT = OrderedDict(
     sufqb_logsm_x0=LGSM_X0_BOUNDS,

diffsky-0.2.1/diffsky/burstpop/tests/test_diffqburstpop_mono_noise.py

@@ -0,0 +1,137 @@
+""" """
+
+import numpy as np
+from dsps.sfh import diffburst
+from jax import random as jran
+
+from .. import diffqburstpop_mono_noise as dbp
+from ..fburstpop_mono import FburstPopUParams
+from ..freqburst_mono import FreqburstUParams
+from ..tburstpop import TburstPopUParams
+from ...experimental.scatter import DEFAULT_SCATTER_PARAMS
+
+TOL = 1e-2
+
+
+def test_calc_bursty_age_weights_from_diffburstpop_params_evaluates_on_defaults():
+    n_age = 107
+    ssp_lg_age_gyr = np.linspace(5.5, 10.5, n_age)
+    ran_key = jran.PRNGKey(0)
+
+    random_draw_burst = 0.5
+
+    n_tests = 1_000
+    for __ in range(n_tests):
+        ran_key, logsm_key, logssfr_key, smooth_key = jran.split(ran_key, 4)
+        logsm = jran.uniform(logsm_key, minval=0, maxval=10, shape=())
+        logssfr = jran.uniform(logssfr_key, minval=-12, maxval=-8, shape=())
+        smooth_age_weights = jran.uniform(
+            smooth_key, minval=0, maxval=1, shape=(n_age,)
+        )
+
+        smooth_age_weights = smooth_age_weights / smooth_age_weights.sum()
+        args = (
+            dbp.DEFAULT_DIFFBURSTPOP_PARAMS,
+            logsm,
+            logssfr,
+            ssp_lg_age_gyr,
+            smooth_age_weights,
+            random_draw_burst,
+            DEFAULT_SCATTER_PARAMS,
+        )
+        (
+            age_weights,
+            burst_params,
+        ) = dbp.calc_bursty_age_weights_from_diffburstpop_params(*args)
+        assert age_weights.shape == (n_age,)
+        assert np.all(np.isfinite(age_weights))
+
+        unity = np.sum(age_weights)
+        assert np.allclose(unity, 1.0, atol=1e-2)
+
+        assert diffburst.LGFBURST_MIN < burst_params.lgfburst < diffburst.LGFBURST_MAX
+        assert (
+            diffburst.LGYR_PEAK_MIN < burst_params.lgyr_peak < diffburst.LGYR_PEAK_MAX
+        )
+
+        assert burst_params.lgyr_peak < burst_params.lgyr_max < diffburst.LGAGE_MAX
+
+
+def test_calc_bursty_age_weights_from_diffburstpop_u_params_evaluates_on_u_randoms():
+    n_age = 107
+    ssp_lg_age_gyr = np.linspace(5.5, 10.5, n_age) - 9.0
+
+    n_freqburst_params = len(dbp.DEFAULT_DIFFBURSTPOP_PARAMS.freqburst_params)
+    n_fburstpop_params = len(dbp.DEFAULT_DIFFBURSTPOP_PARAMS.fburstpop_params)
+    n_tburstpop_params = len(dbp.DEFAULT_DIFFBURSTPOP_PARAMS.tburstpop_params)
+    ran_key = jran.PRNGKey(0)
+
+    random_draw_burst = 0.5
+
+    n_tests = 1_000
+    for __ in range(n_tests):
+        ran_key, logsm_key, logssfr_key, smooth_key = jran.split(ran_key, 4)
+        logsm = jran.uniform(logsm_key, minval=0, maxval=10, shape=())
+        logssfr = jran.uniform(logssfr_key, minval=-12, maxval=-8, shape=())
+        smooth_age_weights = jran.uniform(
+            smooth_key, minval=0, maxval=1, shape=(n_age,)
+        )
+        ran_key, fqb_key, fb_key, tb_key = jran.split(ran_key, 4)
+        u_freqb = jran.uniform(
+            fqb_key, minval=-10, maxval=10, shape=(n_freqburst_params,)
+        )
+        u_fb = jran.uniform(fb_key, minval=-10, maxval=10, shape=(n_fburstpop_params,))
+        u_tb = jran.uniform(tb_key, minval=-10, maxval=10, shape=(n_tburstpop_params,))
+        freqburst_u_params = FreqburstUParams(*u_freqb)
+        fburstpop_u_params = FburstPopUParams(*u_fb)
+        tburstpop_u_params = TburstPopUParams(*u_tb)
+
+        diffburstpop_u_params = dbp.DiffburstPopUParams(
+            freqburst_u_params, fburstpop_u_params, tburstpop_u_params
+        )
+
+        smooth_age_weights = smooth_age_weights / smooth_age_weights.sum()
+        args = (
+            diffburstpop_u_params,
+            logsm,
+            logssfr,
+            ssp_lg_age_gyr,
+            smooth_age_weights,
+            random_draw_burst,
+            DEFAULT_SCATTER_PARAMS,
+        )
+        (
+            age_weights,
+            burst_params,
+        ) = dbp.calc_bursty_age_weights_from_diffburstpop_u_params(*args)
+        assert age_weights.shape == (n_age,)
+        assert np.all(np.isfinite(age_weights))
+
+        unity = np.sum(age_weights)
+        assert np.allclose(unity, 1.0, atol=1e-2)
+
+        assert diffburst.LGFBURST_MIN <= burst_params.lgfburst <= diffburst.LGFBURST_MAX
+        assert (
+            diffburst.LGYR_PEAK_MIN <= burst_params.lgyr_peak <= diffburst.LGYR_PEAK_MAX
+        )
+
+        assert burst_params.lgyr_peak <= burst_params.lgyr_max <= diffburst.LGAGE_MAX
+
+
+def test_diffburstpop_u_param_inversion():
+    u_params = dbp.get_unbounded_diffburstpop_params(dbp.DEFAULT_DIFFBURSTPOP_PARAMS)
+    u_params_freqburst, u_params_fburst, u_params_tburst = u_params
+
+    assert np.allclose(
+        dbp.DEFAULT_DIFFBURSTPOP_U_PARAMS.freqburst_u_params,
+        u_params_freqburst,
+        rtol=TOL,
+    )
+
+    assert np.allclose(
+        dbp.DEFAULT_DIFFBURSTPOP_U_PARAMS.fburstpop_u_params, u_params_fburst, rtol=TOL
+    )
+
+    assert np.allclose(
+        dbp.DEFAULT_DIFFBURSTPOP_U_PARAMS.tburstpop_u_params, u_params_tburst, rtol=TOL
+    )

{diffsky-0.2.0 → diffsky-0.2.1}/diffsky/burstpop/tests/test_freqburst_mono.py

@@ -1,5 +1,4 @@
-"""
-"""
+""" """
 
 import numpy as np
 from jax import nn
@@ -201,7 +200,7 @@ def test_zeroburst_params_produce_zero_burstiness():
     assert fqb.shape == (n_gals,)
     assert np.all(np.isfinite(fqb))
     assert np.all(fqb >= 0.0)
-    assert np.all(fqb < 0.01)
+    assert np.all(fqb < 0.1), fqb.max()
 
     sufq_max = SUFQB_BOUNDS[1]
     fqb_max = nn.softplus(sufq_max)

diffsky-0.2.1/diffsky/data_loaders/__init__.py

@@ -0,0 +1,5 @@
+""""""
+
+# flake8: noqa
+
+from .io_utils import load_flat_hdf5

diffsky-0.2.1/diffsky/data_loaders/hacc_utils/__init__.py

@@ -0,0 +1,7 @@
+""""""
+
+# flake8: noqa
+
+from .. import load_flat_hdf5
+from .defaults import *
+from .load_lc_cf import get_diffsky_info_from_hacc_sim

diffsky-0.2.1/diffsky/data_loaders/hacc_utils/data_validation/tests/test_validate_lc_cores.py

@@ -0,0 +1,89 @@
+""""""
+
+import os
+from glob import glob
+
+import pytest
+
+from .. import validate_lc_cores as vlcc
+
+DRN_LC_CORES_POBOY = "/Users/aphearin/work/DATA/LastJourney/lc_cores"
+
+try:
+    from haccytrees import Simulation as HACCSim  # noqa
+
+    HAS_HACCYTREES = True
+except ImportError:
+    HAS_HACCYTREES = False
+
+
+try:
+    assert os.path.isdir(DRN_LC_CORES_POBOY)
+    CAN_RUN_LJ_DATA_TESTS = True
+except AssertionError:
+    CAN_RUN_LJ_DATA_TESTS = False
+CAN_RUN_LJ_DATA_TESTS = CAN_RUN_LJ_DATA_TESTS & HAS_HACCYTREES
+POBOY_MSG = "This test only runs on poboy machine with haccytrees installed"
+
+
+@pytest.mark.skipif(not CAN_RUN_LJ_DATA_TESTS, reason=POBOY_MSG)
+def test_check_zrange_passes_for_all_last_journey_data_on_poboy():
+    """"""
+    bnpat = vlcc.BNPAT_LC_CORES.format("*", "*")
+    fn_list = glob(os.path.join(DRN_LC_CORES_POBOY, bnpat))
+    for fn_lc_cores in fn_list:
+        msg = vlcc.check_zrange(fn_lc_cores, "LastJourney")
+        assert len(msg) == 0
+
+
+@pytest.mark.skipif(not CAN_RUN_LJ_DATA_TESTS, reason=POBOY_MSG)
+def test_check_core_tag_uniqueness():
+    """Every core_tag should be unique. A very small number of repeats is allowed."""
+    bnpat = vlcc.BNPAT_LC_CORES.format("*", "*")
+    fn_list = glob(os.path.join(DRN_LC_CORES_POBOY, bnpat))
+    for fn_lc_cores in fn_list:
+        msg = vlcc.check_core_tag_uniqueness(fn_lc_cores)
+
+        if len(msg) > 0:
+            # No more than 10 core tags that have a repetition
+            s = msg[1]
+            n_distinct_repeats = int(s.split("=")[-1])
+            assert n_distinct_repeats < 10
+
+            # Only a single repetition is allowed
+            s = msg[2]
+            max_repetitions = int(s.split("=")[-1])
+            assert max_repetitions == 2
+
+
+@pytest.mark.skipif(not CAN_RUN_LJ_DATA_TESTS, reason=POBOY_MSG)
+def test_check_top_host_tag_has_match():
+    """top_host_tag should always agree with the result recalculated by diffsky
+
+    In the current implementation, this test is skipped for lc_cores in which
+    there is a repeated entry of core_tag
+
+    """
+    bnpat = vlcc.BNPAT_LC_CORES.format("*", "*")
+    fn_list = glob(os.path.join(DRN_LC_CORES_POBOY, bnpat))
+    for fn_lc_cores in fn_list:
+        msg = vlcc.check_top_host_tag_has_match(fn_lc_cores)
+        if len(msg) > 0:
+            s = msg[0]
+            if "Could not run test" in s:
+                pass
+            else:
+                bn = os.path.basename(fn_lc_cores)
+                raise ValueError(f"{bn} has mismatching top_host_idx")
+
+
+@pytest.mark.skipif(not CAN_RUN_LJ_DATA_TESTS, reason=POBOY_MSG)
+def test_check_top_host_idx_tag_agreement():
+    """top_host_tag should always be consistent with top_host_idx"""
+    bnpat = vlcc.BNPAT_LC_CORES.format("*", "*")
+    fn_list = glob(os.path.join(DRN_LC_CORES_POBOY, bnpat))
+    for fn_lc_cores in fn_list:
+        msg = vlcc.check_top_host_idx_tag_agreement(fn_lc_cores)
+        if len(msg) > 0:
+            bn = os.path.basename(fn_lc_cores)
+            raise ValueError(f"{bn} has mismatching top_host_idx")

diffsky-0.2.1/diffsky/data_loaders/hacc_utils/data_validation/validate_lc_cores.py

@@ -0,0 +1,104 @@
+""" """
+
+import os
+
+import numpy as np
+
+from ....utils import crossmatch
+from .. import lightcone_utils as lcu
+from .. import load_flat_hdf5
+
+BNPAT_LC_CORES = "lc_cores-{0}.{1}.hdf5"
+
+
+def check_zrange(fn_lc_cores, sim_name, tol=0.0002, lc_cores=None):
+    """Redshift range of the data should be bounded by the expected range"""
+
+    bn_lc_cores = os.path.basename(fn_lc_cores)
+    a_min_expected, a_max_expected = lcu.get_a_range_of_lc_cores_file(
+        bn_lc_cores, sim_name
+    )
+
+    if lc_cores is None:
+        lc_cores = load_flat_hdf5(fn_lc_cores)
+
+    a_min_data = lc_cores["scale_factor"].min()
+    a_max_data = lc_cores["scale_factor"].max()
+
+    msg = []
+    if a_min_data < a_min_expected - tol:
+        msg.append(f"a_min_data={a_min_data} < a_min_expected={a_min_expected}\n")
+    if a_max_data > a_max_expected + tol:
+        msg.append(f"a_max_data={a_max_data} > a_max_expected={a_max_expected}\n")
+
+    return msg
+
+
+def check_core_tag_uniqueness(fn_lc_cores, lc_cores=None):
+    if lc_cores is None:
+        lc_cores = load_flat_hdf5(fn_lc_cores)
+
+    msg = []
+    u_core_tags, counts = np.unique(lc_cores["core_tag"], return_counts=True)
+    if u_core_tags.size < lc_cores["core_tag"].size:
+        example_repeated_core_tag = u_core_tags[counts > 1][0]
+        s = f"repeated core_tag = {example_repeated_core_tag}"
+        msg.append(s)
+        n_distinct_repeats = np.sum(counts > 1)
+        max_repetitions = counts.max()
+        s = f"Number of distinct repeats = {n_distinct_repeats}"
+        msg.append(s)
+        s = f"Max num repetitions = {max_repetitions}"
+        msg.append(s)
+    return msg
+
+
+def check_top_host_tag_has_match(fn_lc_cores, lc_cores=None):
+
+    if lc_cores is None:
+        lc_cores = load_flat_hdf5(fn_lc_cores)
+
+    u_core_tag, u_indx, counts = np.unique(
+        lc_cores["core_tag"], return_counts=True, return_index=True
+    )
+
+    msg = []
+    if u_core_tag.size < lc_cores["core_tag"].size:
+        s = "Could not run test of top_host_tag due to repeated values of core_tag"
+        msg.append(s)
+    else:
+        idxA, idxB = crossmatch(lc_cores["top_host_tag"], lc_cores["core_tag"])
+        indarr = np.arange(len(u_core_tag)).astype(int)
+        indarr[idxA] = indarr[idxB]
+
+        if not np.allclose(indarr, lc_cores["top_host_idx"]):
+            msk = indarr != lc_cores["top_host_idx"]
+            example_core_tag = lc_cores["core_tag"][msk][0]
+            example_top_host_tag = lc_cores["top_host_tag"][msk][0]
+            s = f"core_tag = {example_core_tag} has top_host_tag={example_top_host_tag} "
+            s += "which disagrees with diffsky.utils.crossmatch"
+            msg.append(s)
+
+            n_mismatches = msk.sum()
+            s = f"Number of discrepancies between top_host_tag and diffsky = {n_mismatches}"
+            msg.append(s)
+
+    return msg
+
+
+def check_top_host_idx_tag_agreement(fn_lc_cores, lc_cores=None):
+    """top_host_tag should always be consistent with top_host_idx"""
+    if lc_cores is None:
+        lc_cores = load_flat_hdf5(fn_lc_cores)
+
+    msg = []
+    top_host_tag2 = lc_cores["core_tag"][lc_cores["top_host_idx"]]
+    if not np.allclose(lc_cores["top_host_tag"], top_host_tag2):
+        msk = top_host_tag2 != lc_cores["top_host_idx"]
+        example_core_tag = lc_cores["core_tag"][msk][0]
+        example_top_host_tag = lc_cores["top_host_tag"][msk][0]
+        s = f"core_tag = {example_core_tag} has top_host_tag={example_top_host_tag} "
+        s += "which disagrees with the value inferred from the `top_host_idx` column"
+        msg.append(s)
+
+    return msg

diffsky-0.2.1/diffsky/data_loaders/hacc_utils/defaults.py

@@ -0,0 +1,14 @@
+""""""
+
+# flake8: noqa
+
+__all__ = ("N_MIN_MAH_PTS", "DIFFMAH_MASS_COLNAME", "N_PTCL_COREFOREST")
+
+# Simulated MAHs with fewer points than N_MIN_MAH_PTS will get a synthetic MAH
+N_MIN_MAH_PTS = 4
+
+# DIFFMAH_MASS_COLNAME is the column used in the diffmah fits of the coreforest files
+DIFFMAH_MASS_COLNAME = "infall_tree_node_mass"
+
+# Minimum number of particles to use in coreforest analyses for each simulation
+N_PTCL_COREFOREST = 100