galform-analysis 0.1.0__py3-none-any.whl

galform_analysis/__init__.py

@@ -0,0 +1,86 @@
+"""galform_analysis - A Python library for GALFORM simulation analysis.
+
+This library provides tools for analyzing GALFORM galaxy formation simulation outputs,
+including:
+- Reading HDF5 snapshot data
+- Computing mass functions (stellar and halo)
+- Aggregating data across subvolumes
+
+Quick Start:
+    >>> from config import set_base_dir
+    >>> from analysis import avg_hmf_given_redshift_and_subvolumes
+    >>> from analysis import avg_smf_given_redshift_and_subvolumes
+    >>>
+    >>> # Set your GALFORM output directory
+    >>> set_base_dir('/path/to/galform/output')
+    >>>
+    >>> # Compute stellar mass function
+    >>> smf = avg_smf_given_redshift_and_subvolumes(iz_num=99, ivols=[0, 1, 2])
+
+Configuration:
+    Set the BASE_DIR for your GALFORM outputs:
+    - Via Python: config.set_base_dir('/path')
+    - Via environment: export GALFORM_BASE_DIR=/path
+    - Edit config.py directly
+"""
+
+__version__ = "0.1.0"
+
+# Import key modules for convenience
+from galform_analysis import analysis, config
+from galform_analysis.analysis import (
+    aggregate_snapshot,
+    avg_hmf_given_redshift_and_subvolumes,
+    avg_hmf_given_redshifts_and_subvolume,
+    avg_smf_given_redshift_and_subvolumes,
+    avg_smf_given_redshifts_and_subvolume,
+    # HMF functions
+    hmf_given_redshift_and_subvolume,
+    hmfs_given_redshifts_and_subvolume,
+    # SMF functions
+    smf_given_redshift_and_subvolume,
+    smfs_given_redshifts_and_subvolume,
+)
+
+# Expose commonly used functions at package level
+from galform_analysis.config import (
+    Cosmology,
+    SimulationConfig,
+    find_snapshot_at_redshift,
+    get_base_dir,
+    get_snapshot_redshift,
+    load_redshift_mapping,
+    load_sim_config,
+    set_base_dir,
+)
+from galform_analysis.readers import close_snapshot, read_snapshot_data
+
+__all__ = [
+    "__version__",
+    # Submodules
+    "config",
+    "io",
+    "analysis",
+    # Common functions
+    "set_base_dir",
+    "get_base_dir",
+    "Cosmology",
+    "load_sim_config",
+    "SimulationConfig",
+    "load_redshift_mapping",
+    "get_snapshot_redshift",
+    "find_snapshot_at_redshift",
+    "read_snapshot_data",
+    "close_snapshot",
+    "aggregate_snapshot",
+    # HMF functions
+    "hmf_given_redshift_and_subvolume",
+    "hmfs_given_redshifts_and_subvolume",
+    "avg_hmf_given_redshift_and_subvolumes",
+    "avg_hmf_given_redshifts_and_subvolume",
+    # SMF functions
+    "smf_given_redshift_and_subvolume",
+    "smfs_given_redshifts_and_subvolume",
+    "avg_smf_given_redshift_and_subvolumes",
+    "avg_smf_given_redshifts_and_subvolume",
+]

galform_analysis/analysis/__init__.py

@@ -0,0 +1,47 @@
+"""Analysis subpackage for GALFORM data processing."""
+
+from .aggregation import aggregate_snapshot, completed_galaxies, incomplete_subvolumes
+from .correlation import (
+    avg_correlation_given_redshift_and_subvolumes,
+    compute_xi_corrfunc,
+    correlation_given_redshift_and_subvolume,
+)
+from .mass_functions import (
+    avg_hmf_given_redshift_and_subvolumes,
+    avg_hmf_given_redshifts_and_subvolume,
+    avg_hod_given_redshift_and_subvolumes,
+    avg_hod_given_redshifts_and_subvolume,
+    avg_smf_given_redshift_and_subvolumes,
+    avg_smf_given_redshifts_and_subvolume,
+    hmf_given_redshift_and_subvolume,
+    hmfs_given_redshifts_and_subvolume,
+    hod_given_redshift_and_subvolume,
+    hods_given_redshifts_and_subvolume,
+    smf_given_redshift_and_subvolume,
+    smfs_given_redshifts_and_subvolume,
+)
+
+__all__ = [
+    "aggregate_snapshot",
+    "completed_galaxies",
+    "incomplete_subvolumes",
+    # HMF functions
+    "hmf_given_redshift_and_subvolume",
+    "hmfs_given_redshifts_and_subvolume",
+    "avg_hmf_given_redshift_and_subvolumes",
+    "avg_hmf_given_redshifts_and_subvolume",
+    # SMF functions
+    "smf_given_redshift_and_subvolume",
+    "smfs_given_redshifts_and_subvolume",
+    "avg_smf_given_redshift_and_subvolumes",
+    "avg_smf_given_redshifts_and_subvolume",
+    # HOD functions
+    "hod_given_redshift_and_subvolume",
+    "hods_given_redshifts_and_subvolume",
+    "avg_hod_given_redshift_and_subvolumes",
+    "avg_hod_given_redshifts_and_subvolume",
+    # Correlation functions
+    "compute_xi_corrfunc",
+    "correlation_given_redshift_and_subvolume",
+    "avg_correlation_given_redshift_and_subvolumes",
+]

galform_analysis/analysis/aggregation.py

@@ -0,0 +1,314 @@
+"""Analysis functions for aggregating GALFORM data across subvolumes."""
+
+import glob
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import h5py
+import numpy as np
+import polars as pl
+
+from galform_analysis.config import get_base_dir
+from galform_analysis.readers.loaders import close_snapshot, read_snapshot_data
+
+
+def completed_galaxies(
+    basedir: str = get_base_dir(), iz_snapshots: Optional[List[int]] = None
+) -> pl.DataFrame:
+    """Scan base directory and return DataFrame of all completed galaxy files.
+
+    Looks through all iz*/ivol* directories and checks CompletionFlag in
+    galaxies.hdf5 files.
+
+    Args:
+        basedir: Base directory containing iz* snapshot folders
+        iz_snapshots: Optional list of snapshot numbers (e.g., [82, 100, 105]).
+                     If provided, only these snapshots will be scanned.
+                     If None, all iz* directories are scanned.
+
+    Returns:
+        DataFrame with columns:
+            - iz: Snapshot name (e.g., 'iz100')
+            - iz_num: Numeric iz value (e.g., 100)
+            - ivol: Subvolume number
+            - path: Full path to the galaxies.hdf5 file
+            - completed: Whether CompletionFlag==1
+    """
+    records = []
+
+    # Find all iz* directories
+    if iz_snapshots is not None:
+        # Filter to only the requested snapshots
+        iz_dirs = sorted(
+            [
+                os.path.join(basedir, f"iz{iz}")
+                for iz in iz_snapshots
+                if os.path.isdir(os.path.join(basedir, f"iz{iz}"))
+            ]
+        )
+    else:
+        iz_dirs = sorted(glob.glob(os.path.join(basedir, "iz*")))
+
+    for iz_dir in iz_dirs:
+        iz_name = Path(iz_dir).name
+        iz_records = []  # Track records for this redshift only
+
+        # Extract numeric iz value
+        try:
+            iz_num = int(iz_name.replace("iz", ""))
+        except ValueError:
+            continue
+
+        ivol_dirs = sorted(glob.glob(os.path.join(iz_dir, "ivol*")))
+
+        for ivol_dir in ivol_dirs:
+            ivol_name = Path(ivol_dir).name
+
+            try:
+                ivol_num = int(ivol_name.replace("ivol", ""))
+            except ValueError:
+                continue
+
+            # Check for galaxies.hdf5 file
+            gal_file = os.path.join(ivol_dir, "galaxies.hdf5")
+
+            if not os.path.exists(gal_file):
+                continue
+
+            # Quick file size check - empty or very small files are incomplete
+            try:
+                file_size = os.path.getsize(gal_file)
+                if file_size < 1000:  # Less than 1KB is definitely incomplete
+                    record = {
+                        "iz": iz_name,
+                        "iz_num": iz_num,
+                        "ivol": ivol_num,
+                        "path": gal_file,
+                        "completed": False,
+                    }
+                    records.append(record)
+                    iz_records.append(record)
+                    continue
+            except OSError:
+                continue
+
+            # Try to open the file - if it fails with serialization error,
+            # it's incomplete
+            completed = False
+
+            try:
+                # Use swmr mode for faster read access
+                with h5py.File(gal_file, "r", swmr=True):
+                    # If we can open it without error, it's completed
+                    completed = True
+            except (OSError, KeyError, RuntimeError) as e:
+                # Check if it's the specific serialization error indicating
+                # incomplete file
+                if "Can't deserialize" in str(e) or "bad object header" in str(e):
+                    completed = False
+                else:
+                    # Other errors might be temporary, but mark as incomplete
+                    completed = False
+
+            record = {
+                "iz": iz_name,
+                "iz_num": iz_num,
+                "ivol": ivol_num,
+                "path": gal_file,
+                "completed": completed,
+            }
+            records.append(record)
+            iz_records.append(record)
+
+    df = pl.DataFrame(records)
+
+    if not df.is_empty():
+        df = df.sort(["iz_num", "ivol"])
+
+    return df
+
+
+def incomplete_subvolumes(
+    basedir: str = get_base_dir(), iz_snapshots: Optional[List[int]] = None
+) -> pl.DataFrame:
+    """Scan base directory and return DataFrame of incomplete/missing galaxy files.
+
+    This is the complement of completed_galaxies(). Returns records for subvolumes
+    where galaxies.hdf5 either doesn't exist or is incomplete/corrupted.
+
+    Args:
+        basedir: Base directory containing iz* snapshot folders
+        iz_snapshots: Optional list of snapshot numbers (e.g., [82, 100, 105]).
+                     If provided, only these snapshots will be scanned.
+                     If None, all iz* directories are scanned.
+
+    Returns:
+        DataFrame with columns:
+            - iz: Snapshot name (e.g., 'iz100')
+            - iz_num: Numeric iz value (e.g., 100)
+            - ivol: Subvolume number
+            - path: Path to the expected galaxies.hdf5 file (may not exist)
+            - reason: Why the file is incomplete ('missing', 'incomplete',
+              or 'corrupted')
+    """
+    records = []
+
+    # Find all iz* directories
+    if iz_snapshots is not None:
+        # Filter to only the requested snapshots
+        iz_dirs = sorted(
+            [
+                os.path.join(basedir, f"iz{iz}")
+                for iz in iz_snapshots
+                if os.path.isdir(os.path.join(basedir, f"iz{iz}"))
+            ]
+        )
+    else:
+        iz_dirs = sorted(glob.glob(os.path.join(basedir, "iz*")))
+
+    for iz_dir in iz_dirs:
+        iz_name = Path(iz_dir).name
+        iz_incomplete = []  # Track incomplete records for this redshift
+
+        # Extract numeric iz value
+        try:
+            iz_num = int(iz_name.replace("iz", ""))
+        except ValueError:
+            continue
+
+        ivol_dirs = sorted(glob.glob(os.path.join(iz_dir, "ivol*")))
+
+        for ivol_dir in ivol_dirs:
+            ivol_name = Path(ivol_dir).name
+
+            try:
+                ivol_num = int(ivol_name.replace("ivol", ""))
+            except ValueError:
+                continue
+
+            # Check for galaxies.hdf5 file
+            gal_file = os.path.join(ivol_dir, "galaxies.hdf5")
+
+            if not os.path.exists(gal_file):
+                record = {
+                    "iz": iz_name,
+                    "iz_num": iz_num,
+                    "ivol": ivol_num,
+                    "path": gal_file,
+                    "reason": "missing",
+                }
+                records.append(record)
+                iz_incomplete.append(record)
+                continue
+
+            # Quick file size check - empty or very small files are incomplete
+            try:
+                file_size = os.path.getsize(gal_file)
+                if file_size < 1000:  # Less than 1KB is definitely incomplete
+                    record = {
+                        "iz": iz_name,
+                        "iz_num": iz_num,
+                        "ivol": ivol_num,
+                        "path": gal_file,
+                        "reason": "incomplete",
+                    }
+                    records.append(record)
+                    iz_incomplete.append(record)
+                    continue
+            except OSError:
+                record = {
+                    "iz": iz_name,
+                    "iz_num": iz_num,
+                    "ivol": ivol_num,
+                    "path": gal_file,
+                    "reason": "inaccessible",
+                }
+                records.append(record)
+                iz_incomplete.append(record)
+                continue
+
+            # Try to open the file - if it fails, it's corrupted
+            try:
+                # Use swmr mode for faster read access
+                with h5py.File(gal_file, "r", swmr=True):
+                    pass  # File is valid
+            except (OSError, KeyError, RuntimeError) as e:
+                # Check if it's the specific serialization error indicating
+                # incomplete file
+                if "Can't deserialize" in str(e) or "bad object header" in str(e):
+                    reason = "corrupted"
+                else:
+                    reason = "corrupted"
+
+                record = {
+                    "iz": iz_name,
+                    "iz_num": iz_num,
+                    "ivol": ivol_num,
+                    "path": gal_file,
+                    "reason": reason,
+                }
+                records.append(record)
+                iz_incomplete.append(record)
+
+    df = pl.DataFrame(records)
+
+    if not df.is_empty():
+        df = df.sort(["iz_num", "ivol"])
+
+    return df
+
+
+def aggregate_snapshot(iz_path: str) -> Optional[Dict[str, Any]]:
+    """Aggregate mstar, mhalo, and volume from all ivols in a snapshot.
+
+    Args:
+        iz_path: Path to the snapshot directory
+
+    Returns:
+        Dictionary with keys: 'iz', 'z', 'volume', 'mstar', 'mhalo'
+        Returns None if no data found
+    """
+    ivol_paths = sorted(glob.glob(os.path.join(iz_path, "ivol*")))
+    if not ivol_paths:
+        return None
+
+    all_mstar, all_mhalo = [], []
+    total_vol = 0
+    z = None
+
+    for ivp in ivol_paths:
+        iv = int(Path(ivp).name.replace("ivol", ""))
+        try:
+            data = read_snapshot_data(iz_path, ivol=iv)
+            if data.get("V_ivol") and data["V_ivol"] > 0:
+                total_vol += data["V_ivol"]
+            if z is None:
+                z = data.get("z")
+
+            mstar = data.get("mstar")
+            mhalo = data.get("mhalo")
+            if mstar is not None:
+                all_mstar.append(mstar)
+            if mhalo is not None:
+                all_mhalo.append(mhalo)
+
+            close_snapshot(data)
+        except Exception:
+            continue
+
+    if not all_mstar and not all_mhalo:
+        return None
+
+    return {
+        "iz": Path(iz_path).name,
+        "z": z,
+        "volume": total_vol,
+        "mstar": np.concatenate(all_mstar) if all_mstar else np.array([]),
+        "mhalo": np.concatenate(all_mhalo) if all_mhalo else np.array([]),
+    }
+
+
+if __name__ == "__main__":
+    base_dir = get_base_dir()
+    df = completed_galaxies(str(base_dir))

galform_analysis/analysis/correlation/__init__.py

@@ -0,0 +1,68 @@
+"""Correlation function analysis subpackage."""
+
+from .correlation import (
+    avg_correlation_given_redshift_and_subvolumes,
+    avg_correlation_given_subvolume_and_redshifts,
+    compute_xi_corrfunc,
+    correlation_given_redshift_and_subvolume,
+    correlations_given_redshifts_and_subvolume,
+)
+
+try:
+    from .subvol_weighted_correction import (
+        compute_weighted_wp_for_n_list as compute_weighted_wp_for_n_list,
+    )
+    from .subvol_weighted_correction import (
+        compute_weighted_wp_from_catalogue as compute_weighted_wp_from_catalogue,
+    )
+    from .subvol_weighted_correction import (
+        compute_weighted_xi_for_n_list as compute_weighted_xi_for_n_list,
+    )
+    from .subvol_weighted_correction import (
+        compute_weighted_xi_from_catalogue as compute_weighted_xi_from_catalogue,
+    )
+    from .subvol_weighted_correction import (
+        load_subvolume_galaxies as load_subvolume_galaxies,
+    )
+
+    _HAS_SUBVOL_WEIGHTED = True
+except Exception:  # pragma: no cover - optional dependency path
+    _HAS_SUBVOL_WEIGHTED = False
+
+from .dm_correlation import (
+    avg_dm_correlation_from_tree_files,
+    dm_correlation_from_tree_file,
+)
+from .galaxy_bias import (
+    avg_galaxy_bias_over_subvolumes,
+    compute_galaxy_bias,
+)
+from .satellite_cross_correlation import (
+    compute_xi_cross_corrfunc,
+    satellite_central_cross_correlation,
+)
+
+__all__ = [
+    "compute_xi_corrfunc",
+    "correlation_given_redshift_and_subvolume",
+    "avg_correlation_given_redshift_and_subvolumes",
+    "correlations_given_redshifts_and_subvolume",
+    "avg_correlation_given_subvolume_and_redshifts",
+    "dm_correlation_from_tree_file",
+    "avg_dm_correlation_from_tree_files",
+    "compute_galaxy_bias",
+    "avg_galaxy_bias_over_subvolumes",
+    "satellite_central_cross_correlation",
+    "compute_xi_cross_corrfunc",
+]
+
+if _HAS_SUBVOL_WEIGHTED:
+    __all__.extend(
+        [
+            "compute_weighted_xi_for_n_list",
+            "compute_weighted_xi_from_catalogue",
+            "compute_weighted_wp_for_n_list",
+            "compute_weighted_wp_from_catalogue",
+            "load_subvolume_galaxies",
+        ]
+    )