gpse 0.0.2__py3-none-any.whl

gpse/config/_topsis_config.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+TOPSIS Configuration and Environment Logging Module
+====================================================
+Provides TOPSIS evaluation setup, representative model saving, and environment logging.
+"""
+
+import os
+import traceback
+import json
+import joblib
+import pandas as pd
+from pathlib import Path
+from sklearn.preprocessing import StandardScaler
+from typing import Any, Tuple
+
+from loguru import logger as main_logger
+
+from gpse.config.constants import ModelConstants
+from gpse.utils.configuration import load_topsis_config
+from gpse.utils.genomic_utils import create_representative_model_directory
+from threadpoolctl import threadpool_limits
+from gpse.train._feature_selection import (
+    make_model_artifact,
+)
+from gpse.train._model_pipeline import build_training_pipeline
+
+
+def _format_weight(weight: Any) -> str:
+    """Format a YAML weight value for TOPSISEvaluator's comma-string interface."""
+    if isinstance(weight, float):
+        return f"{weight:g}"
+    return str(weight)
+
+
+def _parse_topsis_task_config(task_type: str) -> Tuple[list, list, str]:
+    """Parse and validate the TOPSIS config for a task type."""
+    config = load_topsis_config()
+    task_configs = config.get("tasks", {})
+    if not isinstance(task_configs, dict):
+        raise ValueError("TOPSIS config must define 'tasks' as a mapping")
+
+    task_key = "classification" if task_type == "classification" else "regression"
+    task_config = task_configs.get(task_key)
+
+    if not isinstance(task_config, dict):
+        raise ValueError(f"Missing TOPSIS configuration for task '{task_key}'")
+
+    criteria_config = task_config.get("criteria")
+    if not isinstance(criteria_config, list) or not criteria_config:
+        raise ValueError(f"TOPSIS task '{task_key}' must define a non-empty criteria list")
+
+    criteria = []
+    criteria_types = []
+    weights = []
+    for index, item in enumerate(criteria_config, start=1):
+        if not isinstance(item, dict):
+            raise ValueError(f"TOPSIS criterion #{index} for task '{task_key}' must be a mapping")
+
+        name = item.get("name")
+        criterion_type = item.get("type")
+        weight = item.get("weight")
+
+        if not name:
+            raise ValueError(f"TOPSIS criterion #{index} for task '{task_key}' is missing 'name'")
+        if criterion_type not in {"max", "min"}:
+            raise ValueError(
+                f"TOPSIS criterion '{name}' for task '{task_key}' must use type 'max' or 'min'"
+            )
+        if weight is None:
+            raise ValueError(f"TOPSIS criterion '{name}' for task '{task_key}' is missing 'weight'")
+        try:
+            float(weight)
+        except (TypeError, ValueError) as exc:
+            raise ValueError(
+                f"TOPSIS criterion '{name}' for task '{task_key}' has a non-numeric weight"
+            ) from exc
+
+        criteria.append(str(name))
+        criteria_types.append(str(criterion_type))
+        weights.append(weight)
+
+    manual_weights = ",".join(_format_weight(weight) for weight in weights)
+    return criteria, criteria_types, manual_weights
+
+
+def get_topsis_configuration(self) -> Tuple[list, list, str]:
+    """
+    Get TOPSIS evaluation configuration based on task type.
+
+    Returns
+    -------
+    Tuple[list, list, str]
+        (criteria, criteria_types, manual_weights)
+    """
+    criteria, criteria_types, manual_weights = _parse_topsis_task_config(self.task_type)
+    task_name = "Classification" if self.task_type == "classification" else "Regression"
+    main_logger.info(
+        f"TOPSIS config: {task_name} task - "
+        f"{' : '.join(criteria)}; weights={manual_weights}"
+    )
+
+    return criteria, criteria_types, manual_weights
+
+
+def _save_representative_model(
+    self,
+    model_name: str,
+    params: dict,
+    X: pd.DataFrame,
+    y: pd.Series,
+    model_dir: Path,
+    repeat_info: dict,
+) -> str:
+    """
+    Train and save a representative model.
+
+    Parameters
+    ----------
+    model_name : str
+        Model name.
+    params : dict
+        Model parameters.
+    X : pd.DataFrame
+        Feature matrix.
+    y : pd.Series
+        Target variable.
+    model_dir : Path
+        Model directory.
+    repeat_info : dict
+        Repeat information dictionary.
+
+    Returns
+    -------
+    str
+        Model save path, or None if saving failed.
+    """
+    representative_model_dir = create_representative_model_directory(model_dir)
+
+    try:
+        pipeline = build_training_pipeline(
+            self.create_model(model_name, params),
+            task_type=self.task_type,
+            model_name=model_name,
+            feature_selection_config=self.feature_selection_config,
+            genotype_imputation_config=self.genotype_imputation_config,
+        )
+
+        with threadpool_limits(limits=self.n_threads):
+            pipeline.fit(X, y)
+        preprocessor = pipeline.named_steps["preprocess"]
+
+        model_path = representative_model_dir / ModelConstants.model_pkl_file
+        joblib.dump(
+            make_model_artifact(
+                pipeline.named_steps["model"],
+                preprocessor.scaler_,
+                preprocessor.selector_,
+                self.feature_selection_config,
+                preprocessor.selected_features_,
+                imputer=preprocessor.imputer_,
+                imputation_config=self.genotype_imputation_config,
+                task_type=self.task_type,
+            ),
+            model_path,
+        )
+
+        info_path = representative_model_dir / ModelConstants.model_info_file
+        with open(info_path, "w") as f:
+            json.dump(
+                {
+                    **repeat_info,
+                    "feature_selection": self.feature_selection_config.as_dict(),
+                    "genotype_imputation": self.genotype_imputation_config.as_dict(),
+                    "selected_feature_count": len(preprocessor.selected_features_),
+                },
+                f,
+                indent=2,
+            )
+
+        return str(model_path)
+
+    except Exception as e:
+        main_logger.error(f"Saving representative model failed: {str(e)}")
+        main_logger.error(traceback.format_exc())
+        return None
+
+
+def log_environment_settings(self) -> None:
+    """Log environment settings."""
+    main_logger.info("Initializing GenomicPredictorV2, base configuration:")
+    main_logger.info(f"- Results directory: {self.results_dir}")
+    main_logger.info(f"- Logs directory: {self.logs_dir}")
+    main_logger.info(f"- Random seed: {self.random_seed}")
+    main_logger.info(f"- Threads per model: {self.n_threads}")
+    main_logger.info(f"- Model workers: {self.max_parallel_jobs}")
+    main_logger.info(f"- Repeat workers: {self.repeat_workers}")
+    main_logger.info(f"- Train folds: {self.n_splits}")
+    main_logger.info(f"- Repeats: {self.n_repeats}")
+    main_logger.info(f"- Use default params: {self.use_default_params}")
+
+    for env_var in ModelConstants.thread_env_vars:
+        main_logger.info(f"{env_var}={os.environ.get(env_var, 'not set')}")

gpse/config/constants.py

@@ -0,0 +1,127 @@
+"""Global constant configuration and log message templates."""
+
+import argparse
+import json
+import numpy as np
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from gpse.utils.cli_display import show_gpse_logo
+
+
+class _LogoHelpAction(argparse.Action):
+    """Custom help action that prints the logo before the help text."""
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        show_gpse_logo()
+        parser.print_help()
+        parser.exit()
+
+
+@dataclass
+class ModelConfig:
+    """Configuration for model hyperparameter optimization"""
+    model_class: Any
+    param_func: Callable
+    is_regression: bool
+
+
+@dataclass
+class ClassificationModelConfig:
+    """Configuration for classification model hyperparameter optimization"""
+    model_class: Any
+    param_func: Callable
+
+
+class NumpyEncoder(json.JSONEncoder):
+    """
+    JSON encoder that handles NumPy types
+    """
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return super(NumpyEncoder, self).default(obj)
+
+
+@dataclass(frozen=True)
+class _ModelConstants:
+    """
+    Constants configuration related to model training
+    """
+    # Optuna optimization related
+    optuna_n_startup_trials: int = 10
+    optuna_n_warmup_steps: int = 5
+    optuna_interval_steps: int = 1
+    optuna_n_jobs: int = 1  # Enforce Optuna serial execution
+
+    # Thread environment variables
+    thread_env_vars: tuple = field(default_factory=lambda: (
+        'OMP_NUM_THREADS', 'MKL_NUM_THREADS', 'OPENBLAS_NUM_THREADS',
+        'NUMEXPR_NUM_THREADS', 'VECLIB_MAXIMUM_THREADS', 'BLIS_NUM_THREADS'
+    ))
+
+    # Default directory names
+    default_results_dir: str = "optimization_results"
+    default_logs_dir: str = "logs"
+    default_cv_dir: str = "cv_folds"
+    representative_model_dir: str = "representative_model"
+
+    # Default file names
+    main_log_file: str = "main.log"
+    model_log_file: str = "model.log"
+    repeat_results_file: str = "repeat_results.json"
+    summary_results_file: str = "summary_results.json"
+    all_predictions_file: str = "all_predictions.json"
+    model_info_file: str = "info.json"
+    model_pkl_file: str = "model.pkl"
+    comparison_csv_file: str = "model_comparison.csv"
+
+    # Log format templates
+    log_separator_short: str = field(default_factory=lambda: "=" * 50)
+    log_separator_long: str = field(default_factory=lambda: "=" * 70)
+
+    # Numerical precision constants
+    metrics_precision: int = 6
+    time_precision: int = 2
+    float_precision: int = 4
+    correlation_min_threshold: float = 1e-10  # Threshold to avoid constant predictions
+
+
+# NOTE: _LogMessages has been deprecated. All log messages are now inline strings.
+# If you need to restore it, uncomment the block below.
+#
+# @dataclass(frozen=True)
+# class _LogMessages:
+#     """Log message template constants"""
+#     logging_initialized: str = "Logging system initialization complete"
+#     predictor_init: str = "Initializing GenomicPredictorV2, basic configuration:"
+#     data_loading: str = "Loading data..."
+#     data_loaded: str = "Successfully loaded genotype data: {}, phenotype data: {}"
+#     common_samples: str = "Number of genotype samples: {}, number of phenotype samples: {}"
+#     final_data_info: str = "Final data dimensions - Number of features: {}, number of samples: {}"
+#     target_stats: str = "Target variable statistics - Mean: {:.4f}, Std: {:.4f}"
+#     model_training_start: str = "Starting training for model: {} repeat: {}"
+#     repeat_training_start: str = "Starting {} repetitions of training for model {}"
+#     fold_training_start: str = "Training fold {}/{}"
+#     parameter_optimization_start: str = "Starting parameter optimization for model {} (repeat {}/{})"
+#     optimization_start: str = "Starting parameter optimization, max trials: {}"
+#     fold_results: str = "Fold {} results:"
+#     repeat_average_performance: str = "Repeat {} average performance:"
+#     model_summary_performance: str = "Model {} - average performance across {} repeats:"
+#     ensemble_test_performance: str = "Fold ensemble test set performance:"
+#     data_file_not_found: str = "Failed to load data file: {}"
+#     data_file_read_error: str = "Error reading data file: {}"
+#     target_trait_not_found: str = "Target trait '{}' not found in phenotype data. Available columns: {}"
+#     id_column_missing_geno: str = "ID column '{}' missing in genotype data"
+#     id_column_missing_pheno: str = "ID column '{}' missing in phenotype data"
+#     no_common_samples: str = "No common sample IDs between genotype and phenotype data"
+#     dimension_mismatch: str = "Mismatch in number of samples between feature matrix and target variable: {} vs {}"
+
+
+# Module-level singleton instance
+ModelConstants = _ModelConstants()
+# LogMessages = _LogMessages()  # deprecated

gpse/config/default.yaml

@@ -0,0 +1,15 @@
+software:
+  version: "0.0.2"
+  app_name: gpse
+  author: "XIAOLIU, JZHANG"
+  email: "1468835852@qq.com, zhangjian199567@outlook.com"
+  url: https://github.com/xsx123123/GPSE
+  description: "Genomic Prediction with Stacking Ensemble for horticultural crops"
+
+
+logs:
+  log_level: INFO
+  more_info: false
+  Label: gpse
+
+

gpse/config/software.yaml

@@ -0,0 +1,20 @@
+software:
+  version: "0.0.2"
+  app_name: gpse
+  author: "XIAOLIU, JZHANG"
+  email: "1468835852@qq.com, zhangjian199567@outlook.com"
+  url: https://github.com/xsx123123/GPSE
+  description: "Genomic Prediction with Stacking Ensemble for horticultural crops"
+
+external_tools:
+  - name: plink
+    cmd: plink
+    version_flag: --version
+    min_version: "1.9"
+    required: true
+    description: "PLINK 1.9 for genomic data format conversion and SNP extraction"
+  - name: java
+    cmd: java
+    version_flag: -version
+    required: false
+    description: "Java runtime for optional Beagle imputation"

gpse/config/topsis.yaml

@@ -0,0 +1,18 @@
+tasks:
+  classification:
+    criteria:
+      - name: "Test Accuracy"
+        type: max
+        weight: 0.8
+      - name: "Test Accuracy (std)"
+        type: min
+        weight: 0.2
+
+  regression:
+    criteria:
+      - name: "Test Pearson"
+        type: max
+        weight: 0.8
+      - name: "Test Pearson (std)"
+        type: min
+        weight: 0.2

gpse/convert/__init__.py

@@ -0,0 +1,11 @@
+"""
+GPSE convert package — data conversion and genotype QC utilities.
+
+This package provides tools for converting between common genomic data
+formats, running quality-control filters, and preparing genotype matrices
+for downstream GPSE analyses.
+"""
+
+from .processor import GenomicDataProcessor
+
+__all__ = ["GenomicDataProcessor"]

gpse/convert/cli.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Command-line and workflow entry point for the GPSE convert module.
+
+This module does two things:
+1. Defines the command-line arguments for ``gpse convert``.
+2. Starts the selected convert workflow: genotype conversion, QC/LD pruning,
+   numeric recoding, or external dependency validation.
+"""
+from __future__ import annotations
+
+import os
+import sys
+import argparse
+from datetime import datetime
+
+from gpse.convert.workflow import run_convert_workflow, validate_convert_mode
+from gpse.utils.cli_display import _build_convert_parser
+from gpse.utils.log_utils import logger_init
+
+# define convert main function
+def main(argv: list[str] | None = None,*,formatter_class=None,prog: str | None = None,help_action=None,parents: list[argparse.ArgumentParser] | None = None) -> int:
+    """
+    Parse convert arguments, select a mode, and start the workflow.
+    """
+    parser = _build_convert_parser(
+        formatter_class = formatter_class or argparse.HelpFormatter,
+        prog = prog or "python -m gpse.convert.cli",
+        help_action = help_action,
+        parents = parents,
+    )
+    args = parser.parse_args(argv)
+
+    # Determine workflow mode first so that missing-argument errors and --help
+    # are handled before initialising the logger.  This avoids spurious
+    # "GPSE config loaded" messages when the user only wants usage info.
+    mode = validate_convert_mode(parser, args)
+
+    # Initialise unified GPSE logging so that convert uses the same
+    # Rich-styled console format and file rotation as the train module.
+    log_level = getattr(args, "log_level", "INFO")
+    log_file = None
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    if getattr(args, "out_prefix", None):
+        log_dir = os.path.dirname(args.out_prefix) or "."
+        os.makedirs(log_dir, exist_ok=True)
+        log_file = os.path.join(log_dir, f"gpse_convert_{timestamp}.log")
+    elif getattr(args, "recode_prefix", None):
+        log_dir = os.path.dirname(args.recode_prefix) or "."
+        os.makedirs(log_dir, exist_ok=True)
+        log_file = os.path.join(log_dir, f"gpse_convert_{timestamp}.log")
+    else:
+        # No output prefix specified (e.g. --check-deps).
+        # Skip file logging — only use console output.
+        log_file = None
+
+    logger_init(logger_name=log_file, log_level=log_level)
+
+    return run_convert_workflow(args, mode)
+
+
+if __name__ == "__main__":
+    # Allows direct execution: python -m gpse.convert.cli ...
+    sys.exit(main())