G2PInsight 1.1.2__tar.gz → 1.1.4__tar.gz

{g2pinsight-1.1.2 → g2pinsight-1.1.4}/G2PInsight/__init__.py

@@ -1,3 +1,3 @@
-__version__ = '1.1.2'
+__version__ = '1.1.4'
 __author__ = 'chenrf'
 __email__ = '12024128035@stu.ynu.edu.cn'

{g2pinsight-1.1.2 → g2pinsight-1.1.4}/G2PInsight/bin/modeltraining.py

@@ -848,7 +848,7 @@ def resolve_training_hyperparameters(model_type: str, task_type: str, X_train: p
         scoring = 'neg_mean_squared_error'
     base_model = init_model(model_type, task_type, random_state=random_state, cpu_cores=cpu_cores)
     search = RandomizedSearchCV(estimator=base_model, param_distributions=param_grid, n_iter=n_iter, cv=cv_splitter, scoring=scoring, n_jobs=DEFAULT_N_JOBS, random_state=random_state, verbose=0)
-    search.fit(X_search, y_search)
+    search.fit(_features_for_estimator(model_type, X_search), y_search)
     tuned_model = search.best_estimator_
     return (tuned_model, tuned_model.get_params())
 TRAIN_TEST_HOLDOUT_RATIO = 0.20
@@ -907,16 +907,25 @@ def resolve_holdout_split_from_metadata(X: pd.DataFrame, y: pd.Series, metadata:
         return holdout_from_sample_split(X, y, sample_split)
     logger.warning('Metadata missing sample_split; falling back to local reproducible 80/20 split. Re-run preprocess to fix the split at feature-selection time.')
     return split_train_test_holdout(X, y, task_type=task_type, test_ratio=TRAIN_TEST_HOLDOUT_RATIO, random_state=random_state)
+def _is_catboost_model(model: Any) -> bool:
+    return isinstance(model, (CatBoostClassifier, CatBoostRegressor))
+def _features_for_estimator(model_or_type: Any, X: Any) -> Any:
+    """CatBoost+pandas2: old CatBoost calls DataFrame.iteritems (removed); use ndarray."""
+    use_numpy = model_or_type == 'CatBoost' if isinstance(model_or_type, str) else _is_catboost_model(model_or_type)
+    if use_numpy and isinstance(X, (pd.DataFrame, pd.Series)):
+        return np.asarray(X)
+    return X
 def _predict_with_optional_proba(model: Any, X: pd.DataFrame, task_type: str) -> Tuple[np.ndarray, Optional[np.ndarray]]:
-    y_pred = model.predict(X)
+    X_in = _features_for_estimator(model, X)
+    y_pred = model.predict(X_in)
     y_prob = None
     if task_type != 'classification':
         return (y_pred, None)
     try:
         if hasattr(model, 'predict_proba'):
-            y_prob = model.predict_proba(X)
+            y_prob = model.predict_proba(X_in)
         elif hasattr(model, 'decision_function'):
-            decision_scores = model.decision_function(X)
+            decision_scores = model.decision_function(X_in)
             from sklearn.utils.extmath import softmax
             if len(decision_scores.shape) == 1:
                 prob_neg = 1 / (1 + np.exp(decision_scores))
@@ -933,7 +942,7 @@ def _fit_model_with_params(model_type: str, task_type: str, X_train: pd.DataFram
         model.set_params(**best_params)
     except ValueError as e:
         pass
-    model.fit(X_train, y_train)
+    model.fit(_features_for_estimator(model_type, X_train), y_train)
     return model
 def evaluate_model(y_true: pd.Series, y_pred: np.ndarray, y_prob: np.ndarray, task_type: str) -> Dict:
     metrics = {}
@@ -1091,14 +1100,16 @@ def _run_one_cv_fold(fold_idx: int, n_folds: int, train_idx: np.ndarray, val_idx
     y_pred_fold = None
     y_prob_fold = None
     try:
-        model_fold.fit(X_train_fold, y_train_fold)
-        y_pred_fold = model_fold.predict(X_val_fold)
+        X_tr = _features_for_estimator(model_type, X_train_fold)
+        X_va = _features_for_estimator(model_type, X_val_fold)
+        model_fold.fit(X_tr, y_train_fold)
+        y_pred_fold = model_fold.predict(X_va)
         if task_type == 'classification':
             try:
                 if hasattr(model_fold, 'predict_proba'):
-                    y_prob_fold = model_fold.predict_proba(X_val_fold)
+                    y_prob_fold = model_fold.predict_proba(X_va)
                 elif hasattr(model_fold, 'decision_function'):
-                    decision_scores = model_fold.decision_function(X_val_fold)
+                    decision_scores = model_fold.decision_function(X_va)
                     from sklearn.utils.extmath import softmax
                     if len(decision_scores.shape) == 1:
                         prob_neg = 1 / (1 + np.exp(decision_scores))
@@ -1279,7 +1290,7 @@ def calculate_shap_values(model: Any, model_type: str, feature_names: List[str],
             y_shap = y_shap.loc[X_input.index]
         if model_type in ['LightGBM', 'XGBoost', 'CatBoost', 'RandomForest']:
             explainer = shap.TreeExplainer(model)
-            shap_values = explainer.shap_values(X_input)
+            shap_values = explainer.shap_values(_features_for_estimator(model_type, X_input))
         elif model_type == 'Logistic':
             explainer = shap.LinearExplainer(model, X_input)
             shap_values = explainer.shap_values(X_input)
@@ -2008,8 +2019,9 @@ def predict_with_model(input_path: str, model_path: str, output_dir: str, task_t
         non_zero_counts = (X_aligned != 0).sum(axis=1)
         if (non_zero_counts == 0).all():
             logger.error(f'  ERROR: All feature values are zero! This will result in constant predictions.\n  Matched features: {matched_count}\n  Missing features: {missing_count}\n  Please check if chr_pos keys match between training and prediction data.')
-        y_pred = model.predict(X_aligned)
-        y_prob = model.predict_proba(X_aligned) if resolved_task_type == 'classification' and hasattr(model, 'predict_proba') else None
+        X_pred = _features_for_estimator(model, X_aligned)
+        y_pred = model.predict(X_pred)
+        y_prob = model.predict_proba(X_pred) if resolved_task_type == 'classification' and hasattr(model, 'predict_proba') else None
         unique_predictions = len(np.unique(y_pred))
         if unique_predictions == 1:
             logger.error(f"  ERROR: All predictions are the same value: {y_pred[0]}\n  This usually indicates one of the following issues:\n  1. chr_pos keys don't match between training and prediction data\n  2. All feature values are zero (check matched features: {matched_count})\n  3. Model is predicting a constant value for all samples\n  Please check the feature alignment logs above.")

{g2pinsight-1.1.2 → g2pinsight-1.1.4}/G2PInsight/bin/plink_ld.py

@@ -161,9 +161,9 @@ def run_ld_filtering(input_path: str, output_prefix: str, ld_window_kb: int=50,
             return 1
         if not check_prune_file_exists(output_prefix):
             return 1
+        # Prune may use --keep (e.g. train-only LD), but the final bed must keep
+        # all input samples so held-out test IDs remain in the training matrix.
         extract_args = input_args + ['--extract', f'{output_prefix_abs}.prune.in', '--make-bed', '--allow-extra-chr', '--allow-no-sex', '--threads', str(threads), '--out', output_prefix_abs]
-        if keep_samples_file:
-            extract_args.extend(['--keep', keep_samples_file])
         if not run_plink_command(plink_executable, extract_args, 'Extracting SNPs to generate bed/bim/fam'):
             return 1
         bed_file = Path(f'{output_prefix_abs}.bed')

{g2pinsight-1.1.2 → g2pinsight-1.1.4}/G2PInsight/bin/preprocess.py

@@ -1019,6 +1019,23 @@ def _validate_ids_file_pair(train_ids_file: Optional[str], test_ids_file: Option
     has_test = test_ids_file is not None
     if has_train ^ has_test:
         raise ValueError('Both --train-ids-file and --test-ids-file must be provided together')
+def _reconcile_sample_split_with_matrix(sample_split: Dict[str, Any], matrix_sample_ids: List[str]) -> Dict[str, Any]:
+    """Keep sample_split IDs that actually appear in the exported training matrix."""
+    matrix_set = {str(s) for s in matrix_sample_ids}
+    train_ids = [str(s) for s in sample_split.get('train_ids') or [] if str(s) in matrix_set]
+    test_ids = [str(s) for s in sample_split.get('test_ids') or [] if str(s) in matrix_set]
+    dropped_train = len(sample_split.get('train_ids') or []) - len(train_ids)
+    dropped_test = len(sample_split.get('test_ids') or []) - len(test_ids)
+    if dropped_train or dropped_test:
+        logger.warning(f'sample_split vs matrix: dropped {dropped_train} train / {dropped_test} test ID(s) absent from the genotype matrix')
+    if not train_ids or not test_ids:
+        raise RuntimeError(f'After building the training matrix, sample_split is invalid (n_train={len(train_ids)}, n_test={len(test_ids)}). Re-run preprocess; if using -f 3/4, ensure LD export keeps all samples.')
+    out = dict(sample_split)
+    out['train_ids'] = train_ids
+    out['test_ids'] = test_ids
+    out['n_train'] = len(train_ids)
+    out['n_test'] = len(test_ids)
+    return out
 def resolve_sample_split(sample_ids: List[str], pheno_df: pd.DataFrame, task_type: str, train_ids_file: Optional[str]=None, test_ids_file: Optional[str]=None, test_ratio: float=TRAIN_TEST_HOLDOUT_RATIO, random_state: int=DEFAULT_SPLIT_RANDOM_STATE) -> Dict[str, Any]:
     _validate_ids_file_pair(train_ids_file, test_ids_file)
     sample_ids_str = [str(s) for s in sample_ids]
@@ -1247,6 +1264,7 @@ def run_preprocess(genotype_path: str, genotype_format: GenotypeFormat, phenotyp
         actual_train_file = matrix_output.train_file
         includes_phenotype_column = matrix_output.includes_phenotype_column
         n_snp_features = matrix_output.n_snp_features
+        sample_split = _reconcile_sample_split_with_matrix(sample_split, matrix_output.sample_ids)
         generate_metadata(output_prefix=str(stage_output_dir / output_prefix), plink_prefix=gwas_genotype_prefix, train_file=actual_train_file, valid_samples=matrix_output.sample_ids, genotype_format=genotype_format, task_type=task_type, preprocess_tmp_dir=tmp_dir, filter_snps=filter_snps, feature_selection_mode=feature_selection_mode, gwas_pvalue=gwas_pvalue, ld_window_kb=ld_window_kb, ld_window=ld_window, ld_window_r2=ld_window_r2, n_feature_columns=n_snp_features, includes_phenotype_column=includes_phenotype_column, sample_split=sample_split)
         logger.info(f'Preprocess completed: {len(matrix_output.sample_ids):,} samples × {n_snp_features:,} SNPs (mode={feature_selection_mode}, split={sample_split["mode"]})')
         gc.collect()

{g2pinsight-1.1.2 → g2pinsight-1.1.4}/G2PInsight.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: G2PInsight
-Version: 1.1.2
+Version: 1.1.4
 Summary: Genome-wide association analysis toolkit
 Author-email: chenrf <2010288571@qq.com>
 License: MIT
@@ -38,7 +38,7 @@ Requires-Dist: lightgbm<4.0.0,>=3.3; python_version < "3.13"
 Requires-Dist: lightgbm>=4.0; python_version >= "3.13"
 Requires-Dist: xgboost<2.1,>=1.6; python_version < "3.13"
 Requires-Dist: xgboost>=2.1; python_version >= "3.13"
-Requires-Dist: catboost<2.0,>=1.0
+Requires-Dist: catboost<2.0,>=1.2
 Requires-Dist: shap>=0.40; python_version < "3.13"
 Requires-Dist: shap>=0.48; python_version >= "3.13"
 Requires-Dist: plotly>=5.0
@@ -183,26 +183,26 @@ wget -r -np -nH --cut-dirs=1 -R "index.html*" https://pr-nxy.ynu.edu.cn/G2PInsig
 **Alternative — `git clone`:** the same files are in the GitHub repo under `data/` (~53 MB VCF; clone may be slow on some networks).
 
 ```bash
-# 1. Preprocess
+# 1. Preprocess  →  test/preprocess/preprocess/preprocess_metadata.json
 G2PInsight preprocess --vcf data/genotype.vcf -p data/pheno.txt -o test/preprocess
 ```
 ```bash
-# 2. Train (check metadata exists first)
+# 2. Train  →  test/train/train/LightGBM/
 G2PInsight train -j test/preprocess/preprocess/preprocess_metadata.json -m LightGBM -o test/train
 ```
 ```bash
-# 3. Visualize
+# 3. Visualize  →  test/plot/visualize/lightgbm_*
 G2PInsight visualize \
-  -i test/train/LightGBM/LightGBM_shap_values.txt \
-  -I test/train/LightGBM/LightGBM_plotting_data.json \
-  -o results/plot/lightgbm
+  -i test/train/train/LightGBM/LightGBM_shap_values.txt \
+  -I test/train/train/LightGBM/LightGBM_plotting_data.json \
+  -o test/plot/lightgbm
 ```
 
 **Larger SNP sets** — use preprocess feature selection (`-f 4`), then train conservatively:
 
 ```bash
 G2PInsight preprocess --vcf data/genotype.vcf -p data/pheno.txt -o test/preprocess -f 4
-G2PInsight train-all -j test/preprocess/preprocess/preprocess_metadata.json -o test/train/ \
+G2PInsight train-all -j test/preprocess/preprocess/preprocess_metadata.json -o test/train \
   --parallel_models 1 --threads 4
 ```
 
@@ -535,6 +535,14 @@ Default is `--shap_dependence_top 50`. Use `0` only if you need all SNPs (can be
 
 Your env has an old `bed-reader` (0.2.x). Upgrade: `pip install -U 'bed-reader>=1.0'`. Current G2PInsight also accepts the older `fam_filepath` API as a fallback.
 
+### train / train-all: test IDs from metadata missing in matrix
+
+Usually from an older preprocess with **`-f 3` / `-f 4`**: LD used train-only `--keep` and the final `.bed` dropped test samples while metadata still listed them. Fixed in current source — **re-run preprocess**, then train again.
+
+### train-all / CatBoost: `DataFrame` has no attribute `iteritems`
+
+Old CatBoost + **pandas 2.x** (removed `iteritems`). Current G2PInsight passes numpy arrays into CatBoost. Temporary workaround: `pip install -U 'catboost>=1.2,<2'` or `pip install 'pandas>=1.3,<2'`.
+
 ### train: XGBoost “Mismatched version … Python package … Shared object”
 
 Usually **conda + pip mixed installs**: e.g. conda left `libxgboost.so` 2.0.x while pip upgraded the Python package to 3.x. Fix in the same env:

{g2pinsight-1.1.2 → g2pinsight-1.1.4}/G2PInsight.egg-info/requires.txt

@@ -2,7 +2,7 @@ scikit-learn>=1.0
 joblib>=1.0
 matplotlib>=3.5
 Pillow<12.3,>=8
-catboost<2.0,>=1.0
+catboost<2.0,>=1.2
 plotly>=5.0
 kaleido<1,>=0.2
 seaborn>=0.11

{g2pinsight-1.1.2 → g2pinsight-1.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: G2PInsight
-Version: 1.1.2
+Version: 1.1.4
 Summary: Genome-wide association analysis toolkit
 Author-email: chenrf <2010288571@qq.com>
 License: MIT
@@ -38,7 +38,7 @@ Requires-Dist: lightgbm<4.0.0,>=3.3; python_version < "3.13"
 Requires-Dist: lightgbm>=4.0; python_version >= "3.13"
 Requires-Dist: xgboost<2.1,>=1.6; python_version < "3.13"
 Requires-Dist: xgboost>=2.1; python_version >= "3.13"
-Requires-Dist: catboost<2.0,>=1.0
+Requires-Dist: catboost<2.0,>=1.2
 Requires-Dist: shap>=0.40; python_version < "3.13"
 Requires-Dist: shap>=0.48; python_version >= "3.13"
 Requires-Dist: plotly>=5.0
@@ -183,26 +183,26 @@ wget -r -np -nH --cut-dirs=1 -R "index.html*" https://pr-nxy.ynu.edu.cn/G2PInsig
 **Alternative — `git clone`:** the same files are in the GitHub repo under `data/` (~53 MB VCF; clone may be slow on some networks).
 
 ```bash
-# 1. Preprocess
+# 1. Preprocess  →  test/preprocess/preprocess/preprocess_metadata.json
 G2PInsight preprocess --vcf data/genotype.vcf -p data/pheno.txt -o test/preprocess
 ```
 ```bash
-# 2. Train (check metadata exists first)
+# 2. Train  →  test/train/train/LightGBM/
 G2PInsight train -j test/preprocess/preprocess/preprocess_metadata.json -m LightGBM -o test/train
 ```
 ```bash
-# 3. Visualize
+# 3. Visualize  →  test/plot/visualize/lightgbm_*
 G2PInsight visualize \
-  -i test/train/LightGBM/LightGBM_shap_values.txt \
-  -I test/train/LightGBM/LightGBM_plotting_data.json \
-  -o results/plot/lightgbm
+  -i test/train/train/LightGBM/LightGBM_shap_values.txt \
+  -I test/train/train/LightGBM/LightGBM_plotting_data.json \
+  -o test/plot/lightgbm
 ```
 
 **Larger SNP sets** — use preprocess feature selection (`-f 4`), then train conservatively:
 
 ```bash
 G2PInsight preprocess --vcf data/genotype.vcf -p data/pheno.txt -o test/preprocess -f 4
-G2PInsight train-all -j test/preprocess/preprocess/preprocess_metadata.json -o test/train/ \
+G2PInsight train-all -j test/preprocess/preprocess/preprocess_metadata.json -o test/train \
   --parallel_models 1 --threads 4
 ```
 
@@ -535,6 +535,14 @@ Default is `--shap_dependence_top 50`. Use `0` only if you need all SNPs (can be
 
 Your env has an old `bed-reader` (0.2.x). Upgrade: `pip install -U 'bed-reader>=1.0'`. Current G2PInsight also accepts the older `fam_filepath` API as a fallback.
 
+### train / train-all: test IDs from metadata missing in matrix
+
+Usually from an older preprocess with **`-f 3` / `-f 4`**: LD used train-only `--keep` and the final `.bed` dropped test samples while metadata still listed them. Fixed in current source — **re-run preprocess**, then train again.
+
+### train-all / CatBoost: `DataFrame` has no attribute `iteritems`
+
+Old CatBoost + **pandas 2.x** (removed `iteritems`). Current G2PInsight passes numpy arrays into CatBoost. Temporary workaround: `pip install -U 'catboost>=1.2,<2'` or `pip install 'pandas>=1.3,<2'`.
+
 ### train: XGBoost “Mismatched version … Python package … Shared object”
 
 Usually **conda + pip mixed installs**: e.g. conda left `libxgboost.so` 2.0.x while pip upgraded the Python package to 3.x. Fix in the same env:

{g2pinsight-1.1.2 → g2pinsight-1.1.4}/README.md

@@ -134,26 +134,26 @@ wget -r -np -nH --cut-dirs=1 -R "index.html*" https://pr-nxy.ynu.edu.cn/G2PInsig
 **Alternative — `git clone`:** the same files are in the GitHub repo under `data/` (~53 MB VCF; clone may be slow on some networks).
 
 ```bash
-# 1. Preprocess
+# 1. Preprocess  →  test/preprocess/preprocess/preprocess_metadata.json
 G2PInsight preprocess --vcf data/genotype.vcf -p data/pheno.txt -o test/preprocess
 ```
 ```bash
-# 2. Train (check metadata exists first)
+# 2. Train  →  test/train/train/LightGBM/
 G2PInsight train -j test/preprocess/preprocess/preprocess_metadata.json -m LightGBM -o test/train
 ```
 ```bash
-# 3. Visualize
+# 3. Visualize  →  test/plot/visualize/lightgbm_*
 G2PInsight visualize \
-  -i test/train/LightGBM/LightGBM_shap_values.txt \
-  -I test/train/LightGBM/LightGBM_plotting_data.json \
-  -o results/plot/lightgbm
+  -i test/train/train/LightGBM/LightGBM_shap_values.txt \
+  -I test/train/train/LightGBM/LightGBM_plotting_data.json \
+  -o test/plot/lightgbm
 ```
 
 **Larger SNP sets** — use preprocess feature selection (`-f 4`), then train conservatively:
 
 ```bash
 G2PInsight preprocess --vcf data/genotype.vcf -p data/pheno.txt -o test/preprocess -f 4
-G2PInsight train-all -j test/preprocess/preprocess/preprocess_metadata.json -o test/train/ \
+G2PInsight train-all -j test/preprocess/preprocess/preprocess_metadata.json -o test/train \
   --parallel_models 1 --threads 4
 ```
 
@@ -486,6 +486,14 @@ Default is `--shap_dependence_top 50`. Use `0` only if you need all SNPs (can be
 
 Your env has an old `bed-reader` (0.2.x). Upgrade: `pip install -U 'bed-reader>=1.0'`. Current G2PInsight also accepts the older `fam_filepath` API as a fallback.
 
+### train / train-all: test IDs from metadata missing in matrix
+
+Usually from an older preprocess with **`-f 3` / `-f 4`**: LD used train-only `--keep` and the final `.bed` dropped test samples while metadata still listed them. Fixed in current source — **re-run preprocess**, then train again.
+
+### train-all / CatBoost: `DataFrame` has no attribute `iteritems`
+
+Old CatBoost + **pandas 2.x** (removed `iteritems`). Current G2PInsight passes numpy arrays into CatBoost. Temporary workaround: `pip install -U 'catboost>=1.2,<2'` or `pip install 'pandas>=1.3,<2'`.
+
 ### train: XGBoost “Mismatched version … Python package … Shared object”
 
 Usually **conda + pip mixed installs**: e.g. conda left `libxgboost.so` 2.0.x while pip upgraded the Python package to 3.x. Fix in the same env:

{g2pinsight-1.1.2 → g2pinsight-1.1.4}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "G2PInsight"
-version = "1.1.2"
+version = "1.1.4"
 description = "Genome-wide association analysis toolkit"
 readme = "README.md"
 # Cap below 3.14: pandas/numpy for 3.14+ only ship manylinux_2_24+ wheels.
@@ -61,7 +61,8 @@ dependencies = [
     "lightgbm>=4.0; python_version >= '3.13'",
     "xgboost>=1.6,<2.1; python_version < '3.13'",
     "xgboost>=2.1; python_version >= '3.13'",
-    "catboost>=1.0,<2.0",
+    # >=1.2: pandas 2.x removed DataFrame.iteritems; older CatBoost still calls it.
+    "catboost>=1.2,<2.0",
     "shap>=0.40; python_version < '3.13'",
     "shap>=0.48; python_version >= '3.13'",
     "plotly>=5.0",