holistic-math-harness 0.0.0__tar.gz

holistic_math_harness-0.0.0/LICENSE

@@ -0,0 +1,27 @@
+
+---
+
+# 📄 **LICENSE (MIT License)**
+
+```text id="license001"
+MIT License
+
+Copyright (c) 2026 CIOL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

holistic_math_harness-0.0.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: holistic-math-harness
+Version: 0.0.0
+Summary: A lightweight framework for benchmarking multimodal AI agents with parallel execution, prompt variation, and automated evaluation.
+Author-email: Wahid Faisal <wahiddhrubo@gmail.com>
+License: MIT
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: alive-progress
+Requires-Dist: gitpython
+Requires-Dist: huggingface-hub
+Requires-Dist: openai
+Requires-Dist: scipy
+Requires-Dist: scikit-learn
+Requires-Dist: torch
+Requires-Dist: transformers
+Requires-Dist: umap-learn
+Requires-Dist: pillow
+Requires-Dist: cohere
+Requires-Dist: hdbscan
+Dynamic: license-file

holistic_math_harness-0.0.0/holistic_math_harness/__init__.py

@@ -0,0 +1,19 @@
+from .evaluator import HarnessEvaluator,structured_signal_extraction,cross_model_diagnostic_synthesis
+from .holistic_harness import HarnessRunner
+from .initialize_benchmark import process_benchmark, BenchmarkStats
+from .prompt_variation import PromptVariationGenerator
+
+
+
+__version__ = "0.1.5"
+
+__all__ = [
+    "HarnessEvaluator",
+    "cross_model_diagnostic_synthesis",
+    "structured_signal_extraction",
+    "HarnessRunner",
+    "process_benchmark",
+    "PromptVariationGenerator",
+    "BenchmarkStats"
+]
+

holistic_math_harness-0.0.0/holistic_math_harness/evaluator.py

@@ -0,0 +1,62 @@
+# ================= STANDARD LIB =================
+import os
+from pathlib import Path
+
+# ================= DATA =================
+import pandas as pd
+
+
+
+from .evaluator_cross_model_diagnostic_synthesis import cross_model_diagnostic_synthesis
+from .evaluator_error_taxonomy_induction import error_taxonomy_induction
+from .evaluator_structured_signal_extraction import structured_signal_extraction
+
+
+
+
+
+def HarnessEvaluator(results_path,custom_error_taxonomies=None,max_concurrent=10,model_name="gemma-4-31b-it"):
+
+    results_path=Path(results_path)
+
+
+    error_taxonomies= custom_error_taxonomies if custom_error_taxonomies else {
+        "E1":"perceptual failure — model misidentified or failed to detect a spatial element in the image",
+        "E2":"grounding failure — model identified elements but failed to anchor them in the correct spatial frame of reference",
+        "E3":"spatial transformation failure — model understood the spatial arrangement but failed to correctly apply a rotation, reflection, or perspective change)",
+        "E4":"quantitative spatial failure — failure on counting, distance estimation, or measurement",
+        "E5":"multi-step reasoning failure — model solved one spatial step correctly but failed on a subsequent step requiring the first as input",
+        "E6":"language-spatial mapping failure — model misunderstood the spatial language in the question, e.g., confused 'left of X' with 'right of X'",
+        "E7":"format/refusal failure — model did not produce a valid answer regardless of spatial competence",
+    } 
+
+    for model_folder in results_path.iterdir():
+        if model_folder.is_dir():
+            response_csv_path= model_folder / "agent_response.csv"
+            score_csv_path= model_folder / "agent_scores.csv"
+            prev_df=None
+            
+            if os.path.exists(score_csv_path) :
+                prev_df=pd.read_csv(score_csv_path)
+                
+            structured_signal_extraction(
+                response_csv_path,
+                max_concurrent=max_concurrent,
+                model_name=model_name,
+                prev_df=prev_df
+            )
+            error_taxonomy_induction(
+                score_csv_path,
+                error_taxonomies=error_taxonomies,
+                model_name=model_name,
+            )
+            
+            if score_csv_path.exists():
+                df=pd.read_csv(score_csv_path)
+                if "accuracy_score" in df.columns:
+                    print("Numbers of Failed Tasks: ",len(df[df["accuracy_score"].isna()]))
+
+    
+
+    cross_model_diagnostic_synthesis(results_path.rglob("agent_scores.csv"),results_path)
+

holistic_math_harness-0.0.0/holistic_math_harness/evaluator_cross_model_diagnostic_synthesis.py

@@ -0,0 +1,321 @@
+# ================= STANDARD LIB =================
+import json
+from itertools import combinations
+
+# ================= DATA =================
+import numpy as np
+import pandas as pd
+
+# ================= ML / STATS =================
+from scipy.stats import chi2
+from sklearn.preprocessing import MultiLabelBinarizer
+
+
+# ================= BOOTSTRAP CI =================
+def bootstrap_ci(scores_lst, n_bootstrap=1000, alpha=0.05):
+    scores_lst = np.asarray(scores_lst)
+    scores_lst = scores_lst[~np.isnan(scores_lst)]
+
+    if len(scores_lst) == 0:
+        return {"mean": np.nan, "ci_lower": np.nan, "ci_upper": np.nan, "std": np.nan}
+
+    boot_means = []
+    n = len(scores_lst)
+
+    for _ in range(n_bootstrap):
+        idx = np.random.randint(0, n, n)
+        sample = scores_lst[idx]
+        boot_means.append(np.mean(sample))  # ✅ scalar
+
+    boot_means = np.array(boot_means)
+
+    return {
+        "mean": float(np.mean(boot_means)),
+        "ci_lower": float(np.percentile(boot_means, 100 * alpha / 2)),
+        "ci_upper": float(np.percentile(boot_means, 100 * (1 - alpha / 2))),
+        "std": float(np.std(boot_means))
+    }
+
+
+# ================= COHEN'S D =================
+def cohens_d(x, y):
+    x, y = np.asarray(x), np.asarray(y)
+
+    x = x[~np.isnan(x)]
+    y = y[~np.isnan(y)]
+
+    nx, ny = len(x), len(y)
+    if nx < 2 or ny < 2:
+        return np.nan
+
+    sx, sy = np.var(x, ddof=1), np.var(y, ddof=1)
+    pooled = np.sqrt(((nx - 1) * sx + (ny - 1) * sy) / (nx + ny - 2))
+
+    return (np.mean(x) - np.mean(y)) / pooled if pooled != 0 else 0.0
+
+
+# ================= MCNEMAR TEST =================
+def mcnemar_test(x, y):
+    x, y = np.asarray(x), np.asarray(y)
+
+    mask = ~np.isnan(x) & ~np.isnan(y)
+    x, y = x[mask], y[mask]
+
+    b = np.sum((x == 1) & (y == 0))
+    c = np.sum((x == 0) & (y == 1))
+
+    if b + c == 0:
+        return 0.0, 1.0
+
+    stat = (abs(b - c) - 1) ** 2 / (b + c)
+    p = 1 - chi2.cdf(stat, df=1)
+
+    return float(stat), float(p)
+
+
+# ================= FULL PAIRWISE =================
+def full_pairwise_evaluation(model_dfs, score_col="accuracy_score"):
+    models = list(model_dfs.keys())
+    pairs = list(combinations(models, 2))
+
+    m = len(pairs)
+    bonf_alpha = 0.05 / m if m > 0 else 0.05
+
+    results = []
+
+    for m1, m2 in pairs:
+        df1 = model_dfs[m1]
+        df2 = model_dfs[m2]
+
+        # ✅ align by id
+        merged = df1.merge(df2, on="id", suffixes=("_1", "_2"))
+
+        x = merged[f"{score_col}_1"].values
+        y = merged[f"{score_col}_2"].values
+
+        # McNemar
+        mcn_stat, mcn_p = mcnemar_test(x, y)
+
+        # Cohen's d
+        d = cohens_d(x, y)
+
+        results.append({
+            "model_1": m1,
+            "model_2": m2,
+            "mcnemar_stat": mcn_stat,
+            "mcnemar_p": mcn_p,
+            "cohens_d": d,
+            "bonferroni_alpha": bonf_alpha,
+            "significant": mcn_p < bonf_alpha
+        })
+
+    df = pd.DataFrame(results)
+
+    # reverse pairs
+    if not df.empty:
+        rev = df.copy()
+        rev[["model_1", "model_2"]] = rev[["model_2", "model_1"]]
+        rev["cohens_d"] = -rev["cohens_d"]
+        df = pd.concat([df, rev], ignore_index=True)
+
+    return df
+
+
+# ================= MAIN PIPELINE =================
+def cross_model_diagnostic_synthesis(csv_paths, save_dir):
+    model_dfs = {}
+    model_evals = {}
+    error_types = set()
+
+    score_col = "accuracy_score"
+
+    # ================= LOAD + PREPROCESS =================
+    for csv_path in csv_paths:
+        df = pd.read_csv(csv_path)
+
+        if score_col not in df.columns:
+            continue
+
+        model_name = str(csv_path.parent).split("/")[-1]
+
+        df = df[df["error_type"].notna()].copy()
+        df["error_type"] = df["error_type"].apply(
+            lambda x: [x] if isinstance(x, str) else []
+        )
+
+        error_types.update(df["error_type"].explode().dropna().unique())
+
+        mlb = MultiLabelBinarizer()
+        dummies = pd.DataFrame(
+            mlb.fit_transform(df["error_type"]),
+            columns=mlb.classes_,
+            index=df.index
+        )
+
+        df = pd.concat([df, dummies], axis=1)
+
+        # clean scores
+        scores_clean = pd.to_numeric(df[score_col], errors="coerce").dropna()
+        refusal_score=pd.to_numeric(df["refusal_score"], errors="coerce").dropna()
+        off_topic_score=pd.to_numeric(df["off_topic_score"], errors="coerce").dropna()
+        spatial_elements_score=pd.to_numeric(df["spatial_elements_score"], errors="coerce").dropna()
+        spatial_relations_score=pd.to_numeric(df["spatial_relations_score"], errors="coerce").dropna()
+        spatial_reasoning_score=pd.to_numeric(df["spatial_reasoning_score"], errors="coerce").dropna()
+
+        model_evals[model_name] = {
+            "bootstrap_ci": bootstrap_ci(scores_clean, n_bootstrap=500),
+            "average_score": df[score_col].mean().round(5),
+            "error_types": list(mlb.classes_),
+            "overall_score": float(scores_clean.mean()) if len(scores_clean) > 0 else np.nan,
+            "refusal_score": float(refusal_score.mean()) if len(scores_clean) > 0 else np.nan,
+            "off_topic_score": float(off_topic_score.mean()) if len(scores_clean) > 0 else np.nan,
+            "spatial_elements_score": float(spatial_elements_score.mean()) if len(scores_clean) > 0 else np.nan,
+            "spatial_relations_score": float(spatial_relations_score.mean()) if len(scores_clean) > 0 else np.nan,
+            "spatial_reasoning_score": float(spatial_reasoning_score.mean()) if len(scores_clean) > 0 else np.nan,
+        }
+
+        model_dfs[model_name] = df
+
+    error_types = list(error_types)
+
+    # initialize
+    for model in model_evals:
+        model_evals[model]["Cohen_d"] = {}
+        model_evals[model]["Bonferroni"] = {}
+        model_evals[model]["Mcnemar"] = {}
+
+    # ================= PER ERROR TYPE =================
+    for error_type in error_types:
+        selected_models = [
+            m for m, v in model_evals.items()
+            if error_type in v["error_types"]
+        ]
+
+        selected_model_dfs = {
+            k: v for k, v in model_dfs.items() if k in selected_models
+        }
+
+        if len(selected_model_dfs) <= 1:
+            continue
+
+        eval_res_df = full_pairwise_evaluation(
+            selected_model_dfs, score_col=error_type
+        )
+
+        for model in selected_models:
+            model_evals[model].setdefault("Cohen_d", {}).setdefault(error_type, [])
+            model_evals[model].setdefault("Bonferroni", {}).setdefault(error_type, [])
+            model_evals[model].setdefault("Mcnemar", {}).setdefault(error_type, [])
+
+            for _, row in eval_res_df.iterrows():
+                if row["model_1"] == model:
+                    model_evals[model]["Cohen_d"][error_type].append({
+                        "to": row["model_2"],
+                        "cohens_d": row["cohens_d"]
+                    })
+
+                    model_evals[model]["Bonferroni"][error_type].append({
+                        "to": row["model_2"],
+                        "bonferroni_alpha": row["bonferroni_alpha"]
+                    })
+
+                    model_evals[model]["Mcnemar"][error_type].append({
+                        "to": row["model_2"],
+                        "mcnemar_stat": row["mcnemar_stat"],
+                        "mcnemar_p": row["mcnemar_p"]
+                    })
+
+    # ================= EXPORT =================
+    rows = []
+
+    for model, data in model_evals.items():
+
+        rows.append({
+            "model": model,
+            "to": None,
+            "metric": "Score",
+            "error_type": "Overall",
+            "value": data.get("average_score", 0)
+        })
+        rows.append({
+            "model": model,
+            "to": None,
+            "metric": "Refusal Score",
+            "error_type": "Overall",
+            "value": data.get("refusal_score", 0)
+        })
+        rows.append({
+            "model": model,
+            "to": None,
+            "metric": "Off Topic Score",
+            "error_type": "Overall",
+            "value": data.get("off_topic_score", 0)
+        })
+        rows.append({
+            "model": model,
+            "to": None,
+            "metric": "Spatial Elements Score",
+            "error_type": "Overall",
+            "value": data.get("spatial_elements_score", 0)
+        })
+        rows.append({
+            "model": model,
+            "to": None,
+            "metric": "Spatial Relations Score",
+            "error_type": "Overall",
+            "value": data.get("spatial_relations_score", 0)
+        })
+        rows.append({
+            "model": model,
+            "to": None,
+            "metric": "Spatial Reasoning Score",
+            "error_type": "Overall",
+            "value": data.get("spatial_reasoning_score", 0)
+        })
+        # bootstrap
+        bc = data.get("bootstrap_ci", {})
+        if bc:
+            rows.append({
+                "model": model,
+                "to": None,
+                "metric": "bootstrap_mean",
+                "error_type": "Overall",
+                "value": bc.get("mean")
+            })
+
+        # Cohen d
+        for e, lst in data.get("Cohen_d", {}).items():
+            for v in lst:
+                rows.append({
+                    "model": model,
+                    "to": v["to"],
+                    "metric": "cohens_d",
+                    "error_type": e,
+                    "value": v["cohens_d"]
+                })
+
+        # Bonferroni
+        for e, lst in data.get("Bonferroni", {}).items():
+            for v in lst:
+                rows.append({
+                    "model": model,
+                    "to": v["to"],
+                    "metric": "bonferroni_alpha",
+                    "error_type": e,
+                    "value": v["bonferroni_alpha"]
+                })
+
+        # McNemar
+        for e, lst in data.get("Mcnemar", {}).items():
+            for v in lst:
+                rows.append({
+                    "model": model,
+                    "to": v["to"],
+                    "metric": "mcnemar_stat",
+                    "error_type": e,
+                    "value": v["mcnemar_stat"]
+                })
+
+
+    pd.DataFrame(rows).to_csv(save_dir / "model_evaluations.csv", index=False)
+    json.dump(model_evals, open(save_dir / "model_evaluations.json", "w"), indent=4)