hal-spatial-harness 0.1.0__tar.gz

hal_spatial_harness-0.1.0/LICENSE

@@ -0,0 +1,27 @@
+
+---
+
+# 📄 **LICENSE (MIT License)**
+
+```text id="license001"
+MIT License
+
+Copyright (c) 2026 CIOL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

hal_spatial_harness-0.1.0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: hal-spatial-harness
+Version: 0.1.0
+Summary: A lightweight framework for benchmarking multimodal AI agents with parallel execution, prompt variation, and automated evaluation.
+Author-email: Wahid Faisal <wahiddhrubo@gmail.com>
+License: MIT
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: alive-progress
+Requires-Dist: gitpython
+Requires-Dist: huggingface-hub
+Requires-Dist: openai
+Dynamic: license-file

hal_spatial_harness-0.1.0/hal_spatial_harness.egg-info/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: hal-spatial-harness
+Version: 0.1.0
+Summary: A lightweight framework for benchmarking multimodal AI agents with parallel execution, prompt variation, and automated evaluation.
+Author-email: Wahid Faisal <wahiddhrubo@gmail.com>
+License: MIT
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: alive-progress
+Requires-Dist: gitpython
+Requires-Dist: huggingface-hub
+Requires-Dist: openai
+Dynamic: license-file

hal_spatial_harness-0.1.0/hal_spatial_harness.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+LICENSE
+pyproject.toml
+hal_spatial_harness.egg-info/PKG-INFO
+hal_spatial_harness.egg-info/SOURCES.txt
+hal_spatial_harness.egg-info/dependency_links.txt
+hal_spatial_harness.egg-info/requires.txt
+hal_spatial_harness.egg-info/top_level.txt
+hal_spatial_harness_package/__init__.py
+hal_spatial_harness_package/evaluator.py
+hal_spatial_harness_package/hal_harness.py
+hal_spatial_harness_package/initialize_benchmark.py
+hal_spatial_harness_package/prompt_variation.py
+hal_spatial_harness_package/utils.py

hal_spatial_harness-0.1.0/hal_spatial_harness.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

hal_spatial_harness-0.1.0/hal_spatial_harness.egg-info/requires.txt

@@ -0,0 +1,6 @@
+pandas
+numpy
+alive-progress
+gitpython
+huggingface-hub
+openai

hal_spatial_harness-0.1.0/hal_spatial_harness.egg-info/top_level.txt

@@ -0,0 +1 @@
+hal_spatial_harness_package

hal_spatial_harness-0.1.0/hal_spatial_harness_package/__init__.py

@@ -0,0 +1,17 @@
+from .evaluator import evaluation_result
+from .hal_harness import HarnessRunner
+from .initialize_benchmark import initialize_benchmark, BenchmarkStats
+from .prompt_variation import PromptVariationGenerator
+
+
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "evaluation_result",
+    "HarnessRunner",
+    "initialize_benchmark",
+    "PromptVariationGenerator",
+    "BenchmarkStats"
+]
+

hal_spatial_harness-0.1.0/hal_spatial_harness_package/evaluator.py

@@ -0,0 +1,245 @@
+from utils import get_client, encode_image
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Dict
+
+# 3rd-party packages
+import pandas as pd
+from alive_progress import alive_bar
+
+
+def compute_metrics(df: pd.DataFrame):
+    TP = (df["score"] == 1).sum()
+    FP = (df["score"] == 0).sum()
+    FN = FP
+    precision = TP / (TP + FP) if (TP + FP) > 0 else 0
+    recall = TP / (TP + FN) if (TP + FN) > 0 else 0
+    f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0
+    return round(precision*100,2), round(recall*100,2), round(f1*100,2)
+
+
+
+
+
+def llm_judge_binary(task, model_name="gpt-4o-mini-2024-07-18"):
+    client = get_client()
+
+    prompt = f"""
+You are a strict evaluator.
+
+Return ONLY:
+1 → if the model answer is correct
+0 → if the model answer is incorrect
+
+No explanation.
+
+Question:
+{task['question']}
+
+Model Answer:
+{task['response']}
+
+Choices:
+{task['choices']}
+
+Correct Answer:
+{task['answer']}
+
+
+"""
+    images_paths = [p for p in  task["ques_image_path_lst"]]
+    
+    content = [{"type": "text", "text": prompt}]
+    
+    content += [
+        {
+            "type": "image_url",
+            "image_url": {"url": f"data:image/jpeg;base64,{encode_image(path)}"}
+        }
+        for path in images_paths
+    ]
+    
+    response = client.chat.completions.create(
+        model=model_name,
+        temperature=0,
+        messages=[{"role": "user", "content": content}],
+        max_tokens=5,
+    )
+
+    output = response.choices[0].message.content.strip()
+
+    # safe parsing
+    if "1" in output:
+        return 1
+    elif "0" in output:
+        return 0
+    else:
+        return 0.5  # fallback if model misbehaves
+
+
+def evaluator(
+    dataset: Dict,
+    evaluate_mcq_by_judge_llm: bool,
+    mcq_match_full_answer: bool,
+    var: bool = False,
+    eval_path: str = "",
+    max_workers: int = 4
+) -> Dict:
+    """
+    Parallel evaluation of dataset with optional LLM scoring.
+    """
+    evaluation = {}
+
+    # Exclude already-evaluated items
+    if eval_path and os.path.exists(eval_path):
+        
+        exclude_ids = set(pd.read_csv(eval_path)["id"].to_list())
+        dataset = {k: v for k, v in dataset.items() if k not in exclude_ids}
+        evaluation=pd.read_csv(eval_path).set_index('id').to_dict(orient='index')
+
+    def evaluate_item(d_id, data):
+        """Evaluate a single dataset item and return result tuple"""
+        # MCQ scoring
+        if data.get("question_type") == "MCQ" and not evaluate_mcq_by_judge_llm:
+            if mcq_match_full_answer:
+                score = 1 if data["answer"] in data["response"] else 0
+            else:
+                score = 1 if data["correct_option"] == data["response"] else 0
+        else:
+            score = llm_judge_binary(task=data)
+        
+        key_id = "".join(d_id.split("-")[:2]) if var else d_id
+        return key_id, data, score
+
+
+    # Run in parallel
+    with alive_bar(len(dataset), bar='blocks', spinner='dots', title='Running Evaluation', force_tty=True) as bar:
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            future_to_id = {executor.submit(evaluate_item, d_id, data): d_id for d_id, data in dataset.items()}
+            
+            for future in as_completed(future_to_id):
+                key_id, data, score = future.result()
+                
+                # Merge results safely
+                if var and key_id in evaluation:
+                    evaluation[key_id]["prompt"].append(data["task_description"])
+                    evaluation[key_id]["response"].append(data["response"])
+                    evaluation[key_id]["scores"].append(score)
+                else:
+                    evaluation[key_id] = {**data}
+                    if var:
+                        evaluation[key_id]["prompt"] = [data["task_description"]]
+                        evaluation[key_id]["response"] = [data["response"]]
+                        evaluation[key_id]["scores"] = [score]
+                    else:
+                        evaluation[key_id]["score"] = score
+
+                bar()  # update progress
+
+    # Compute averaged scores for var=True
+    if var:
+        for i in evaluation:
+            evaluation[i]["score"] = sum(evaluation[i]["scores"]) / len(evaluation[i]["scores"])
+
+    # Save final CSV once
+    if eval_path:
+        pd.DataFrame.from_dict(evaluation, orient="index").rename_axis("id").reset_index().to_csv(eval_path, index=False)
+        
+    return evaluation
+
+
+def get_evaluation_result(df):
+    TP = (df["score"] == 1).sum()
+    FP = (df["score"] == 0).sum()
+    
+    # If every row has a prediction, FN ~ FP for binary correctness
+    FN = FP  
+
+    precision = TP / (TP + FP) if (TP + FP) > 0 else 0
+    recall = TP / (TP + FN) if (TP + FN) > 0 else 0
+    f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0
+
+    return round(precision*100,2), round(recall*100,2), round(f1*100,2)
+
+
+
+def evaluation_result(eval_path):
+    df = pd.read_csv(eval_path)
+
+    # ---------------- Aggregations ----------------
+    cat_df = (
+        df.groupby("category")
+        .apply(lambda x: pd.Series({
+            "cat_score": x["score"].mean()*100,
+            "cat_count": len(x),
+            "precision": compute_metrics(x)[0],
+            "recall": compute_metrics(x)[1],
+            "f1": compute_metrics(x)[2],
+        }))
+        .reset_index()
+    )
+
+    sub_df = (
+        df.groupby(["category", "sub_category"])
+        .apply(lambda x: pd.Series({
+            "subcat_score": x["score"].mean()*100,
+            "subcat_count": len(x),
+            "precision": compute_metrics(x)[0],
+            "recall": compute_metrics(x)[1],
+            "f1": compute_metrics(x)[2],
+        }))
+        .reset_index()
+    )
+
+    task_df = (
+        df.groupby(["category", "sub_category", "task"])
+        .apply(lambda x: pd.Series({
+            "task_score": x["score"].mean()*100,
+            "task_count": len(x),
+            "precision": compute_metrics(x)[0],
+            "recall": compute_metrics(x)[1],
+            "f1": compute_metrics(x)[2],
+        }))
+        .reset_index()
+    )
+
+    # Merge
+    merged = (
+        task_df
+        .merge(sub_df, on=["category", "sub_category"])
+        .merge(cat_df, on="category")
+    )
+
+    # Sort worst → best
+    merged = merged.sort_values(
+        by=["cat_score", "subcat_score", "task_score"],
+        ascending=[True, True, True]
+    )
+
+    # ---------------- Pretty Print ----------------
+    for cat, cat_group in merged.groupby("category", sort=False):
+        row = cat_group.iloc[0]
+        print(f"\n📊 Category: {cat} | Score: {row['cat_score']:.2f}% | Count: {row['cat_count']}")
+        print(f"   Precision: {row['precision']}% | Recall: {row['recall']}% | F1: {row['f1']}%")
+        print("-" * 90)
+
+        for sub, sub_group in cat_group.groupby("sub_category", sort=False):
+            row = sub_group.iloc[0]
+            print(f"  🔹 Sub-category: {sub} | Score: {row['subcat_score']:.2f}% | Count: {row['subcat_count']}")
+            print(f"     Precision: {row['precision']}% | Recall: {row['recall']}% | F1: {row['f1']}%")
+
+            for _, r in sub_group.iterrows():
+                print(
+                    f"      ▪ Task: {r['task']:<25} "
+                    f"| Score: {r['task_score']:>6.2f}% "
+                    f"| P: {r['precision']:>5}% "
+                    f"| R: {r['recall']:>5}% "
+                    f"| F1: {r['f1']:>5}% "
+                    f"| Count: {r['task_count']}"
+                )
+
+    # ---------------- Overall ----------------
+    p, r, f1 = compute_metrics(df)
+    print("\nOverall Score:", round(df["score"].mean()*100, 2), "%")
+    print(f"Overall Precision: {p}% | Recall: {r}% | F1: {f1}%")
+

hal_spatial_harness-0.1.0/hal_spatial_harness_package/hal_harness.py

@@ -0,0 +1,230 @@
+# =========================
+# Standard library
+# =========================
+import os
+import logging
+import ast
+import shutil
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+# =========================
+# 3rd-party packages
+# =========================
+import pandas as pd
+from alive_progress import alive_bar
+
+# =========================
+# Local modules
+# =========================
+from initialize_benchmark import initialize_benchmark, process_benchmark, prompt_generator
+from prompt_variation import PromptVariationGenerator, generate_var_bench
+from evaluator import evaluation_result, evaluator
+from utils import encode_image, get_client, verify_run_name
+
+
+
+
+logger = logging.getLogger(__name__)
+
+
+
+
+
+def agent(inputs: dict[str, dict], model_name: str,**kwargs):    
+    
+    client = get_client()
+
+    task_id, task = list(inputs.items())[0]
+
+    images_paths = [p for p in  task["ques_image_path_lst"]]
+    
+    # build content list
+    content = [{"type": "text", "text": task["task_description"]}]
+    
+    # add all images
+    content += [
+        {
+            "type": "image_url",
+            "image_url": {"url": f"data:image/jpeg;base64,{encode_image(path)}"}
+        }
+        for path in images_paths
+    ]
+    
+
+    response=None
+    
+    try:
+        response = client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {
+                    "role": "user",
+                    "content": content
+                }
+            ],
+            temperature=0.1,
+            max_tokens=1000,
+        )
+        
+    except Exception as e:
+        print(f"Error Answering task {task_id}: {e}")
+
+    
+    inputs[task_id]["response"] = response.choices[0].message.content.strip()
+
+    return inputs
+
+
+def run_agent_function(inputs: dict[str, dict], agent_function , use_template: bool =False, model_name: str = "" , **kwargs):
+    if use_template:
+        res=agent_function(inputs,model_name=model_name)
+    else:
+        res=agent_function(inputs,)
+
+    return res
+                
+    
+
+
+def HarnessRunner(
+        use_template=True,
+        custom_function=None,
+        template_name="default_agent",
+        dataset_dir=None,
+        clean_dataset_dir=True,
+        template_model="gpt-4o-mini-2024-07-18",
+        run_id = "test",
+        max_concurrent = 5,
+        continue_run = False,
+        max_tasks = 0,
+        prompt_sensitivity = False,
+        num_variations = 3,
+        variation_strength = "mild",
+        results_dir = "results",
+        task_ids = [],
+        categories = [],
+        sub_categories = [],
+        openai_api_key = None,
+        open_router_api_key = None,
+        evaluation_needed=True,
+        evaluate_mcq_by_judge_llm=False,
+        mcq_match_full_answer=False,
+    ):
+
+    
+    function_template = {
+        "default_agent": agent,
+    }
+
+    if openai_api_key:
+        os.environ["OPENAI_API_KEY"]=openai_api_key
+        
+    if open_router_api_key:
+        os.environ["OPEN_ROUTER_API_KEY"]=open_router_api_key
+
+    
+        
+    if (not use_template) and (not custom_function):
+        raise ValueError(
+            "Select use_template or provide custom_function "
+        )
+        
+    
+    if use_template:
+        agent_function= function_template[template_name]
+    else:
+        agent_function= custom_function
+
+    temp_dir="./temp_dir"
+    verify_run_name(run_id)
+    Path(f"{results_dir}/{run_id}").mkdir(parents=True, exist_ok=True)
+    response_path=f"{results_dir}/{run_id}/agent_response.csv"
+    eval_path=f"{results_dir}/{run_id}/evaluations.csv"
+
+    
+    if os.path.exists(response_path) and (not continue_run):
+        raise ValueError("Run Already Exist. Pass  continue_run if you want to Continue Run.")
+
+    if dataset_dir:
+        temp_dir=dataset_dir
+    if not os.path.exists(temp_dir) :
+        initialize_benchmark(temp_dir)
+    
+    
+    
+    df=pd.read_csv(f"{temp_dir}/annotations.csv").drop_duplicates(subset=["id"])
+    
+    df["ques_image_path_lst"] = df["ques_image_path_lst"].apply(ast.literal_eval)
+
+    df["ques_image_path_lst"] = df["ques_image_path_lst"].apply(
+        lambda paths: [os.path.join(temp_dir, "question_images", p) for p in paths]
+    )
+    
+    dataset = process_benchmark(
+        df,
+        categories,
+        sub_categories,
+        task_ids,
+        max_tasks,
+        continue_run=continue_run,
+        eval_path=response_path,
+    )
+
+    eval_results = {}
+
+    if continue_run:
+        eval_results=pd.read_csv(response_path).set_index('id').to_dict(orient='index')
+    
+    var_dataset=None
+
+
+    if prompt_sensitivity:
+        
+        
+        generator = PromptVariationGenerator(num_variations=num_variations, strength=variation_strength)
+        prompt_map = generator.generate_variations_for_dataset(dataset)
+        var_dataset=generate_var_bench(dataset, prompt_map)
+
+    active_dataset = var_dataset if var_dataset else dataset
+    
+    def process_item(key, item):
+        """Generate prompt and run agent for a single item"""
+        value = prompt_generator(item)
+        res = run_agent_function(
+            inputs={key: value},
+            agent_function=agent_function,
+            use_template=use_template,
+            model_name=template_model,
+        )
+        return key, res[key]
+
+
+    # Run in parallel with progress bar
+    with alive_bar(len(active_dataset), bar='blocks', spinner='dots', title='Running Agent', force_tty=True) as bar:
+        with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
+            futures = {executor.submit(process_item, k, v): k for k, v in active_dataset.items()}
+
+            for future in as_completed(futures):
+                key, result = future.result()
+                eval_results[key] = result
+                pd.DataFrame.from_dict(eval_results, orient="index").rename_axis("id").reset_index().to_csv(response_path, index=False)
+                bar()  # update progress
+
+    if evaluation_needed:
+        # Run evaluator on results
+        evaluation = evaluator(
+            dataset=eval_results,
+            evaluate_mcq_by_judge_llm=evaluate_mcq_by_judge_llm,
+            mcq_match_full_answer=mcq_match_full_answer,
+            var= True if  var_dataset else False,
+            eval_path=eval_path,
+        )
+        evaluation_result(eval_path)
+    
+    if clean_dataset_dir:
+        shutil.rmtree(temp_dir)
+
+
+
+

hal_spatial_harness-0.1.0/hal_spatial_harness_package/initialize_benchmark.py

@@ -0,0 +1,143 @@
+import os
+from pathlib import Path
+
+# 3rd-party packages
+import pandas as pd
+from alive_progress import alive_bar
+from huggingface_hub import snapshot_download
+from zipfile import ZipFile
+
+
+
+class BenchmarkStats:
+    def __init__(self, ):
+        self.df=pd.read_csv("./annotations.csv")
+        
+    def _print_stats(self, column, title):
+        pd.set_option("display.max_rows", None)
+
+        df_counts = (
+            self.df[column]
+            .value_counts()
+            .sort_values()
+            .rename_axis(title)
+            .reset_index(name="Count")
+        )
+
+        print(df_counts.to_markdown(index=False))  # clean table
+
+
+    def get_dataset_stats(self):
+        self._print_stats("dataset", "Dataset")
+
+
+    def get_category_stats(self):
+        self._print_stats("category", "Category")
+
+
+    def get_sub_category_stats(self):
+        self._print_stats("sub_category", "Sub Category")
+
+
+    def get_task_stats(self):
+        self._print_stats("task", "Task")
+
+
+
+
+def initialize_benchmark(local_path):
+    token = os.environ["HF_TOKEN"]
+    repo_id="Wahiddhrubo/hal-spatial"
+
+    
+    snapshot_download(
+        repo_id=repo_id,
+        repo_type="dataset",   # "dataset" or "model"
+        local_dir=local_path,  # your path
+        token=token,
+        allow_patterns=None,   # optional: download only some files
+        ignore_patterns=None,  # optional: ignore some files
+    )
+
+    
+    zip_path = Path(local_path) / "question_images.zip"
+    extract_to = Path(local_path) / "question_images"
+    
+    with ZipFile(zip_path, 'r') as z:
+        members = z.infolist()
+    
+        with alive_bar(len(members), title="Extracting files") as bar:
+            for m in members:
+                z.extract(m, extract_to)
+                bar()
+
+    zip_path.unlink()
+
+
+def prompt_generator(row):
+
+    # print(row)
+    
+    prompt = f"""You are a vision-language reasoning agent.
+Understand the intent of the question based on the category {row['category']}. Focus on key visual or logical cues such as shading, color consistency, spatial relations, or context depending on the category.
+
+Carefully read and compare all choices, paying attention to small differences and ignoring formatting issues. Use relative reasoning when needed (e.g., darker vs lighter, same vs different).
+
+Choose the {'Answer' if row['question_type'] != 'MCQ' else 'OPTION LETTER ONLY'} that best matches the question’s intent based on sound reasoning, not assumptions. 
+
+JUST GIVE THE ANSWER NO NEED OF REASONING.
+
+Question:
+{row['question']}
+"""
+
+    choices= f"""Choices: {row['choices']} """ if row["choices"] else ""
+
+    
+
+    return  {**row, "task_description": prompt + choices}
+
+
+
+
+
+def process_benchmark(
+        df,
+        categories,
+        sub_categories,
+        task_ids,
+        max_tasks,
+        continue_run=False,
+        eval_path="",  
+    ):
+    
+    if len(categories):
+        df=df[df["categories"].isin(categories)]
+    
+    if len(task_ids):
+        df=df[df["task"].isin(task_ids)]
+    
+    if len(sub_categories):
+        df=df[df["sub_categories"].isin(sub_categories)]
+        
+    ids=[]
+    if continue_run:
+        if os.path.exists(eval_path):
+            ids=pd.read_csv(eval_path)["id"].to_list()
+        
+    
+    
+    
+    
+    
+    df=df[~df["id"].isin(ids)]
+    dataset=df.set_index('id').to_dict(orient='index')
+    
+    if max_tasks and max_tasks > 0 and max_tasks < len(dataset):
+        print(f"Limiting to the first {max_tasks} tasks as requested")
+        keep_task_ids = list(dataset.keys())[: max_tasks]
+        dataset = {task_id: dataset[task_id] for task_id in keep_task_ids}
+
+    
+    return dataset
+

hal_spatial_harness-0.1.0/hal_spatial_harness_package/prompt_variation.py

@@ -0,0 +1,117 @@
+
+from typing import Dict,  List
+from utils import get_client
+
+
+
+
+
+
+
+# ---------------- Config ----------------
+
+# Instructions per strength
+STRENGTH_INSTRUCTIONS = {
+    "mild": (
+        "Generate small, surface-level variations of the prompt. "
+        "Use synonyms, minor changes in formality, active/passive voice changes. "
+        "Keep all information, meaning, and structure intact. "
+        "Output should be fluent and similar in style to the original."
+    ),
+    "medium": (
+        "Generate variations that restructure sentences and reorder information. "
+        "Combine or split sentences, slightly change phrasing, but preserve all meaning. "
+        "Output should still be clear and professional, resembling the original style."
+    ),
+    "strong": (
+        "Generate majorly different variations that sound like different people wrote them. "
+        "Use conversational phrasing, varied sentence structures, and style differences. "
+        "Preserve all information and constraints exactly. Output should maintain readability."
+    ),
+    "naturalistic": (
+        "Generate realistic user-style variations as if typed by a real person. "
+        "Include casual phrasing, abbreviations, typos, and informal chat patterns. "
+        "Keep all meaning and requirements intact. Output should still resemble the original prompt in content."
+    ),
+}
+
+STRENGTH_TEMPERATURE = {
+    "mild": 0.7,
+    "medium": 0.8,
+    "strong": 0.9,
+    "naturalistic": 0.9,
+}
+
+
+class PromptVariationGenerator:
+    def __init__(self, model_name="gpt-4o-mini-2024-07-18", num_variations=2, strength="medium"):
+        self.model_name = model_name
+        self.num_variations = num_variations
+        self.strength = strength.lower()
+        self.client = get_client()
+
+    def generate_variations_for_task(self, task_id: str, prompt: str) -> List[Dict]:
+        instruction = STRENGTH_INSTRUCTIONS.get(self.strength, STRENGTH_INSTRUCTIONS["mild"])
+        temperature = STRENGTH_TEMPERATURE.get(self.strength, 0.7)
+
+        system_prompt = f"You are an expert at paraphrasing prompts while preserving meaning.\nInstruction: {instruction}"
+        user_prompt = f"Original prompt:\n{prompt}\nGenerate {self.num_variations} variations, one per line. Do NOT number them."
+
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_prompt},
+                ],
+                temperature=temperature,
+                max_tokens=1000,
+            )
+
+            lines = response.choices[0].message.content.strip().split("\n")
+            variations = [line.strip() for line in lines if len(line.strip()) > 10]
+
+        except Exception as e:
+            print(f"Error generating variations for task {task_id}: {e}")
+            variations = []
+
+        all_variations = [prompt] + variations[:self.num_variations]
+
+        return [
+            {"question": v, "prompt_variation_id": i, "prompt_variation_strength": self.strength}
+            for i, v in enumerate(all_variations)
+        ]
+
+    def generate_variations_for_dataset(self, dataset: Dict[str, Dict], prompt_field: str = "question") -> Dict[str, List[Dict]]:
+        result = {}
+        for task_id, task_data in dataset.items():
+            if prompt_field not in task_data:
+                result[task_id] = [{
+                    "question": task_data.get(prompt_field, ""),
+                    "prompt_variation_id": 0,
+                    "prompt_variation_strength": self.strength
+                }]
+                continue
+            prompt = task_data[prompt_field]
+            result[task_id] = self.generate_variations_for_task(task_id, prompt)
+        return result
+
+
+
+def generate_var_bench( dataset, prompt_map):
+    new_dataset={}
+    for d_id,d in dataset.items():
+        
+        for idx,q in  enumerate(prompt_map[d_id]):
+            new_dataset[f'{d_id}-{str(idx).zfill(2)}'] = { 
+                **dataset[d_id],
+                "question":q['question']
+            } 
+
+    return new_dataset
+
+
+
+
+
+

hal_spatial_harness-0.1.0/hal_spatial_harness_package/utils.py

@@ -0,0 +1,61 @@
+import os
+from openai import OpenAI
+import base64
+
+import re
+import os
+
+class InvalidPathNameError(ValueError):
+    """Custom exception for invalid path names."""
+    pass
+
+def verify_run_name(name: str) -> str:
+    """
+    Verify that the given name is a valid filename/folder name.
+    
+    Rules:
+    - Cannot contain forbidden characters: \ / : * ? " < > |
+    - Cannot be empty or only whitespace
+    - Cannot be a reserved Windows name (CON, PRN, AUX, NUL, COM1..9, LPT1..9)
+    
+    Returns the stripped valid name if valid.
+    Raises InvalidPathNameError if invalid.
+    """
+    if not name or name.strip() == "":
+        raise InvalidPathNameError("Path name cannot be empty or whitespace.")
+
+    name = name.strip()
+    
+    # Forbidden characters
+    forbidden_chars = r'[<>:"/\\|?*]'
+    if re.search(forbidden_chars, name):
+        raise InvalidPathNameError(
+            f"Path name '{name}' contains forbidden characters: \\ / : * ? \" < > |"
+        )
+
+    # Reserved Windows names
+    reserved = {"CON","PRN","AUX","NUL"} | {f"COM{i}" for i in range(1,10)} | {f"LPT{i}" for i in range(1,10)}
+    if name.upper() in reserved:
+        raise InvalidPathNameError(f"Path name '{name}' is a reserved system name.")
+
+    return name
+
+
+def encode_image(path):
+    with open(path, "rb") as f:
+        return base64.b64encode(f.read()).decode("utf-8")
+
+
+def get_client():
+    OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+    OPEN_ROUTER_API_KEY = os.environ.get("OPEN_ROUTER_API_KEY")
+    if OPEN_ROUTER_API_KEY:
+        return OpenAI(
+            base_url="https://openrouter.ai/api/v1",
+            api_key=OPEN_ROUTER_API_KEY
+        )
+    elif OPENAI_API_KEY:
+        return OpenAI(api_key=OPENAI_API_KEY)
+    else:
+        raise ValueError("No API key found. Set OPENAI_API_KEY or OPEN_ROUTER_API_KEY.")
+        

hal_spatial_harness-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hal-spatial-harness"   # ⚠️ must be unique on PyPI
+version = "0.1.0"
+description = "A lightweight framework for benchmarking multimodal AI agents with parallel execution, prompt variation, and automated evaluation."
+readme = "README.md"
+requires-python = ">=3.8"
+authors = [
+  { name="Wahid Faisal", email="wahiddhrubo@gmail.com" }
+]
+license = { text = "MIT" }
+
+dependencies = [
+  "pandas",
+  "numpy",
+  "alive-progress",
+  "gitpython",
+  "huggingface-hub",
+  "openai"
+]

hal_spatial_harness-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+