ciol-astro-vqa 0.0.1__tar.gz

ciol_astro_vqa-0.0.1/LICENSE

@@ -0,0 +1,27 @@
+
+---
+
+# 📄 **LICENSE (MIT License)**
+
+```text id="license001"
+MIT License
+
+Copyright (c) 2026 CIOL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ciol_astro_vqa-0.0.1/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: ciol-astro-vqa
+Version: 0.0.1
+Summary: A lightweight framework for benchmarking multimodal AI agents with parallel execution, prompt variation, and automated evaluation.
+Author-email: Wahid Faisal <wahiddhrubo@gmail.com>
+License: MIT
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: alive-progress
+Requires-Dist: gitpython
+Requires-Dist: huggingface-hub
+Requires-Dist: openai
+Requires-Dist: scipy
+Requires-Dist: scikit-learn
+Requires-Dist: torch
+Requires-Dist: transformers
+Requires-Dist: umap-learn
+Requires-Dist: pillow
+Requires-Dist: cohere
+Requires-Dist: hdbscan
+Dynamic: license-file

ciol_astro_vqa-0.0.1/ciol_astro_vqa.egg-info/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: ciol-astro-vqa
+Version: 0.0.1
+Summary: A lightweight framework for benchmarking multimodal AI agents with parallel execution, prompt variation, and automated evaluation.
+Author-email: Wahid Faisal <wahiddhrubo@gmail.com>
+License: MIT
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: alive-progress
+Requires-Dist: gitpython
+Requires-Dist: huggingface-hub
+Requires-Dist: openai
+Requires-Dist: scipy
+Requires-Dist: scikit-learn
+Requires-Dist: torch
+Requires-Dist: transformers
+Requires-Dist: umap-learn
+Requires-Dist: pillow
+Requires-Dist: cohere
+Requires-Dist: hdbscan
+Dynamic: license-file

ciol_astro_vqa-0.0.1/ciol_astro_vqa.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+LICENSE
+pyproject.toml
+./ciol_astrovqa_package/__init__.py
+./ciol_astrovqa_package/evaluator.py
+./ciol_astrovqa_package/evaluator_structured_signal_extraction.py
+./ciol_astrovqa_package/holistic_harness.py
+./ciol_astrovqa_package/initialize_benchmark.py
+./ciol_astrovqa_package/prompt_variation.py
+./ciol_astrovqa_package/utils.py
+ciol_astro_vqa.egg-info/PKG-INFO
+ciol_astro_vqa.egg-info/SOURCES.txt
+ciol_astro_vqa.egg-info/dependency_links.txt
+ciol_astro_vqa.egg-info/requires.txt
+ciol_astro_vqa.egg-info/top_level.txt
+ciol_astrovqa_package/__init__.py
+ciol_astrovqa_package/evaluator.py
+ciol_astrovqa_package/evaluator_structured_signal_extraction.py
+ciol_astrovqa_package/holistic_harness.py
+ciol_astrovqa_package/initialize_benchmark.py
+ciol_astrovqa_package/prompt_variation.py
+ciol_astrovqa_package/utils.py

ciol_astro_vqa-0.0.1/ciol_astro_vqa.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

ciol_astro_vqa-0.0.1/ciol_astro_vqa.egg-info/requires.txt

@@ -0,0 +1,14 @@
+pandas
+numpy
+alive-progress
+gitpython
+huggingface-hub
+openai
+scipy
+scikit-learn
+torch
+transformers
+umap-learn
+pillow
+cohere
+hdbscan

ciol_astro_vqa-0.0.1/ciol_astro_vqa.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+ciol_astrovqa_package
+dist

ciol_astro_vqa-0.0.1/ciol_astrovqa_package/__init__.py

@@ -0,0 +1,21 @@
+from .evaluator import HarnessEvaluator,structured_signal_extraction,cross_model_diagnostic_synthesis
+from .holistic_harness import HarnessRunner
+from .initialize_benchmark import process_benchmark, BenchmarkStats
+from .prompt_variation import PromptVariationGenerator
+from .evaluator_error_taxonomy_induction import cluster_name_mapping
+
+
+
+__version__ = "0.1.5"
+
+__all__ = [
+    "HarnessEvaluator",
+    "cross_model_diagnostic_synthesis",
+    "structured_signal_extraction",
+    "HarnessRunner",
+    "process_benchmark",
+    "PromptVariationGenerator",
+    "BenchmarkStats",
+    "cluster_name_mapping"
+]
+

ciol_astro_vqa-0.0.1/ciol_astrovqa_package/evaluator.py

@@ -0,0 +1,56 @@
+# ================= STANDARD LIB =================
+import os
+from pathlib import Path
+
+# ================= DATA =================
+import pandas as pd
+
+
+
+from .evaluator_structured_signal_extraction import structured_signal_extraction
+
+
+
+
+
+def HarnessEvaluator(
+    results_path,
+    max_concurrent=10,
+    run_limit=None,
+    mcq_run=False,
+    model_name="gemma-4-31b-it",
+    ):
+
+    results_path=Path(results_path)
+
+    current_run=0
+
+    for model_folder in results_path.iterdir():
+        if model_folder.is_dir():
+            if run_limit is not None and current_run >= run_limit:
+                break
+
+            current_run += 1
+            
+            response_csv_path= model_folder / "agent_response.csv"
+            score_csv_path= model_folder / "agent_scores.csv"
+            prev_df=None
+            
+            if os.path.exists(score_csv_path) :
+                prev_df=pd.read_csv(score_csv_path)
+                
+            structured_signal_extraction(
+                response_csv_path,
+                mcq_run=mcq_run,
+                max_concurrent=max_concurrent,
+                model_name=model_name,
+                prev_df=prev_df
+            )
+            
+            
+            if score_csv_path.exists():
+                df=pd.read_csv(score_csv_path)
+                if "accuracy_score" in df.columns:
+                    print("Numbers of Failed Tasks: ",len(df[df["accuracy_score"].isna()]))
+
+    

ciol_astro_vqa-0.0.1/ciol_astrovqa_package/evaluator_structured_signal_extraction.py

@@ -0,0 +1,180 @@
+# ================= STANDARD LIB =================
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+# ================= DATA =================
+import pandas as pd
+
+# ================= UTILITIES =================
+from alive_progress import alive_bar
+
+from .utils import get_client,json_parser
+
+
+structured_signal_scoring_prompt_template = """
+You are an expert evaluation agent. Your job is to rigorously evaluate a model's response.
+
+Return ONLY a valid JSON object. Do NOT include any explanation, markdown, or extra text.
+
+================ INPUT ================
+
+Question: {question}
+
+Ground Truth Answer: {ground_truth}
+
+Ground Truth Reasoning: {reasoning}
+
+Model Response: {model_output}
+
+
+
+================ INSTRUCTIONS ================
+
+Follow these rules strictly:
+
+1. Accuracy Score (0/1):
+- 1 if the final answer is correct.
+- 0 otherwise.
+- For OE: use semantic equivalence.
+
+2. Resoning Score (1-10):
+- How well the model reasoning matches Ground Truth Reasoning or the answer.
+- The Better the Match The Higher The Score.
+
+3. Refusal Score (0/1):
+- 1 if the model refuses (e.g., "I cannot answer", "insufficient information").
+- 0 otherwise.
+
+4. Off Topic Score (0/1):
+- 1 if response is irrelevant.
+- 0 if relevant.
+
+5. Extract Reasoning:
+- Extract and summarize the reasoning for the Score.
+
+================ OUTPUT FORMAT ================
+
+Return STRICT JSON only (no trailing commas, no Python types, use null not None):
+
+
+{{
+  "accuracy_score": 0,
+  "refusal_score": 0,
+  "off_topic_score": 0,
+  "reasoning": "",
+}}
+
+
+"""
+
+
+
+
+
+def process_row_for_signal_extraction(row,model_name,error_logs):
+
+    max_retry=5
+    retry_count=0
+    while retry_count<max_retry:
+        client = get_client()
+        
+        try:
+            
+            correct_answer = row["answer"]
+            model_answer = row["response"]
+            question = row["task_description"]
+            reasoning = row["Explanation"]
+            
+
+
+            prompt = structured_signal_scoring_prompt_template.format(
+                question=question,
+                ground_truth=correct_answer,
+                model_output=model_answer,
+                reasoning=reasoning
+                
+            )
+
+            content = [{"type": "text", "text": prompt}]
+
+            response = client.chat.completions.create(
+                model=model_name,
+                messages=[{"role": "user", "content": content}],
+                temperature=0.1,
+                max_tokens=1000,
+            )
+            res = json_parser(response.choices[0].message.content) if response else {}
+            
+            return {**row, **res}
+    
+        except Exception as e:
+            
+            retry_count+=1
+            if retry_count==max_retry:
+                error_logs.append(f"Task {row["id"]}:{e}")
+                return row  # return original row if failure
+
+def structured_signal_extraction(csv_path,prev_df=None,model_name="google/gemini-3.1-flash-image-preview",max_concurrent=10):
+    save_path=(Path(csv_path.parent))
+    
+    error_logs=[]
+    
+
+    
+    df=pd.read_csv(csv_path)
+    ids=[]
+    
+    score_col="accuracy_score"
+    df=df[df["response"].notna()]
+    
+    if prev_df is not None:
+        if score_col in prev_df.columns:
+            prev_df=prev_df[(prev_df[score_col].notna()) & (prev_df["response"].notna())]
+            ids = prev_df["id"].tolist()
+            df=df[~df["id"].isin(ids)]
+            print(f"Found ({len(ids)}) Previous tasks | New Tasks Remaining ({len(df)})")
+            
+            
+            
+    ds= df.to_dict(orient="records")
+    model=str(save_path).split("/")[-1]
+
+    print(f"Scoring {model}")
+    results = []
+
+    with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
+        futures = [executor.submit(process_row_for_signal_extraction, row,model_name,error_logs ) for row in ds]
+    
+        with alive_bar(len(futures), bar='blocks', spinner='dots',
+                       title='Scoring Agent Response', force_tty=True) as bar:
+    
+            for future in as_completed(futures):
+                results.append(future.result())
+                pd.DataFrame(results).to_csv(save_path/"agent_scores.csv",index=False)
+                bar()
+                
+    if prev_df is not None:
+
+        new_scores = pd.DataFrame(results)
+
+        # ✅ Ensure same columns (important)
+        all_cols = list(set(new_scores.columns).union(set(prev_df.columns)))
+        new_scores = new_scores.reindex(columns=all_cols)
+        prev_df = prev_df.reindex(columns=all_cols)
+
+        # ✅ Concat
+        df = pd.concat([prev_df, new_scores], ignore_index=True)
+
+
+        df.to_csv(save_path / "agent_scores.csv", index=False)
+    else:
+
+        pd.DataFrame(results).to_csv(save_path/"agent_scores.csv",index=False)
+    
+    try:
+        if len(error_logs)>0:
+            f=open(save_path/f"Scoring_error{model}.txt","w")
+            f.write(str(error_logs))
+            f.close()
+    except Exception as E:
+        print("Error Saving Error Logs: ",E)

ciol_astro_vqa-0.0.1/ciol_astrovqa_package/holistic_harness.py

@@ -0,0 +1,321 @@
+# =========================
+# Standard library
+# =========================
+import os
+import logging
+import ast
+import shutil
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import time
+
+# =========================
+# 3rd-party packages
+# =========================
+import pandas as pd
+from alive_progress import alive_bar
+
+# =========================
+# Local modules
+# =========================
+from .initialize_benchmark import  process_benchmark
+from .prompt_variation import PromptVariationGenerator, generate_var_bench
+from .evaluator import HarnessEvaluator
+from .utils import encode_image, get_client, verify_run_name
+
+
+
+
+logger = logging.getLogger(__name__)
+
+
+
+
+
+def agent(inputs: dict[str, dict], model_name: str,error_log,**kwargs):    
+    
+    client = get_client()
+
+    task_id, task = list(inputs.items())[0]
+
+    images_paths = [p for p in  task["ques_image_path_lst"]]
+    response=None
+
+    max_retries = 1
+
+    if os.environ.get("COHERE_API_KEY", False):
+        for attempt in range(max_retries):
+            try:
+                # build content list
+                content = [{"type": "text", "text": task["task_description"]}]
+                
+                for path in images_paths[:4]:
+                    fmt, b64 = encode_image(path)
+
+                    content.append({
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/{fmt};base64,{b64}"
+                        }
+                    })
+
+                response = client.chat(
+                    model=model_name,
+                    messages=[
+                        {
+                            "role": "user",
+                            "content": content
+                        }
+                    ],
+                    temperature=0.1,
+                    max_tokens=600,
+                )
+                
+                content=response.message.content[0].text
+                
+                inputs[task_id]["response"] = (
+                    content.strip() 
+                )
+                break  # ✅ success → exit loop
+
+            except Exception as e:
+                
+
+
+                if attempt < max_retries - 1:
+                    sleep_time = 2 ** attempt  # exponential backoff: 1s, 2s, 4s
+                    time.sleep(sleep_time)
+                else:
+                    error_log.append(f"❌ Failed at {max_retries} task {task_id}:{e}")
+                    inputs[task_id]["response"] = None
+    
+    
+    
+    else:
+        for attempt in range(max_retries):
+            try:
+                # build content list
+                content = [{"type": "text", "text": task["task_description"]}]
+                if "llama-3.2-11b-vision-instruct" in model_name :
+                    path_len = 1 
+                elif "qwen3.5-9b" in model_name:
+                    path_len = 5 
+                else:
+                    path_len=len(images_paths)
+                
+                for path in images_paths[:path_len]:
+                    fmt, b64 = encode_image(path)
+
+                    content.append({
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/{fmt};base64,{b64}"
+                        }
+                    })
+
+                response = client.chat.completions.create(
+                    model=model_name,
+                    messages=[
+                        {
+                            "role": "user",
+                            "content": content
+                        }
+                    ],
+                    temperature=0.1,
+                    max_tokens=1000,
+                    reasoning_effort="low"
+                )
+                if not (response.choices):
+                    print(response)
+                    
+                content=response.choices[0].message.content
+                reasoning=response.choices[0].message.reasoning
+
+                if not (content or reasoning):
+                    print(response)
+                
+                inputs[task_id]["response"] = (
+                    content.strip() if content else reasoning
+                )
+                break  # ✅ success → exit loop
+
+            except Exception as e:
+                
+
+
+                if attempt < max_retries - 1:
+                    sleep_time = 2 ** attempt  # exponential backoff: 1s, 2s, 4s
+                    time.sleep(sleep_time)
+                else:
+                    error_log.append(f"❌ Failed at {max_retries} task {task_id}:{e}")
+                    inputs[task_id]["response"] = None
+    
+        
+
+    return inputs
+
+
+
+def run_agent_function(inputs: dict[str, dict], agent_function ,error_log, use_template: bool =False, model_name: str = "" , **kwargs):
+    if use_template:
+        res=agent_function(inputs,model_name=model_name,error_log=error_log)
+    else:
+        res=agent_function(inputs,)
+
+    return res
+                
+    
+
+
+def HarnessRunner(
+        use_template=True,
+        custom_function=None,
+        template_name="default_agent",
+        run_mcq=False,
+        dataset_dir=None,
+        clean_dataset_dir=True,
+        template_model="gpt-4o-mini-2024-07-18",
+        run_id = "test",
+        max_concurrent = 5,
+        continue_run = False,
+        max_tasks = 0,
+        prompt_sensitivity = False,
+        num_variations = 3,
+        variation_strength = "mild",
+        results_dir = None,
+        task_ids = [],
+        categories = [],
+        sub_categories = [],
+        openai_api_key = None,
+        open_router_api_key = None,
+        cohere_api_key = None,
+        evaluation_needed=False,
+        custom_error_taxonomies=None
+    ):
+    
+    failed_ids=[]
+    
+    error_log=[]
+    
+    results_dir = results_dir or ("mcq_results" if run_mcq else "results")
+    
+    function_template = {
+        "default_agent": agent,
+    }
+
+    if openai_api_key:
+        os.environ["OPENAI_API_KEY"]=openai_api_key
+        
+    if open_router_api_key:
+        os.environ["OPEN_ROUTER_API_KEY"]=open_router_api_key
+    if cohere_api_key:
+        os.environ["COHERE_API_KEY"]=cohere_api_key
+
+    
+        
+    if (not use_template) and (not custom_function):
+        raise ValueError(
+            "Select use_template or provide custom_function "
+        )
+        
+    
+    if use_template:
+        agent_function= function_template[template_name]
+    else:
+        agent_function= custom_function
+
+    temp_dir="./temp_dir"
+    verify_run_name(run_id)
+    Path(f"{results_dir}/{run_id}").mkdir(parents=True, exist_ok=True)
+    response_path=f"{results_dir}/{run_id}/agent_response.csv"
+    
+    
+    if os.path.exists(response_path) and (not continue_run):
+        raise ValueError("Run Already Exist. Pass  continue_run if you want to Continue Run.")
+
+    if dataset_dir:
+        temp_dir=dataset_dir
+        
+    
+    df = process_benchmark(
+        temp_dir,
+        categories,
+        sub_categories,
+        task_ids,
+        max_tasks,
+        continue_run=continue_run,
+        eval_path=response_path,
+        mcq_prompt=run_mcq
+    )
+    
+    dataset=df.set_index('id').to_dict(orient='index')
+
+
+    eval_results = {}
+
+    if continue_run and os.path.exists(response_path):
+        eval_results=pd.read_csv(response_path).set_index('id').to_dict(orient='index')
+    
+    var_dataset=None
+
+
+    if prompt_sensitivity:
+        
+        
+        generator = PromptVariationGenerator(num_variations=num_variations, strength=variation_strength)
+        prompt_map = generator.generate_variations_for_dataset(dataset)
+        var_dataset=generate_var_bench(dataset, prompt_map)
+
+    active_dataset = var_dataset if var_dataset else dataset
+    
+    def process_item(key, item):
+        """Generate prompt and run agent for a single item"""
+        
+        res = run_agent_function(
+            inputs={key: item},
+            agent_function=agent_function,
+            use_template=use_template,
+            model_name=template_model,
+            error_log=error_log,
+        )
+        
+        if (not res[key].get("response", False)):
+            failed_ids.append(key)
+            
+        return key, res[key]
+
+
+    # Run in parallel with progress bar
+    with alive_bar(len(active_dataset), bar='blocks', spinner='dots', title=f'Running {run_id}', force_tty=True) as bar:
+        with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
+            futures = {executor.submit(process_item, k, v): k for k, v in active_dataset.items()}
+
+            for future in as_completed(futures):
+                key, result = future.result()
+                eval_results[key] = result
+                pd.DataFrame.from_dict(eval_results, orient="index").rename_axis("id").reset_index().to_csv(response_path, index=False)
+                bar()  # update progress
+
+    if len(failed_ids) > 0:
+        print(f"Failed {len(failed_ids)} tasks: {failed_ids}")
+            
+        with open(f"{results_dir}/{run_id}/error_log.txt", "w") as f:
+            for e in error_log:
+                f.write(f"{repr(e)}\n")
+        
+    if evaluation_needed:
+        # Run evaluator on results
+        HarnessEvaluator(
+           custom_error_taxonomies= custom_error_taxonomies,
+           max_concurrent=max_concurrent,
+           model_name=template_model,
+           results_path=results_dir,
+           
+        )
+        
+    if clean_dataset_dir:
+        shutil.rmtree(temp_dir)
+
+
+
+

ciol_astro_vqa-0.0.1/ciol_astrovqa_package/initialize_benchmark.py

@@ -0,0 +1,189 @@
+import os
+import ast
+from pathlib import Path
+
+# 3rd-party packages
+import pandas as pd
+from alive_progress import alive_bar
+from huggingface_hub import snapshot_download
+from zipfile import ZipFile
+from importlib.resources import files
+import re
+
+
+
+def clean_options(text):
+    if pd.isna(text):
+        return ""
+    
+    text = str(text)  # ensure string
+    text = re.sub(r'\b[A-Za-z]\s*[\.\)\:-]\s*', ' | ', text)
+    return re.sub(r'^\s*\|\s*', '', text).strip()
+
+class BenchmarkStats:
+    def __init__(self, ):
+        csv_path = files("ciol_astrovqa_package") / "annotations.csv"
+        self.df = pd.read_csv(csv_path)   
+    
+    def _print_stats(self, column, title):
+        pd.set_option("display.max_rows", None)
+
+        df_counts = (
+            self.df[column]
+            .value_counts()
+            .sort_values()
+            .rename_axis(title)
+            .reset_index(name="Count")
+        )
+
+        print(df_counts.to_markdown(index=False))  # clean table
+
+
+
+
+    def get_category_stats(self):
+        self._print_stats("Category", "Category")
+
+
+    def get_sub_category_stats(self):
+        self._print_stats("Sub-Category", "Sub Category")
+
+
+    def get_difficulty_stats(self):
+        self._print_stats("Difficulty", "Difficulty")
+
+
+
+
+def initialize_benchmark(local_path):
+    token = os.environ.get("HF_TOKEN",None)
+    repo_id="ciol-research/AstroVQA"
+    
+    if not token:
+        raise ValueError("Dataset is no Public Yet!!!")
+
+    
+    snapshot_download(
+        repo_id=repo_id,
+        repo_type="dataset",   # "dataset" or "model"
+        local_dir=local_path,  #  path
+        token=token,
+        allow_patterns=None,   #download only some files
+        ignore_patterns=None,  #ignore some files
+    )
+
+    
+    zip_path = Path(local_path) / "question_images.zip"
+    extract_to = Path(local_path) / "question_images"
+    
+    with ZipFile(zip_path, 'r') as z:
+        members = z.infolist()
+    
+        with alive_bar(len(members), title="Extracting files") as bar:
+            for m in members:
+                z.extract(m, extract_to)
+                bar()
+
+    zip_path.unlink()
+
+
+def prompt_generator(row):
+    
+    prompt = f"""Answer the question using clear reasoning.
+    
+    Question:
+    {row['question']}
+    """
+    
+    prompt += "\n\nGive only the final answer and Add a short Reasoning Behind Your Answer."
+    
+    return   prompt 
+
+def mcq_prompt_generator(row):
+    
+    prompt = f"""Answer the question using clear reasoning.
+    
+    Question:
+    {row['question']}
+    """
+    
+    choices=f"{row['Option 1']}, {row['Option 2']}, {row['Option 3']}, {row['Option 4']}"
+    prompt += f"\nContext (Some possible answers): {choices}"
+
+    prompt += "\n\nGive only the final answer in one sentence."
+    
+    return   prompt 
+
+
+
+
+
+def process_benchmark(
+        temp_dir,
+        categories=[],
+        sub_categories=[],
+        task_ids=[],
+        max_tasks=0,
+        continue_run=False,
+        eval_path="",  
+        mcq_prompt=False,
+    ):
+    
+    if not os.path.exists(temp_dir) :
+        initialize_benchmark(temp_dir)
+    
+    
+    
+    df=pd.read_csv(f"{temp_dir}/annotations.csv").drop_duplicates(subset=["id"])
+    
+    df["ques_image_path_lst"] = df["ques_image_path_lst"].apply(ast.literal_eval)
+
+    df["ques_image_path_lst"] = df["ques_image_path_lst"].apply(
+        lambda paths: [os.path.join(temp_dir, "question_images", p) for p in paths]
+    )
+    
+    
+    
+    if len(categories):
+        df=df[df["category"].isin(categories)]
+    
+    if len(task_ids):
+        df=df[df["task"].isin(task_ids)]
+    
+    if len(sub_categories):
+        df=df[df["sub_category"].isin(sub_categories)]
+        
+    ids = []
+
+    if continue_run:
+        if os.path.exists(eval_path):
+            prev_df = pd.read_csv(eval_path)
+
+            # keep only valid completed rows
+            if "response" in prev_df.columns:
+                valid_prev = prev_df[prev_df["response"].notna()]
+
+                ids = valid_prev["id"].tolist()
+            else:
+                ids = []
+                
+            # filter current df
+            df = df[~df["id"].isin(ids)].copy()
+
+            print(f"Found ({len(ids)}) Previous tasks | New Tasks Remaining ({len(df)})")
+    
+    
+    
+    if max_tasks and max_tasks > 0 and max_tasks < len(df):
+        print(f"Limiting to the first {max_tasks} tasks as requested")
+        df=df[:max_tasks]
+        
+    
+    df["task_description"]=(
+        df.apply(lambda x: mcq_prompt_generator(x),axis=1) 
+        if mcq_prompt 
+        else df.apply(lambda x: prompt_generator(x),axis=1) 
+    )
+    
+    return df
+

ciol_astro_vqa-0.0.1/ciol_astrovqa_package/prompt_variation.py

@@ -0,0 +1,117 @@
+
+from typing import Dict,  List
+from .utils import get_client
+
+
+
+
+
+
+
+# ---------------- Config ----------------
+
+# Instructions per strength
+STRENGTH_INSTRUCTIONS = {
+    "mild": (
+        "Generate small, surface-level variations of the prompt. "
+        "Use synonyms, minor changes in formality, active/passive voice changes. "
+        "Keep all information, meaning, and structure intact. "
+        "Output should be fluent and similar in style to the original."
+    ),
+    "medium": (
+        "Generate variations that restructure sentences and reorder information. "
+        "Combine or split sentences, slightly change phrasing, but preserve all meaning. "
+        "Output should still be clear and professional, resembling the original style."
+    ),
+    "strong": (
+        "Generate majorly different variations that sound like different people wrote them. "
+        "Use conversational phrasing, varied sentence structures, and style differences. "
+        "Preserve all information and constraints exactly. Output should maintain readability."
+    ),
+    "naturalistic": (
+        "Generate realistic user-style variations as if typed by a real person. "
+        "Include casual phrasing, abbreviations, typos, and informal chat patterns. "
+        "Keep all meaning and requirements intact. Output should still resemble the original prompt in content."
+    ),
+}
+
+STRENGTH_TEMPERATURE = {
+    "mild": 0.7,
+    "medium": 0.8,
+    "strong": 0.9,
+    "naturalistic": 0.9,
+}
+
+
+class PromptVariationGenerator:
+    def __init__(self, model_name="gpt-4o-mini-2024-07-18", num_variations=2, strength="medium"):
+        self.model_name = model_name
+        self.num_variations = num_variations
+        self.strength = strength.lower()
+        self.client = get_client()
+
+    def generate_variations_for_task(self, task_id: str, prompt: str) -> List[Dict]:
+        instruction = STRENGTH_INSTRUCTIONS.get(self.strength, STRENGTH_INSTRUCTIONS["mild"])
+        temperature = STRENGTH_TEMPERATURE.get(self.strength, 0.7)
+
+        system_prompt = f"You are an expert at paraphrasing prompts while preserving meaning.\nInstruction: {instruction}"
+        user_prompt = f"Original prompt:\n{prompt}\nGenerate {self.num_variations} variations, one per line. Do NOT number them."
+
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_prompt},
+                ],
+                temperature=temperature,
+                max_tokens=1000,
+            )
+
+            lines = response.choices[0].message.content.strip().split("\n")
+            variations = [line.strip() for line in lines if len(line.strip()) > 10]
+
+        except Exception as e:
+            print(f"Error generating variations for task {task_id}: {e}")
+            variations = []
+
+        all_variations = [prompt] + variations[:self.num_variations]
+
+        return [
+            {"question": v, "prompt_variation_id": i, "prompt_variation_strength": self.strength}
+            for i, v in enumerate(all_variations)
+        ]
+
+    def generate_variations_for_dataset(self, dataset: Dict[str, Dict], prompt_field: str = "question") -> Dict[str, List[Dict]]:
+        result = {}
+        for task_id, task_data in dataset.items():
+            if prompt_field not in task_data:
+                result[task_id] = [{
+                    "question": task_data.get(prompt_field, ""),
+                    "prompt_variation_id": 0,
+                    "prompt_variation_strength": self.strength
+                }]
+                continue
+            prompt = task_data[prompt_field]
+            result[task_id] = self.generate_variations_for_task(task_id, prompt)
+        return result
+
+
+
+def generate_var_bench( dataset, prompt_map):
+    new_dataset={}
+    for d_id,d in dataset.items():
+        
+        for idx,q in  enumerate(prompt_map[d_id]):
+            new_dataset[f'{d_id}-{str(idx).zfill(2)}'] = { 
+                **dataset[d_id],
+                "question":q['question']
+            } 
+
+    return new_dataset
+
+
+
+
+
+

ciol_astro_vqa-0.0.1/ciol_astrovqa_package/utils.py

@@ -0,0 +1,135 @@
+from PIL import Image
+import os
+from openai import OpenAI
+import base64
+
+import re
+import os
+
+import json
+
+
+
+import re
+import json
+
+def json_parser(json_text):
+    try:
+        # 1. Extract JSON block
+        match = re.search(r"\{.*\}", json_text, re.S)
+        if not match:
+            raise ValueError("No JSON object found")
+
+        text = match.group()
+
+        # 2. Clean problematic patterns
+
+        # Remove LaTeX blocks: $...$
+        text = re.sub(r"\$.*?\$", "", text)
+
+        # Remove backslashes (e.g., \rightarrow)
+        text = re.sub(r"\\+", "", text)
+
+        # Remove non-ASCII chars (optional but useful)
+        text = re.sub(r"[^\x00-\x7F]+", "", text)
+
+        # Fix trailing commas
+        text = re.sub(r",\s*}", "}", text)
+        text = re.sub(r",\s*]", "]", text)
+
+        # 3. Try parsing
+        return json.loads(text)
+
+    except Exception as e:
+        print(f"❌ JSON Parse Error: {e}")
+        print("🔎 Raw text:", json_text)
+
+        # 4. Fallback: return empty dict instead of crashing
+        return {}
+
+
+class InvalidPathNameError(ValueError):
+    """Custom exception for invalid path names."""
+    pass
+
+def verify_run_name(name: str) -> str:
+    """
+    Verify that the given name is a valid filename/folder name.
+    
+    Rules:
+    - Cannot contain forbidden characters: \ / : * ? " < > |
+    - Cannot be empty or only whitespace
+    - Cannot be a reserved Windows name (CON, PRN, AUX, NUL, COM1..9, LPT1..9)
+    
+    Returns the stripped valid name if valid.
+    Raises InvalidPathNameError if invalid.
+    """
+    if not name or name.strip() == "":
+        raise InvalidPathNameError("Path name cannot be empty or whitespace.")
+
+    name = name.strip()
+    
+    # Forbidden characters
+    forbidden_chars = r'[<>:"/\\|?*]'
+    if re.search(forbidden_chars, name):
+        raise InvalidPathNameError(
+            f"Path name '{name}' contains forbidden characters: \\ / : * ? \" < > |"
+        )
+
+    # Reserved Windows names
+    reserved = {"CON","PRN","AUX","NUL"} | {f"COM{i}" for i in range(1,10)} | {f"LPT{i}" for i in range(1,10)}
+    if name.upper() in reserved:
+        raise InvalidPathNameError(f"Path name '{name}' is a reserved system name.")
+
+    return name
+
+import io, base64
+
+def encode_image(path):
+    img = Image.open(path)
+    buffer = io.BytesIO()
+
+    fmt = img.format if img.format else "PNG"
+    img.save(buffer, format=fmt)
+
+    return fmt.lower(), base64.b64encode(buffer.getvalue()).decode()
+
+
+import os
+from openai import OpenAI
+import cohere
+
+
+def get_client(provider="auto"):
+    OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+    OPEN_ROUTER_API_KEY = os.environ.get("OPEN_ROUTER_API_KEY")
+    COHERE_API_KEY = os.environ.get("COHERE_API_KEY")
+
+    # Auto selection priority
+    if provider == "auto":
+        if OPEN_ROUTER_API_KEY:
+            provider = "openrouter"
+        elif OPENAI_API_KEY:
+            provider = "openai"
+        elif COHERE_API_KEY:
+            provider = "cohere"
+        else:
+            raise ValueError("No API key found.")
+
+    # OpenRouter (supports Cohere models via routing)
+    if provider == "openrouter":
+        return OpenAI(
+            base_url="https://openrouter.ai/api/v1",
+            api_key=OPEN_ROUTER_API_KEY
+        )
+
+    # OpenAI
+    elif provider == "openai":
+        return OpenAI(api_key=OPENAI_API_KEY)
+
+    # Cohere (native)
+    elif provider == "cohere":
+        return cohere.ClientV2(COHERE_API_KEY)
+
+    else:
+        raise ValueError(f"Unknown provider: {provider}")

ciol_astro_vqa-0.0.1/pyproject.toml

@@ -0,0 +1,41 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ciol-astro-vqa"
+version = "0.0.1"
+description = "A lightweight framework for benchmarking multimodal AI agents with parallel execution, prompt variation, and automated evaluation."
+readme = "README.md"
+requires-python = ">=3.8"
+authors = [
+  { name = "Wahid Faisal", email = "wahiddhrubo@gmail.com" }
+]
+license = { text = "MIT" }
+
+dependencies = [
+  "pandas",
+  "numpy",
+  "alive-progress",
+  "gitpython",
+  "huggingface-hub",
+  "openai",
+  "scipy",
+  "scikit-learn",
+  "torch",
+  "transformers",
+  "umap-learn",
+  "pillow",
+  "cohere",
+  "hdbscan"
+]
+
+[tool.setuptools]
+package-dir = {"" = "."}
+include-package-data = true
+
+[tool.setuptools.package-data]
+holistic_math_harness = ["*.csv"]
+
+[tool.setuptools.packages.find]
+where = ["."]

ciol_astro_vqa-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+