adaptivetesting 1.1.0__tar.gz → 1.1.1__tar.gz

{adaptivetesting-1.1.0 → adaptivetesting-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adaptivetesting
-Version: 1.1.0
+Version: 1.1.1
 Summary: adaptivetesting is a Python package that can be used to simulate and evaluate custom CAT scenarios as well as implement them in real-world testing scenarios from a single codebase
 Project-URL: Homepage, https://github.com/condecon/adaptivetesting
 Project-URL: Documentation, https://github.com/condecon/adaptivetesting/tree/main/docs

{adaptivetesting-1.1.0 → adaptivetesting-1.1.1}/adaptivetesting/math/__gen_response_pattern.py

@@ -17,7 +17,12 @@ def generate_response_pattern(ability: float,
     Returns:
         list[int]: response pattern
     """
-    # check if items is list of items or ItemPool
+    # Set seed once at the beginning if provided
+    if seed is not None:
+        rng = np.random.RandomState(seed)
+    else:
+        rng = np.random.RandomState()
+
     responses: list[int] = []
 
     for item in items:
@@ -27,10 +32,19 @@ def generate_response_pattern(ability: float,
                                                 c=np.array(item.c),
                                                 d=np.array(item.d))
         
-        # simulate response based on probability of scucess
-        if seed is not None:
-            np.random.seed(seed)
-        response = int(np.random.binomial(n=1, p=probability_of_success)) # ensure that the result is not an array
+        # Handle numpy scalar/array return properly
+        if hasattr(probability_of_success, 'item'):
+            prob_scalar = probability_of_success.item()
+        else:
+            prob_scalar = float(probability_of_success)
+        
+        # Validate probability bounds
+        if not (0 <= prob_scalar <= 1):
+            raise ValueError(f"Invalid probability: {prob_scalar}. Must be between 0 and 1.")
+        
+        # simulate response based on probability of success
+        random_val = rng.random_sample()
+        response = 1 if random_val < prob_scalar else 0
         responses.append(response)
 
     return responses

adaptivetesting-1.1.1/adaptivetesting/tests/test_generate_response_pattern.py

@@ -0,0 +1,106 @@
+# flake8: noqa
+import unittest
+import math
+from adaptivetesting.models import ItemPool
+from adaptivetesting.math.estimators import MLEstimator
+from adaptivetesting.math import generate_response_pattern
+
+
+source_dictionary = {"a": [1.0507,
+                           0.9943,
+                           0.9914,
+                           1.2737,
+                           0.9548,
+                           1.3033,
+                           0.6902,
+                           1.1169,
+                           1.0248,
+                           1.0432,
+                           1.0759,
+                           0.8995,
+                           0.9334,
+                           0.7963,
+                           0.7856,
+                           1.0607,
+                           1.0896,
+                           1.0106,
+                           1.1845,
+                           1.41,
+                           0.9018,
+                           0.5382,
+                           1.2011,
+                           0.8582,
+                           0.8624, 1.2051, 0.943, 0.7559, 1.0363, 0.9722, 1.0012, 1.0771, 0.9259, 1.1289, 0.9559, 1.0664, 1.2194, 1.087, 0.9348, 1.2298, 1.1987, 1.1097, 1.0477, 0.8744, 1.2721, 0.8799, 1.4375, 1.3065, 0.9529, 0.7947],
+                     "b":
+                     [-0.5605, -0.2302, 1.5587, 0.0705, 0.1293, 1.7151, 0.4609, -1.2651, -0.6869, -0.4457, 1.2241, 0.3598, 0.4008, 0.1107, -0.5558, 1.7869, 0.4979, -1.9666, 0.7014, -0.4728, -1.0678, -0.218, -1.026, -0.7289, -0.625, -1.6867, 0.8378, 0.1534, -1.1381, 1.2538, 0.4265, -0.2951, 0.8951, 0.8781, 0.8216, 0.6886, 0.5539, -0.0619, -0.306, -0.3805, -0.6947, -0.2079, -1.2654, 2.169, 1.208, -1.1231, -
+                      0.4029, -0.4667, 0.78, -0.0834], "c": [0.0597, 0.2406, 0.1503, 0.1288, 0.1006, 0.2201, 0.091, 0.0721, 0.0427, 0.043, 0.1205, 0.0632, 0.0541, 0.1686, 0.0119, 0.1752, 0.088, 0.1022, 0.2052, 0.2297, 0.0706, 0.2403, 0.1821, 0.1716, 0.0132, 0.0988, 0.1195, 0.1401, 0.1746, 0.2289, 0.1546, 0.1071, 0.1355, 0.0146, 0.0652, 0.0993, 0.0494, 0.208, 0.0382, 0.2009, 0.1367, 0.1656, 0.0429, 0.1583, 0.078, 0.1811, 0.0997, 0.2423, 0.2418, 0.1817], "d": [0.8143, 0.8054, 0.8983, 0.8169, 0.8828, 0.9463, 0.792, 0.8511, 0.8679, 0.967, 0.9814, 0.9705, 0.9185, 0.9875, 0.8791, 0.8941, 0.8341, 0.8368, 0.755, 0.8757, 0.9678, 0.7516, 0.768, 0.7911, 0.9426, 0.9338, 0.993, 0.8666, 0.7686, 0.9122, 0.9396, 0.7843, 0.8491, 0.8062, 0.7645, 0.849, 0.7662, 0.8065, 0.7637, 0.9176, 0.8244, 0.7752, 0.768, 0.9701, 0.9386, 0.9542, 0.9955, 0.7759, 0.7748, 0.9497]}
+
+
+class TestGenerateResponsePattern(unittest.TestCase):
+    def test_compare_generation_to_estimation(self):
+        item_pool = ItemPool.load_from_dict(source_dictionary)
+
+        # results
+        results = []
+
+        for i in range(100):
+            # generate pattern
+            responses = generate_response_pattern(0, item_pool.test_items, seed=i)
+        
+            # estimate
+            estimator = MLEstimator(responses, item_pool.test_items)
+            estimation = estimator.get_estimation()
+            results.append(estimation)
+
+        mean = sum(results) / len(results)
+        print(mean)
+        self.assertAlmostEqual(mean, 0, delta=0.3)
+
+    def test_debug_probabilities(self):
+        from adaptivetesting.math.estimators.__functions.__estimators import probability_y1
+        import numpy as np
+        
+        item_pool = ItemPool.load_from_dict(source_dictionary)
+        
+        print("First 5 items - Expected probabilities for ability=0:")
+        for i, item in enumerate(item_pool.test_items[:5]):
+            prob = probability_y1(mu=np.array(0.0),
+                                a=np.array(item.a),
+                                b=np.array(item.b), 
+                                c=np.array(item.c),
+                                d=np.array(item.d))
+            print(f"Item {i}: a={item.a:.3f}, b={item.b:.3f}, c={item.c:.3f}, d={item.d:.3f} -> P={float(prob):.3f}")
+
+    def test_calculate_expected_vs_actual(self):
+        from adaptivetesting.math.estimators.__functions.__estimators import probability_y1
+        import numpy as np
+            
+        item_pool = ItemPool.load_from_dict(source_dictionary)
+            
+        # Calculate expected probabilities for all items
+        expected_probs = []
+        for item in item_pool.test_items:
+            prob = probability_y1(mu=np.array(0.0),
+                                     a=np.array(item.a),
+                                     b=np.array(item.b), 
+                                     c=np.array(item.c),
+                                     d=np.array(item.d))
+            expected_probs.append(float(prob))
+            
+        expected_total = sum(expected_probs)
+        expected_percentage = expected_total / len(expected_probs) * 100
+            
+        percentage_results = []
+        for i in range(100):    
+            # Generate actual responses
+            responses = generate_response_pattern(0, item_pool.test_items, seed=i)
+            actual_correct = sum(responses)
+            actual_percentage = actual_correct / len(responses) * 100
+            percentage_results.append(actual_percentage)
+                
+        print(f"Expected percentage correct: {expected_percentage:.1f}%")
+        print(f"Actual percentage correct: {actual_percentage:.1f}%")
+            
+        # The difference should be within reasonable bounds for random sampling
+        # With 50 items, we expect some variation
+        self.assertAlmostEqual(actual_percentage, expected_percentage, delta=3)           

adaptivetesting-1.1.1/examples/.gitignore

@@ -0,0 +1,3 @@
+item_pool.csv
+data/
+*.pdf

adaptivetesting-1.1.1/examples/simulation.py

@@ -0,0 +1,134 @@
+# flake8: noqa
+import pandas as pd
+previc_item_pool = pd.read_csv("item_pool.csv")
+previc_item_pool.head()
+
+from adaptivetesting.models import ItemPool
+item_pool = ItemPool.load_from_list(
+    b=previc_item_pool["Difficulty"]
+)
+
+from scipy.stats import norm
+from numpy.random import seed
+seed(1234)
+theta_samples = norm.rvs(loc=0, scale=2, size=100)
+
+
+from adaptivetesting.implementations import TestAssembler
+from adaptivetesting.simulation import SimulationPool
+from adaptivetesting.models import ResultOutputFormat, StoppingCriterion
+from adaptivetesting.math.item_selection import maximum_information_criterion
+from adaptivetesting.math.estimators import BayesModal, NormalPrior
+
+tests = [
+    TestAssembler(
+        item_pool=item_pool,
+        simulation_id="example_bm",
+        participant_id=str(index),
+        ability_estimator=BayesModal,
+        estimator_args={
+            "prior": NormalPrior(0,1),
+            "optimization_interval":(-10, 10)
+        },
+        item_selector=maximum_information_criterion,
+        true_ability_level=theta,
+        simulation=True,
+        seed=index
+    )
+    for index, theta in enumerate(theta_samples)
+]
+
+sim_pool = SimulationPool(
+    adaptive_tests=tests,
+    test_result_output=ResultOutputFormat.CSV,
+    criterion=StoppingCriterion.SE,
+    value=0.4
+)
+sim_pool.start()
+
+import os
+
+dfs = []
+for i in range(100):
+    filename = f"data/example_bm/{i}.csv"
+    if os.path.exists(filename):
+        try:
+            df = pd.read_csv(filename)
+            # Get the last row of every file
+            last_row = df.tail(1)
+            dfs.append(last_row)
+        # If there is an error, skip the file
+        except Exception:
+            pass
+results_df = pd.concat(dfs, ignore_index=True)
+mse = ((results_df["ability_estimation"] - results_df["true_ability_level"]) ** 2).mean()
+print(f"MSE of ability estimates: {mse:.4f}")
+
+
+import matplotlib.pyplot as plt
+
+fig, ax = plt.subplots()
+ax.scatter(results_df["true_ability_level"],
+           results_df["ability_estimation"],
+           label="BM")
+ax.plot(results_df["ability_estimation"],
+        results_df["ability_estimation"],
+        color="black")
+ax.set_xlabel("True ability level")
+ax.set_ylabel("Estimated ability level")
+fig.savefig("sim_results_bm.pdf")
+
+# other example
+from adaptivetesting.math.estimators import ExpectedAPosteriori, CustomPrior
+from scipy.stats.distributions import t
+
+tests = [
+    TestAssembler(
+        item_pool=item_pool,
+        simulation_id="example_eap",
+        participant_id=str(index),
+        ability_estimator=ExpectedAPosteriori,
+        estimator_args={
+            "prior": CustomPrior(t, 3),
+            "optimization_interval":(-10, 10)
+        },
+        item_selector=maximum_information_criterion,
+        true_ability_level=theta,
+        simulation=True,
+        seed=index
+    )
+    for index, theta in enumerate(theta_samples)
+]
+
+sim_pool = SimulationPool(
+    adaptive_tests=tests,
+    test_result_output=ResultOutputFormat.CSV,
+    criterion=StoppingCriterion.SE,
+    value=0.4
+)
+sim_pool.start()
+
+dfs = []
+for i in range(100):
+    filename = f"data/example_eap/{i}.csv"
+    if os.path.exists(filename):
+        try:
+            df = pd.read_csv(filename)
+            # Get the last row of every file
+            last_row = df.tail(1)
+            dfs.append(last_row)
+        # If there is an error, skip the file
+        except Exception:
+            pass
+results_df = pd.concat(dfs, ignore_index=True)
+mse = ((results_df["ability_estimation"] - results_df["true_ability_level"]) ** 2).mean()
+print(f"MSE of ability estimates: {mse:.4f}")
+
+
+ax.scatter(results_df["true_ability_level"],
+           results_df["ability_estimation"],
+           label="EAP")
+ax.set_xlabel("True ability level")
+ax.set_ylabel("Estimated ability level")
+ax.legend()
+fig.savefig("sim_results_eap.pdf")

adaptivetesting-1.1.0/examples/real_world.py → adaptivetesting-1.1.1/examples/testing.py

@@ -1,31 +1,33 @@
+# flake8: noqa
+# setup item pool
+import pandas as pd
+previc_item_pool = pd.read_csv("item_pool.csv")
+previc_item_pool.head()
+
+from adaptivetesting.models import ItemPool
+item_pool = ItemPool.load_from_list(
+    b=previc_item_pool["Difficulty"]
+)
+
+# import psychopy
 from psychopy import visual, event
 from psychopy.hardware import keyboard
 from adaptivetesting.implementations import TestAssembler
 from adaptivetesting.models import AdaptiveTest, ItemPool, TestItem
 from adaptivetesting.data import CSVContext
-from adaptivetesting.math.estimators import BayesModal, CustomPrior
+from adaptivetesting.math.estimators import ExpectedAPosteriori, CustomPrior
 from adaptivetesting.math.item_selection import maximum_information_criterion
 from scipy.stats import t
 import pandas as pd
 
-
-# Create item pool from DataFrame
-items_data = pd.DataFrame({
-    "a": [1.32, 1.07, 0.84, 1.19, 0.95],  # discrimination
-    "b": [-0.63, 0.18, -0.84, 0.41, -0.25],  # difficulty
-    "c": [0.17, 0.10, 0.19, 0.15, 0.12],  # guessing
-    "d": [0.87, 0.93, 1.0, 0.89, 0.94]   # upper asymptote
-})
-item_pool = ItemPool.load_from_dataframe(items_data)
-
 # Create adaptive test
 adaptive_test: AdaptiveTest = TestAssembler(
         item_pool=item_pool,
         simulation_id="example",
         participant_id="dummy",
-        ability_estimator=BayesModal,
+        ability_estimator=ExpectedAPosteriori,
         estimator_args={
-            "prior": CustomPrior(t, 100),
+            "prior": CustomPrior(t, 3),
             "optimization_interval":(-10, 10)
         },
         item_selector=maximum_information_criterion,
@@ -51,7 +53,8 @@ keyboard.Keyboard()
 def get_response(item: TestItem) -> int:
     # get index
     item_difficulty: float = item.b
-    stimuli: str = [item for item in item_pool.test_items if item["Difficulty"] == item_difficulty][0]["word"]
+    # select corresponding word from item pool data frame
+    stimuli: str = previc_item_pool.loc[previc_item_pool["Difficulty"] == item_difficulty, "word"].values[0]
 
     # create text box and display stimulus
     text_box = visual.TextBox2(win=win,

{adaptivetesting-1.1.0 → adaptivetesting-1.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name="adaptivetesting"
-version="1.1.0"
+version="1.1.1"
 author="Jonas Engicht"
 author_email="jonas.engicht@uni-jena.de"
 description="adaptivetesting is a Python package that can be used to simulate and evaluate custom CAT scenarios as well as implement them in real-world testing scenarios from a single codebase"
@@ -37,6 +37,12 @@ dev = [
     "sphinx-markdown-builder>=0.6.8",
     "types-tqdm>=4.67.0.20250516",
 ]
+plot = [
+    "matplotlib>=3.10.5",
+]
+test = [
+    "psychopy>=2025.1.1",
+]
 
 [tool.uv.sources]
 adaptivetesting = { workspace = true }