adaptivetesting 1.0.0__py3-none-any.whl

adaptivetesting/models/__adaptive_test.py

@@ -0,0 +1,183 @@
+from typing import List
+from .__test_item import TestItem
+from ..math.item_selection.__urrys_rule import urrys_rule
+from ..math.__standard_error import standard_error
+import abc
+from .__test_result import TestResult
+from .__item_pool import ItemPool
+
+
+class AdaptiveTest(abc.ABC):
+    def __init__(self, item_pool: ItemPool,
+                 simulation_id: str,
+                 participant_id: int,
+                 true_ability_level: float,
+                 initial_ability_level: float = 0,
+                 simulation: bool = True,
+                 DEBUG=False):
+        """Abstract implementation of an adaptive test.
+        All abstract methods have to be overridden
+        to create an instance of this class.
+
+        Abstract methods:
+            - estimate_ability_level
+
+        Args:
+            item_pool (ItemPool): item pool used for the test
+
+            simulation_id (str): simulation id
+
+            participant_id (int): participant id
+
+            true_ability_level (float): true ability level (must always be set)
+
+            initial_ability_level (float): initially assumed ability level
+
+            simulation (bool): will the test be simulated
+
+            DEBUG (bool): enables debug mode
+        """
+        self.true_ability_level: float = true_ability_level
+        self.simulation_id = simulation_id
+        self.participant_id: int = participant_id
+        # set start values
+        self.ability_level = initial_ability_level
+        self.answered_items: List[TestItem] = []
+        self.response_pattern: List[int] = []
+        self.test_results: List[TestResult] = []
+        # load items
+        self.item_pool = item_pool
+
+        # debug
+        self.DEBUG = DEBUG
+        self.simulation = simulation
+
+    def get_item_difficulties(self) -> List[float]:
+        """
+        Returns:
+             List[float]: difficulties of items in the item pool
+        """
+        return [item.b for item in self.item_pool.test_items]
+
+    def get_answered_items_difficulties(self) -> List[float]:
+        """
+        Returns:
+            List[float]: difficulties of answered items
+        """
+        return [item.b for item in self.answered_items]
+    
+    def get_answered_items(self) -> List[TestItem]:
+        """
+        Returns:
+            List[TestItem]: answered items
+        """
+        return self.answered_items
+
+    def get_ability_se(self) -> float:
+        """
+        Calculate the current standard error
+        of the ability estimation.
+
+        Returns:
+            float: standard error of the ability estimation
+
+        """
+        answered_items= self.get_answered_items()
+        return standard_error(answered_items, self.ability_level)
+
+    def get_next_item(self) -> TestItem:
+        """Select next item using Urry's rule.
+
+        Returns:
+            TestItem: selected item
+        """
+        item = urrys_rule(self.item_pool.test_items, self.ability_level)
+        return item
+
+    @abc.abstractmethod
+    def estimate_ability_level(self) -> float:
+        """
+        Estimates ability level.
+        The method has to be implemented by subclasses.
+
+        Returns:
+            float: estimated ability level
+        """
+        pass
+
+    def get_response(self, item: TestItem) -> int:
+        """If the adaptive test is not used for simulation.
+        This method is used to get user feedback.
+
+        Args:
+            item (TestItem): test item shown to the participant
+
+        Return:
+            int: participant's response
+        """
+        raise NotImplementedError("This functionality is not implemented by default.")
+
+    def run_test_once(self):
+        """
+        Runs the test procedure once.
+        Saves the result to test_results of
+        the current instance.
+        """
+        # get item
+        item = self.get_next_item()
+        if item is not None:
+            if self.DEBUG:
+                print(f"Selected {item.b} for an ability level of {self.ability_level}.")
+
+        # check if simulation is running
+        response = None
+        if self.simulation is True:
+            response = self.item_pool.get_item_response(item)
+        else:
+            # not simulation
+            response = self.get_response(item)
+
+        if self.DEBUG:
+            print(f"Response: {response}")
+
+        # add response to response pattern
+        self.response_pattern.append(response)
+        # add item to answered items list
+        self.answered_items.append(item)
+
+        # estimate ability level
+        estimation = self.estimate_ability_level()
+
+        # update estimated ability level
+        self.ability_level = estimation
+        if self.DEBUG:
+            print(f"New estimation is {self.ability_level}")
+        # remove item from item pool
+        self.item_pool.delete_item(item)
+        if self.DEBUG:
+            print(f"Now, there are only {len(self.item_pool.test_items)} left in the item pool.")
+        # create result
+        result: TestResult = TestResult(
+            ability_estimation=estimation,
+            standard_error=self.get_ability_se(),
+            showed_item=item.b,
+            response=response,
+            test_id=self.simulation_id,
+            true_ability_level=self.true_ability_level
+        )
+
+        # add result to memory
+        self.test_results.append(result)
+
+    def check_se_criterion(self, value: float) -> bool:
+        if self.get_ability_se() <= value:
+            return True
+        else:
+            return False
+
+    def check_length_criterion(self, value: float) -> bool:
+        if len(self.answered_items) >= value:
+            return True
+        else:
+            return False
+

adaptivetesting/models/__algorithm_exception.py

@@ -0,0 +1,3 @@
+class AlgorithmException(Exception):
+    """Exception that is thrown when the estimation process did not find a maximum."""
+

adaptivetesting/models/__init__.py

@@ -0,0 +1,7 @@
+from .__adaptive_test import AdaptiveTest
+from .__algorithm_exception import AlgorithmException
+from .__item_pool import ItemPool 
+from .__item_selection_exception import ItemSelectionException
+from .__test_item import TestItem
+from .__test_result import TestResult
+

adaptivetesting/models/__item_pool.py

@@ -0,0 +1,246 @@
+from .__test_item import TestItem
+from typing import List, Tuple
+from pandas import DataFrame
+
+
+class ItemPool:
+    def __init__(self,
+                 test_items: List[TestItem],
+                 simulated_responses: List[int] | None = None):
+        """An item pool has to be created for an adaptive test.
+        For that, a list of test items has to be provided. If the package is used
+        to simulate adaptive tests, simulated responses have to be supplied as well.
+        The responses are matched to the items internally.
+        Therefore, both have to be in the same order.
+
+        Args:
+            test_items (List[TestItem]): A list of test items. Necessary for any adaptive test.
+
+            simulated_responses (List[int]): A list of simulated responses. Required for CAT simulations.
+        """
+        self.test_items: List[TestItem] = test_items
+        self.simulated_responses: List[int] | None = simulated_responses
+
+    def get_item_by_index(self, index: int) -> Tuple[TestItem, int] | TestItem:
+        """Returns item and if defined the simulated response.
+
+        Args:
+            index (int): Index of the test item in the item pool to return.
+
+        Returns:
+            TestItem or (TestItem, Simulated Response)
+        """
+        selected_item = self.test_items[index]
+        if self.simulated_responses is not None:
+            simulated_response = self.simulated_responses[index]
+            return selected_item, simulated_response
+        else:
+            return selected_item
+    
+    def get_item_by_item(self, item: TestItem) -> Tuple[TestItem, int] | TestItem:
+        """Returns item and if defined the simulated response.
+
+        Args:
+            item (TestItem): item to return.
+
+        Returns:
+            TestItem or (TestItem, Simulated Response)
+        """
+        index = self.test_items.index(item)
+        selected_item = self.test_items[index]
+        if self.simulated_responses is not None:
+            simulated_response = self.simulated_responses[index]
+            return selected_item, simulated_response
+        else:
+            return selected_item
+
+    def get_item_response(self, item: TestItem) -> int:
+        """
+        Gets the simulated response to an item if available.
+        A `ValueError` will be raised if a simulated response is not available.
+
+        Args:
+            item (TestItem): item to get the corresponding response
+
+        Returns:
+            (int): response (either `0` or `1`)
+        """
+        if self.simulated_responses is None:
+            raise ValueError("Simulated responses not provided")
+        else:
+            i, res = self.get_item_by_item(item) # type: ignore
+            return res
+
+    def delete_item(self, item: TestItem) -> None:
+        """Deletes item from item pool.
+        If simulated responses are defined, they will be deleted as well.
+
+        Args:
+            item (TestItem): The test item to delete.
+        """
+        # get index
+        index = self.test_items.index(item)
+        # remove item at index
+        self.test_items.pop(index)
+        # remove response at index
+        if self.simulated_responses is not None:
+            self.simulated_responses.pop(index)
+
+# STATIC LOAD METHODS
+    @staticmethod
+    def load_from_list(
+        b: List[float],
+        a: List[float] | None = None,
+        c: List[float] | None = None,
+        d: List[float] | None = None,
+        simulated_responses: List[int] | None = None
+        ) -> "ItemPool":
+        """
+        Creates test items from a list of floats.
+
+        Args:
+            a (List[float]): discrimination parameter
+            
+            b (List[float]): difficulty parameter
+            
+            c (List[float]): guessing parameter
+            
+            d (List[float]): slipping parameter
+            
+            simulated_responses (List[int]): simulated responses
+
+        Returns:
+            List[TestItem]: item pool
+        
+        """
+        items: List[TestItem] = []
+
+        for difficulty in b:
+            item = TestItem()
+            item.b = difficulty
+            items.append(item)
+
+        # check if a, b, c, d are the same length
+        if a is not None:
+            if len(a) != len(b):
+                raise ValueError("Length of a and b has to be the same.")
+            for i, discrimination in enumerate(a):
+                items[i].a = discrimination
+        
+        if c is not None:
+            if len(c) != len(b):
+                raise ValueError("Length of c and b has to be the same.")
+            for i, guessing in enumerate(c):
+                items[i].c = guessing
+        
+        if d is not None:
+            if len(d) != len(b):
+                raise ValueError("Length of d and b has to be the same.")
+            for i, slipping in enumerate(d):
+                items[i].d = slipping
+
+        item_pool = ItemPool(items)
+        item_pool.simulated_responses = simulated_responses
+
+        return item_pool
+
+    @staticmethod
+    def load_from_dict(source: dict[str, List[float]],
+                       simulated_responses: List[int] | None = None) -> "ItemPool":
+        """Creates test items from a dictionary.
+        The dictionary has to have the following keys:
+            
+            - a
+            - b
+            - c
+            - d
+        each containing a list of float.
+
+        Args:
+            source (dict[str, List[float]]): item pool dictionary
+
+        Returns:
+            List[TestItem]: item pool
+        """
+        a = source.get("a")
+        b = source.get("b")
+        c = source.get("c")
+        d = source.get("d")
+
+        # check none
+        if a is None:
+            raise ValueError("a cannot be None")
+        
+        if b is None:
+            raise ValueError("b cannot be None")
+        
+        if c is None:
+            raise ValueError("c cannot be None")
+        
+        if d is None:
+            raise ValueError("d cannot be None")
+
+        # check if a, b, c, and d have the same length
+        if not (len(a) == len(b) == len(c) == len(d)):
+            raise ValueError("All lists in the source dictionary must have the same length")
+
+        n_items = len(b)
+        items: List[TestItem] = []
+        for i in range(n_items):
+            item = TestItem()
+            item.a = a[i]
+            item.b = b[i]
+            item.c = c[i]
+            item.d = d[i]
+
+            items.append(item)
+
+        
+        item_pool = ItemPool(items, simulated_responses)
+        
+        return item_pool
+    
+    @staticmethod
+    def load_from_dataframe(source: DataFrame) -> "ItemPool":
+        """Creates item pool from a pandas DataFrame.
+        Required columns are: `a`, `b`, `c`, `d`. 
+        Each column has to contain float values.
+        A `simulated_responses` (int values) column can be added to 
+        the DataFrame to provide simulated responses.
+        
+
+        Args:
+            source (DataFrame): _description_
+
+        Returns:
+            ItemPool: _description_
+        """
+
+        # check if columns are present
+        if "a" not in source.columns:
+            raise ValueError("Column 'a' not found.")
+        
+        if "b" not in source.columns:
+            raise ValueError("Column 'b' not found.")
+        
+        if "c" not in source.columns:
+            raise ValueError("Column 'c' not found.")
+        
+        if "d" not in source.columns:
+            raise ValueError("Column 'd' not found.")
+        
+        # get values
+        a: List[float] = source["a"].values.tolist() # type: ignore
+        b: List[float] = source["b"].values.tolist() # type: ignore
+        c: List[float] = source["c"].values.tolist() # type: ignore
+        d: List[float] = source["d"].values.tolist() # type: ignore
+
+        # create item pool
+        item_pool = ItemPool.load_from_list(a=a, b=b, c=c, d=d)
+    
+        # check if simulated responses are present
+        if "simulated_responses" in source.columns:
+            simulated_responses: List[int] = source["simulated_responses"].values.tolist() # type: ignore
+            item_pool.simulated_responses = simulated_responses
+        
+        return item_pool

adaptivetesting/models/__item_selection_exception.py

@@ -0,0 +1,3 @@
+class ItemSelectionException(Exception):
+    def __init__(self, *args):
+        super().__init__(*args)

adaptivetesting/models/__test_item.py

@@ -0,0 +1,29 @@
+from typing import Any, Dict, List
+from dataclasses import dataclass
+
+
+class TestItem:
+    def __init__(self):
+        """Representation of a test item in the item pool.
+        The format is equal to the implementation in catR.
+
+        Properties:
+            - a (float):
+            - b (float): difficulty
+            - c (float): 
+            - d (float):
+
+        """
+        self.id: int | None = None
+        self.a: float = 1
+        self.b: float = float("nan")
+        self.c: float = 0
+        self.d: float = 1
+    
+    def as_dict(self) -> dict[str, float]:
+        return {
+            "a": self.a,
+            "b": self.b,
+            "c": self.c,
+            "d": self.d
+        }

adaptivetesting/models/__test_result.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from typing import Dict
+
+
+@dataclass
+class TestResult:
+    """Representation of simulation test results"""
+    test_id: str
+    ability_estimation: float
+    standard_error: float
+    showed_item: float
+    response: int
+    true_ability_level: float
+
+    @staticmethod
+    def from_dict(dictionary: Dict) -> 'TestResult':
+        """Create a TestResult from a dictionary
+
+        Args:
+            dictionary: with the fields `test_id`, `ability_estimation`, `standard_error`, `showed_item`, `response`,
+            `true_ability_level`
+        """
+        return TestResult(**dictionary)

adaptivetesting/services/__estimator_interface.py

@@ -0,0 +1,40 @@
+from abc import ABC, abstractmethod
+from typing import List, Tuple
+import jax.numpy as np
+from ..models.__test_item import TestItem
+
+class IEstimator(ABC):
+    def __init__(self,
+                 response_pattern: List[int] | np.ndarray,
+                 items: List[TestItem],
+                 optimization_interval: Tuple[float, float] = (-10, 10)):
+        """This is the interface required for every possible
+        estimator.
+        Any estimator inherits from this class and implements
+        the `get_estimation` method.
+
+        Args:
+            response_pattern (List[int]): list of responses (0: wrong, 1:right)
+            
+            items (List[TestItem]): list of answered items
+        """
+        if type(response_pattern) is not np.ndarray:
+            self.response_pattern = np.array(response_pattern)
+        else:
+            self.response_pattern = response_pattern
+        self.optimization_interval = optimization_interval
+
+        # convert items to parameter arrays
+        self.a = np.array([i.a for i in items])
+        self.b = np.array([i.b for i in items])
+        self.c = np.array([i.c for i in items])
+        self.d = np.array([i.d for i in items])
+        
+    @abstractmethod
+    def get_estimation(self) -> float:
+        """Get the currently estimated ability.
+
+        Returns:
+            float: ability
+        """
+        pass

adaptivetesting/services/__init__.py

@@ -0,0 +1,2 @@
+from .__test_results_interface import ITestResults
+from .__estimator_interface import IEstimator

adaptivetesting/services/__test_results_interface.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+from ..models.__test_result import TestResult
+from typing import List
+
+
+class ITestResults(ABC):
+
+    def __init__(self, simulation_id: str, participant_id: int):
+        """Interface for saving and reading test results.
+        This interface may mainly be used for saving simulation results.
+
+        Args:
+            simulation_id (str): The simulation ID. Name of the results file.
+
+            participant_id (int): The participant ID.
+        """
+        self.simulation_id = simulation_id
+        self.filename = f"data/{self.simulation_id}.db"
+        self.participant_id: int = participant_id
+
+    @abstractmethod
+    def save(self, test_results: List[TestResult]) -> None:
+        pass
+
+    @abstractmethod
+    def load(self) -> List[TestResult]:
+        pass

adaptivetesting/simulation/__init__.py

@@ -0,0 +1 @@
+from .__simulation import *

adaptivetesting/simulation/__simulation.py

@@ -0,0 +1,75 @@
+from ..models.__adaptive_test import AdaptiveTest
+from ..data.__sqlite_context import SQLiteContext
+from ..data.__pickle_context import PickleContext
+from ..services.__test_results_interface import ITestResults
+from enum import Enum
+
+
+class ResultOutputFormat(Enum):
+    """
+    Enum for selecting the output format for
+    the test results
+    """
+    SQLITE = 1
+    PICKLE = 2
+
+
+class StoppingCriterion(Enum):
+    """Enum for selecting the stopping criterion
+    for the adaptive test"""
+    SE = 1
+    LENGTH = 2
+
+
+class Simulation:
+    def __init__(self,
+                 test: AdaptiveTest,
+                 test_result_output: ResultOutputFormat):
+        """
+        This class can be used for simulating CAT.
+
+        Args:
+            test (AdaptiveTest): instance of an adaptive test implementation (see implementations module)
+
+            test_result_output (ResultOutputFormat): test results output format
+        """
+        self.test = test
+        self.test_result_output = test_result_output
+
+    def simulate(self,
+                 criterion: StoppingCriterion = StoppingCriterion.SE,
+                 value: float = 0.4):
+        """Runs test until the stopping criterion is met.
+
+        Args:
+            criterion (StoppingCriterion): selected stopping criterion
+
+            value (float): either standard error value or test length value that has to be met by the test
+
+        """
+        stop_test = False
+        while stop_test is False:
+            # run test
+            self.test.run_test_once()
+            # check available items
+            if len(self.test.item_pool.test_items) == 0:
+                stop_test = True
+            else:
+                if criterion == StoppingCriterion.SE:
+                    stop_test = self.test.check_se_criterion(value)
+                elif criterion == StoppingCriterion.LENGTH:
+                    stop_test = self.test.check_length_criterion(value)
+
+    def save_test_results(self):
+        """Saves the test results to the specified output format."""
+        data_context: ITestResults
+        if self.test_result_output == ResultOutputFormat.PICKLE:
+            data_context = PickleContext(simulation_id=self.test.simulation_id,
+                                         participant_id=self.test.participant_id)
+        else:
+            data_context = SQLiteContext(
+                simulation_id=self.test.simulation_id,
+                participant_id=self.test.participant_id
+            )
+        # save results
+        data_context.save(self.test.test_results)