experiment_server 0.3.0__py3-none-any.whl

experiment_server/__init__.py

@@ -0,0 +1,31 @@
+from importlib_metadata import version
+import logging
+from loguru import logger
+
+__version__ = version(__package__)
+
+
+class __InterceptHandler(logging.Handler):
+    def emit(self, record):
+        # Get corresponding Loguru level if it exists
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        # Find caller from where originated the logged message
+        frame, depth = logging.currentframe(), 2
+        while frame.f_code.co_filename == logging.__file__:
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
+
+
+logging.basicConfig(handlers=[__InterceptHandler()], level=0)
+
+from experiment_server._server import server_process
+from experiment_server._client import Client
+from experiment_server._api import Experiment
+
+__all__ = [server_process, Client, Experiment]

experiment_server/_api.py

@@ -0,0 +1,200 @@
+from sys import stdout
+from typing import Any, Dict, Iterable, List, Union
+
+from loguru import logger
+from experiment_server._process_config import process_config_file
+from pathlib import Path
+import json
+
+from experiment_server.utils import ExperimentServerExcetion, FileModifiedWatcher
+
+
+class ParticipantState:
+    def __init__(self, config, participant_index, active):
+        self.participant_index = participant_index
+        self._block_id: int = -1
+        self.config = config
+        self.active = active  # if not started or has ended, then this will be False
+
+    @property
+    def block_id(self) -> int:
+        return self._block_id
+
+    @block_id.setter
+    def block_id(self, block_id: int):
+        if block_id < 0:
+            self._block_id = -1
+            self.active = False
+        elif block_id >= len(self.config):
+            self._block_id = len(self.config)
+            self.active = False
+        else:
+            self.active = True
+            self._block_id = block_id
+        if self.active:
+            assert self._block_id == self.config[self._block_id]["config"]["block_id"]
+
+    @property
+    def block(self) -> Dict[str, Any]|None:
+        if self.block_id >= len(self.config) or self.block_id < 0:
+            return None
+        else:
+            return self.config[self.block_id]
+
+    @property
+    def block_name(self) -> str:
+        if self.block_id >= len(self.config):
+            return "END"
+        elif self.block_id < 0:
+            return "START"
+        return self.config[self.block_id]["name"]
+
+    def move_to_next_block(self) -> str:
+        self.block_id += 1
+        return self.block_name
+
+    def status_string(self):
+        return f'Participant index: {self.participant_index}    \nBlock: {self._block_id} / {len(self.config)}    \n Name: {self.block["name"] if self.block is not None else "N/A"}'
+
+
+class Experiment:
+    """Load and manage experiemnt from a local file."""
+    def __init__(self, config_file:str, default_participant_index:int=1) -> None:
+        assert default_participant_index > 0, "Default participant index should be >0"
+        self.global_state: Dict[int, ParticipantState] = {}
+        self.default_participant_index = default_participant_index
+        self.watchdog = FileModifiedWatcher(config_file, self._config_file_modified_callback)
+        self.config_file = config_file
+
+        if default_participant_index not in self.global_state:
+            self.global_state[default_participant_index] = ParticipantState(process_config_file(config_file, default_participant_index),
+                                                                            default_participant_index,
+                                                                            False)
+
+    def _config_file_modified_callback(self):
+        logger.info("Reloading config")
+        for participantState in self.global_state.values():
+            config = process_config_file(self.config_file, participantState.participant_index)
+            participantState.config = config
+
+    def get_next_participant(self) -> int:
+        """Return the index of the next participant."""
+        new_participant_index = max(self.global_state.keys()) + 1
+        self.add_participant_index(new_participant_index)
+        return new_participant_index
+
+    def add_participant_index(self, participant_index) -> bool:
+        """Add a participant with `participant_index`.
+        If `participant_index` already exists, returns False, else return True"""
+        if participant_index in self.global_state:
+            return False
+        self.global_state[participant_index] = ParticipantState(process_config_file(self.config_file, participant_index),
+                                                                participant_index,
+                                                                False)
+        return True
+
+    def get_participant_state(self, participant_index) -> ParticipantState:
+        if participant_index is None:
+            participant_index = self.default_participant_index
+        if participant_index not in self.global_state:
+            raise ExperimentServerExcetion(f"participant with index {participant_index} is not set. Consider using `add_participant_index`")
+        return self.global_state[participant_index]
+
+    def get_state(self, participant_index:int|None=None):
+        if participant_index is None:
+            participant_index = self.default_participant_index
+        return self.global_state[participant_index].active
+
+    def move_to_next(self, participant_index:int|None=None) -> str:
+        """Moves the pointer to the current block to the next block for `participant_index`.
+        if `participant_index` is None, seld.default_participant_index is used.
+        """
+        if participant_index is None:
+            participant_index = self.default_participant_index
+        return self.global_state[participant_index].move_to_next_block()
+
+    def get_config(self, participant_index:int|None=None) -> Union[Dict[str, Any], None]:
+        """Return the config of the current block for `participant_index`. 
+        if `participant_index` is None, seld.default_participant_index is used.
+        If the experiment has not started (`move_to_next` has not
+        been called atleast once), this will return `None`."""
+        if participant_index is None:
+            participant_index = self.default_participant_index
+        block = self.global_state[participant_index].block
+        if block is None:
+            return None
+        else:
+            return block["config"]
+
+    def reset_participant(self, participant_index:int|None=None) -> bool:
+        """Reset the participant's config to that was loaded from the file."""
+        if participant_index is None:
+            participant_index = self.default_participant_index
+        self.global_state[participant_index].config = process_config_file(self.config_file, participant_index)
+        return True
+
+    def get_blocks_count(self, participant_index:int|None=None) -> int:
+        """Return the total number of blocks."""
+        if participant_index is None:
+            participant_index = self.default_participant_index
+        return len(self.global_state[participant_index].config)
+
+    def get_all_configs(self, participant_index:int|None=None) -> List[dict]:
+        """Return all configs in order for `participant_index`.
+        if `participant_index` is None, seld.default_participant_index is used.
+        """
+        if participant_index is None:
+            participant_index = self.default_participant_index
+        return [c["config"] for c in self.global_state[participant_index].config]
+
+    def move_to_block(self, block_id: int, participant_index:int|None=None) -> str:
+        """For `participant_index` move the pointer of the current
+        block to the block in index a `block_id` in the list of
+        blocks. If `participant_index` is None, seld.default_participant_index is used.
+        """
+        assert isinstance(block_id, int), "`block` should be an int"
+        if participant_index is None:
+            participant_index = self.default_participant_index
+        self.global_state[participant_index].block_id = block_id
+        return self.global_state[participant_index].block_name
+
+    def move_all_to_block(self, block_id: int) -> str:
+        """For active participants move the pointer of the current
+        block to the block in index a `block_id` in the list of
+        blocks.
+        """
+        assert isinstance(block_id, int), "`block` should be an int"
+        for participantState in self.global_state.values():
+            participantState.block_id = block_id
+        return list(self.global_state.values())[0].block_name
+
+
+def _generate_config_json(config_file: Union[str, Path], participant_indices:Iterable[int], out_dir: Union[str, Path, None] = None) -> None:
+    """
+    Write out the json config from the `config_file` for participants in `participant_indices`
+    in the `out_dir`. The `out_dir` should be a directory or should be None.
+    If it is None, it will write the config's, one line per participant. If the passed directory
+    does not exist, it will be created.
+    """
+    if out_dir is not None:
+        out_dir = Path(out_dir)
+        if not out_dir.exists():
+            logger.info(f"Creating direcotry {out_dir}")
+            out_dir.mkdir(parents=True)
+        elif not out_dir.is_dir():
+            raise ExperimentServerExcetion(f"`out_file_location` should be a directory. Got {out_dir}")
+
+    out_files = []
+    for participant_index in participant_indices:
+        config = process_config_file(config_file, participant_index, supress_message=True)
+        if out_dir is not None:
+            out_file = Path(out_dir) / f"{Path(config_file).stem}-participant_{participant_index}.json"
+            out_files.append(out_file)
+
+            with open(out_file, "w") as f:
+                json.dump([c["config"] for c in config], f, indent=2)
+        else:
+            stdout.write(json.dumps([c["config"] for c in config]))
+
+    if len(out_files) != 0:
+        logger.info("Generated files: \n" + "\n".join([str(f) for f in out_files]))

experiment_server/_client.py

@@ -0,0 +1,76 @@
+import json
+from typing import Tuple, Union
+import requests
+
+from experiment_server.utils import ExperimentServerExcetion
+
+
+class Client:
+    def __init__(self, server_host:str ="127.0.0.1", server_port:Union[str, int]="5000") -> None:
+        self._server_url = f"http://{server_host}:{server_port}"
+
+    def _request(self, end_point:str, verb:str) -> Tuple[bool, dict]:
+        url = self._server_url + f"/api/{end_point}"
+        if verb == "GET":
+            r = requests.get(url)
+        elif verb == "POST":
+            r = requests.post(url)
+        elif verb == "PUT":
+            r = requests.put(url)
+        else:
+            raise ExperimentServerExcetion("huh?")
+
+        if r.status_code != 200:
+            return False, {"message": f"status {r.status_code} with text: {r.text}"}
+        return True, json.loads(r.text) if len(r.text) > 0 else ""
+    
+    def _get(self, end_point:str) -> Tuple[bool, dict]:
+        return self._request(end_point, "GET")
+
+    def _post(self, end_point:str) -> Tuple[bool, dict]:
+        return self._request(end_point, "POST")
+
+    def _put(self, end_point:str) -> Tuple[bool, dict]:
+        return self._request(end_point, "PUT")
+
+    def move_to_next(self, participant_index:int|None=None) -> Tuple[bool, dict]:
+        url = _process_participant_index("move-to-next", participant_index)
+        return self._post(url)
+
+    def get_config(self, participant_index:int|None=None) -> Tuple[bool, dict]:
+        url = _process_participant_index("config", participant_index)
+        return self._get(url)
+
+    def server_is_active(self) -> Tuple[bool, dict]:
+        return self._get("active")
+    
+    def get_blocks_count(self, participant_index:int|None=None) -> Tuple[bool, dict]:
+        url = _process_participant_index("blocks-count", participant_index)
+        return self._get(url)
+
+    def get_all_configs(self, participant_index:int|None=None) -> Tuple[bool, dict]:
+        url = _process_participant_index("all-configs", participant_index)
+        return self._get(url)
+
+    def move_to_block(self, block_id:int, participant_index:int|None=None) -> Tuple[bool, dict]:
+        assert isinstance(block_id, int), "`block` should be a int"
+        url = _process_participant_index("move-to-block", participant_index)
+        return self._post(f"{url}/{block_id}")
+
+    def new_participant(self) -> Tuple[bool, dict]:
+        return self._put("new-participant");
+
+    def add_participant(self, participant_index:int) -> Tuple[bool, dict]:
+        assert participant_index is not None
+        url = _process_participant_index("add-participant", participant_index)
+        return self._put(url);
+
+    def shutdown(self) -> Tuple[bool, dict]:
+        return self._post("shutdown")
+
+
+def _process_participant_index(url:str, participant_index:int|None) -> str:
+    if participant_index is not None:
+        assert isinstance(participant_index, int), "`participant_index` should be a int"
+        url += f"/{participant_index}"
+    return url

experiment_server/_participant_ordering.py

@@ -0,0 +1,127 @@
+import random
+import itertools
+from typing import Dict, List, Union
+from easydict import EasyDict as edict
+
+from experiment_server.utils import ExperimentServerConfigurationExcetion, balanced_latin_square
+
+
+ORDERING_BEHAVIOUR = edict({v:v for v in ["randomize", "latin_square", "as_is"]})
+
+
+def construct_participant_condition(config: List[Dict], participant_index: int, order: Union[dict, list], within_groups:str =None, groups:str =None) -> List:
+    if within_groups is None:
+        within_groups = ORDERING_BEHAVIOUR.as_is
+    if groups is None:
+        groups = ORDERING_BEHAVIOUR.as_is
+
+    names = {}
+
+    # To make sure the string based indexing works
+    for idx, c in enumerate(config):
+        c["name"] = str(c["name"])
+        names[c["name"]] = idx
+
+    if len(set(names.values()).symmetric_difference(list(range(len(config))))) != 0:
+        raise ExperimentServerConfigurationExcetion("Duplicate block names: {}".format(set([c["name"] for idx, c in enumerate(config) if idx not in names.values()])))
+
+    if within_groups not in list(ORDERING_BEHAVIOUR.values()):
+        raise ExperimentServerConfigurationExcetion(f"Allowed values for `within_groups` are {ORDERING_BEHAVIOUR.values()}, for {within_groups}")
+    if groups not in list(ORDERING_BEHAVIOUR.values()):
+        raise ExperimentServerConfigurationExcetion(f"Allowed values for `groups` are {ORDERING_BEHAVIOUR.values()}, for {groups}")
+
+    if isinstance(order, list):
+        if not all([isinstance(group, list) for group in order]):
+            order = [order,]
+            # Making sure the stratergy set for groups is used for within groups
+            within_groups = groups
+        if not all([isinstance(g, int) for group in order for g in group]) and not all([isinstance(g, str) for group in order for g in group]):
+            raise ExperimentServerConfigurationExcetion(f"Each group in the order needs to be a list of `int` or list of `str`, got {order}")
+        
+        _filtered_order = order
+        
+    elif isinstance(order, dict):
+        order = {int(k):v for k, v in order.items()}
+        if not all([isinstance(_order, list) for _order in order.values()]) or not all([isinstance(group, list) for _order in order.values() for group in _order]):
+            raise ExperimentServerConfigurationExcetion(f"Each group in orders for all participants needs to be list, got {order}")
+        if not all([isinstance(g, int) for _order in order.values() for group in _order for g in group]) and not all([isinstance(g, str) for _order in order.values() for group in _order for g in group]):
+            raise ExperimentServerConfigurationExcetion(f"Each group in orders for all participants needs to be a list of `int` or list of `str`, got {order}")
+
+        if not all([idx+1 in order.keys() for idx in range(len(order))]):
+            raise ExperimentServerConfigurationExcetion(f"Keys order oredr should match the consecutive indices starting from 1. Got keys {list(order.keys())}, expected keys {list(range(len(order)))}")
+
+        if groups != ORDERING_BEHAVIOUR.as_is:
+            raise ExperimentServerConfigurationExcetion(f"Ordering behaviour for groups should be {ORDERING_BEHAVIOUR.as_is} when order is a dictionary. Got {groups}")
+        _key = ((participant_index - 1) % len(order)) + 1
+        _filtered_order = order[_key]
+
+
+    if groups == ORDERING_BEHAVIOUR.randomize:
+        random.shuffle(_filtered_order)
+    elif groups == ORDERING_BEHAVIOUR.latin_square:
+        _latin_square = balanced_latin_square(len(_filtered_order))
+        _participant_order = _latin_square[(participant_index - 1) % len(_filtered_order)]
+
+        _filtered_order = [_filtered_order[idx] for idx in _participant_order]
+
+    if within_groups == ORDERING_BEHAVIOUR.randomize:
+        for group in _filtered_order:
+            random.shuffle(group)
+    elif within_groups == ORDERING_BEHAVIOUR.latin_square:
+        elements_in_group = set([len(_g) for _g  in _filtered_order])
+        if len(elements_in_group) != 1:
+            raise ExperimentServerConfigurationExcetion(f"Currently {ORDERING_BEHAVIOUR.latin_square} not supported for `within_groups` when the number of elements in all groups are not the same")
+        else:
+            _elements_count = elements_in_group.pop()
+            _latin_square = balanced_latin_square(_elements_count)
+            _group_order = _latin_square[(participant_index - 1) % _elements_count]
+
+            _filtered_order = [[_g[idx] for idx in _group_order] for _g in _filtered_order]
+
+    chained_order = list(itertools.chain(*_filtered_order))
+    if isinstance(chained_order[0], int):
+        return [config[i] for i in chained_order]
+    else:
+        return [config[names[i]] for i in chained_order]
+    
+
+# def _construct_participant_condition_old(config, participant_index, use_latin_square=False, latin_square=None, config_categorization=None, default_configuration=None, randomize=True):
+#     if participant_index < 1:
+#         participant_index = 1
+#     if use_latin_square:
+#         _config = [config[i - 1] for i in latin_square[(participant_index - 1) % len(config)]]
+#     else:
+#         assert len(config_categorization) == 2
+#         if not randomize or participant_index % len(config_categorization) == 0:
+#             init_condition = config_categorization[0][:]
+#             other_condition = config_categorization[1][:]
+#         else:
+#             init_condition = config_categorization[1][:]
+#             other_condition = config_categorization[0][:]
+#         init_condition = [config[i] for i in init_condition]
+#         other_condition = [config[i] for i in other_condition]
+#         random.shuffle(init_condition)
+#         random.shuffle(other_condition)
+
+#         if default_configuration is not None:
+#             default_configuration_config = default_configuration[0]["config"]
+#             non_default_keys = [k for k in default_configuration_config.keys() if k not in ["conditionId"]]
+
+#             init_condition_train = init_condition[0].copy()
+#             init_condition_train["config"] = init_condition_train["config"].copy()
+#             init_condition_train["config"]["conditionId"] = "training1"
+#             for k in non_default_keys:
+#                 init_condition_train["config"][k] = default_configuration_config[k]
+
+#             other_condition_train = other_condition[0].copy()
+#             other_condition_train["config"] = other_condition_train["config"].copy()
+#             other_condition_train["config"]["conditionId"] = "training2"
+#             for k in non_default_keys:
+#                 other_condition_train["config"][k] = default_configuration_config[k]
+
+#             _config = [init_condition_train] + init_condition + [other_condition_train] + other_condition
+
+#         else:
+#             _config = init_condition + other_condition
+#         # _config = [config[i] for i in _config_list]
+#     return _config