rapidata 1.8.3__py3-none-any.whl → 2.44.4__py3-none-any.whl

rapidata/rapidata_client/referee/_early_stopping_referee.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from rapidata.rapidata_client.referee._base_referee import Referee
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from rapidata.api_client.models.i_referee_model import IRefereeModel
+
+
+class EarlyStoppingReferee(Referee):
+    """A referee that stops the task when confidence in the winning category exceeds a threshold.
+
+    This referee implements an early stopping mechanism for classification & compare tasks.
+    It terminates the task when the confidence in the leading category surpasses
+    a specified threshold or when the maximum number of votes is reached.
+
+    The threshold behaves logarithmically, meaning small increments (e.g., from 0.99
+    to 0.999) can significantly impact the stopping criteria.
+
+    This referee is supported for the classification and compare tasks (in compare,
+    the two options are treated as the categories).
+
+    Args:
+        threshold (float, optional): The confidence threshold for early stopping.
+            Defaults to 0.999.
+        max_vote_count (int, optional): The maximum number of votes allowed
+            before stopping. Defaults to 100.
+
+    Attributes:
+        threshold (float): The confidence threshold for early stopping.
+        max_vote_count (int): The maximum number of votes allowed before stopping.
+    """
+
+    def __init__(self, threshold: float = 0.999, max_vote_count: int = 100):
+        if threshold <= 0 or threshold >= 1:
+            raise ValueError("The threshold must be between 0 and 1.")
+        if max_vote_count < 1:
+            raise ValueError("The number of responses must be greater than 0.")
+
+        self.threshold = threshold
+        self.max_vote_count = max_vote_count
+
+    def _to_dict(self):
+        return {
+            "_t": "ProbabilisticAttachCategoryRefereeConfig",
+            "threshold": self.threshold,
+            "maxVotes": self.max_vote_count,
+        }
+
+    def _to_model(self) -> IRefereeModel:
+        from rapidata.api_client.models.i_referee_model_early_stopping_referee_model import (
+            IRefereeModelEarlyStoppingRefereeModel,
+        )
+        from rapidata.api_client.models.i_referee_model import IRefereeModel
+
+        return IRefereeModel(
+            actual_instance=IRefereeModelEarlyStoppingRefereeModel(
+                _t="EarlyStoppingReferee",
+                threshold=self.threshold,
+                maxVotes=self.max_vote_count,
+            )
+        )

rapidata/rapidata_client/referee/_naive_referee.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from rapidata.rapidata_client.referee._base_referee import Referee
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from rapidata.api_client.models.i_referee_model import IRefereeModel
+
+
+class NaiveReferee(Referee):
+    """A simple referee that completes a task after a fixed number of responses.
+
+    This referee implements a straightforward approach to task completion,
+    where the task is considered finished after a predetermined number of
+    responses have been made, regardless of the content or quality of those responses.
+
+    Args:
+        responses (int, optional): The number of responses required
+            to complete the task. Defaults to 10. This is per media item.
+
+    Attributes:
+        responses (int): The number of responses required to complete the task.
+    """
+
+    def __init__(self, responses: int = 10):
+        if responses < 1:
+            raise ValueError("The number of responses must be greater than 0.")
+        super().__init__()
+        self.responses = responses
+
+    def _to_dict(self):
+        return {
+            "_t": "NaiveRefereeConfig",
+            "guessesRequired": self.responses,
+        }
+
+    def _to_model(self) -> IRefereeModel:
+        from rapidata.api_client.models.i_referee_model_naive_referee_model import (
+            IRefereeModelNaiveRefereeModel,
+        )
+        from rapidata.api_client.models.i_referee_model import IRefereeModel
+
+        return IRefereeModel(
+            actual_instance=IRefereeModelNaiveRefereeModel(
+                _t="NaiveReferee",
+                totalVotes=self.responses,
+            )
+        )

rapidata/rapidata_client/selection/__init__.py

@@ -1,6 +1,11 @@
-from .base_selection import Selection
+from ._base_selection import RapidataSelection
 from .demographic_selection import DemographicSelection
 from .labeling_selection import LabelingSelection
 from .validation_selection import ValidationSelection
 from .conditional_validation_selection import ConditionalValidationSelection
 from .capped_selection import CappedSelection
+from .shuffling_selection import ShufflingSelection
+from .ab_test_selection import AbTestSelection
+from .static_selection import StaticSelection
+from .rapidata_retrieval_modes import RapidataRetrievalMode
+from .effort_selection import EffortSelection

rapidata/rapidata_client/selection/_base_selection.py

@@ -0,0 +1,16 @@
+from abc import abstractmethod
+
+from rapidata.api_client.models.i_selection import ISelection
+
+
+class RapidataSelection:
+
+    @abstractmethod
+    def _to_model(self) -> ISelection:
+        pass
+
+    def __str__(self) -> str:
+        return f"{self.__class__.__name__}()"
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}()"

rapidata/rapidata_client/selection/ab_test_selection.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import Sequence
+
+from rapidata.rapidata_client.selection._base_selection import RapidataSelection
+
+
+class AbTestSelection(RapidataSelection):
+    """AbTestSelection Class
+
+    Splits the userbase into two segments and serves them a different collection of rapids.
+
+    Useful for A/B Test.
+
+    Args:
+        a_selections (Sequence[RapidataSelection]): List of selections for group A.
+        b_selections (Sequence[RapidataSelection]): List of selections for group B.
+    """
+
+    def __init__(
+        self,
+        a_selections: Sequence[RapidataSelection],
+        b_selections: Sequence[RapidataSelection],
+    ):
+        self.a_selections = a_selections
+        self.b_selections = b_selections
+
+    def _to_model(self):
+        from rapidata.api_client.models.i_selection import ISelection
+        from rapidata.api_client.models.i_selection_ab_test_selection import (
+            ISelectionAbTestSelection,
+        )
+
+        return ISelection(
+            actual_instance=ISelectionAbTestSelection(
+                _t="AbTestSelection",
+                a=[selection._to_model() for selection in self.a_selections],
+                b=[selection._to_model() for selection in self.b_selections],
+            )
+        )
+
+    def __str__(self) -> str:
+        return f"AbTestSelection(a_selections={self.a_selections}, b_selections={self.b_selections})"
+
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/selection/capped_selection.py

@@ -1,25 +1,42 @@
-from rapidata.api_client.models.capped_selection import (
-    CappedSelection as CappedSelectionModel,
-)
-from rapidata.api_client.models.capped_selection_selections_inner import (
-    CappedSelectionSelectionsInner,
-)
-from rapidata.rapidata_client.selection.base_selection import Selection
+from __future__ import annotations
+
 from typing import Sequence
 
+from rapidata.rapidata_client.selection._base_selection import RapidataSelection
+
+
+class CappedSelection(RapidataSelection):
+    """CappedSelection Class
+
+    Takes in different selections and caps the amount of rapids that can be shown.
+
+    Useful for demographic and conditional validation selections.
 
-class CappedSelection(Selection):
+    Args:
+        selections (Sequence[RapidataSelection]): List of selections to cap.
+        max_rapids (int): The maximum amount of rapids that can be shown for this selection.
+    """
 
-    def __init__(self, selections: Sequence[Selection], max_rapids: int):
+    def __init__(self, selections: Sequence[RapidataSelection], max_rapids: int):
         self.selections = selections
         self.max_rapids = max_rapids
 
-    def to_model(self):
-        return CappedSelectionModel(
-            _t="CappedSelection",
-            selections=[
-                CappedSelectionSelectionsInner(selection.to_model())
-                for selection in self.selections
-            ],
-            max_rapids=self.max_rapids,
+    def _to_model(self):
+        from rapidata.api_client.models.i_selection import ISelection
+        from rapidata.api_client.models.i_selection_capped_selection import (
+            ISelectionCappedSelection,
         )
+
+        return ISelection(
+            actual_instance=ISelectionCappedSelection(
+                _t="CappedSelection",
+                selections=[selection._to_model() for selection in self.selections],
+                maxRapids=self.max_rapids,
+            )
+        )
+
+    def __str__(self) -> str:
+        return f"CappedSelection(selections={self.selections}, max_rapids={self.max_rapids})"
+
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/selection/conditional_validation_selection.py

@@ -1,14 +1,35 @@
-from rapidata.rapidata_client.selection.base_selection import Selection
-from rapidata.api_client.models.conditional_validation_rapid_selection_config import (
-    ValidationChance,
-)
-from rapidata.api_client.models.conditional_validation_selection import (
-    ConditionalValidationSelection as ConditionalValidationSelectionModel,
-)
+from __future__ import annotations
 
+from typing import Optional
 
-class ConditionalValidationSelection(Selection):
-    """Demographic selection class."""
+from rapidata.rapidata_client.config.logger import logger
+from rapidata.rapidata_client.selection._base_selection import RapidataSelection
+
+
+class ConditionalValidationSelection(RapidataSelection):
+    """Conditional validation selection class.
+
+    Probabilistically decides how many validation rapids you want to show per session based on the user score.
+
+    Args:
+        validation_set_id (str): The id of the validation set to be used.
+        thresholds (list[float]): The thresholds to use for the user score.
+        chances (list[float]): The chances of showing a validation rapid for each threshold.
+        rapid_counts (list[int]): The amount of validation rapids that will be shown per session of this validation set for each threshold if selected by probability. (all or nothing)
+        dimensions (Optional[list[str]], optional): The dimensions of the userScore that will be used in the thresholds. Defaults to None.
+
+    Example:
+        ```python
+        ConditionalValidationSelection(
+            validation_set_id="validation_set_id",
+            thresholds=[0, 0.7], # (0 must be the first threshold)
+            chances=[1, 0.2],
+            rapid_counts=[1, 1]
+        )
+        ```
+        This means that there's a 100% chance of showing a validation rapid if the user score is between 0 and 0.7,
+        and a 20% chance of showing a validation rapid if the user score is between 0.7 and 1.
+    """
 
     def __init__(
         self,
@@ -16,22 +37,51 @@ class ConditionalValidationSelection(Selection):
         thresholds: list[float],
         chances: list[float],
         rapid_counts: list[int],
+        dimension: Optional[str] = None,
+        dimensions: Optional[list[str]] = None,
     ):
+        if len(thresholds) != len(chances) or len(thresholds) != len(rapid_counts):
+            raise ValueError(
+                "The lengths of thresholds, chances and rapid_counts must be equal."
+            )
+
+        if dimension:
+            logger.warning("dimension is deprecated, use dimensions instead")
+            dimensions = (dimensions or []) + [dimension]
+
         self.validation_set_id = validation_set_id
         self.thresholds = thresholds
         self.chances = chances
         self.rapid_counts = rapid_counts
+        self.dimensions = dimensions
+
+    def _to_model(self):
+        from rapidata.api_client.models.i_selection import ISelection
+        from rapidata.api_client.models.i_selection_conditional_validation_selection import (
+            ISelectionConditionalValidationSelection,
+        )
+        from rapidata.api_client.models.conditional_validation_selection_validation_chance import (
+            ConditionalValidationSelectionValidationChance,
+        )
 
-    def to_model(self):
-        return ConditionalValidationSelectionModel(
-            _t="ConditionalValidationSelection",
-            validationSetId=self.validation_set_id,
-            validationChances=[
-                ValidationChance(
-                    userScoreThreshold=threshold, chance=chance, rapidCount=rapid_count
-                )
-                for threshold, chance, rapid_count in zip(
-                    self.thresholds, self.chances, self.rapid_counts
-                )
-            ],
+        return ISelection(
+            actual_instance=ISelectionConditionalValidationSelection(
+                _t="ConditionalValidationSelection",
+                validationSetId=self.validation_set_id,
+                validationChances=[
+                    ConditionalValidationSelectionValidationChance(
+                        userScoreThreshold=threshold, chance=chance, rapidCount=rapid_count
+                    )
+                    for threshold, chance, rapid_count in zip(
+                        self.thresholds, self.chances, self.rapid_counts
+                    )
+                ],
+                dimensions=self.dimensions,
+            )
         )
+
+    def __str__(self) -> str:
+        return f"ConditionalValidationSelection(validation_set_id={self.validation_set_id}, thresholds={self.thresholds}, chances={self.chances}, rapid_counts={self.rapid_counts}, dimensions={self.dimensions})"
+
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/selection/demographic_selection.py

@@ -1,13 +1,48 @@
-from rapidata.rapidata_client.selection.base_selection import Selection
-from rapidata.api_client.models.demographic_selection import DemographicSelection as DemographicSelectionModel
+from __future__ import annotations
 
+from rapidata.rapidata_client.selection._base_selection import RapidataSelection
 
-class DemographicSelection(Selection):
-    """Demographic selection class."""
 
-    def __init__(self, keys: list[str], maxRapids: int):
+class DemographicSelection(RapidataSelection):
+    """Demographic selection class.
+
+    This is used to ask demographic questions in an order.
+
+    The keys will select the rapids based on the confidence we already saved for each user.
+
+    If the confidence is high, the users will be selected to solve the rapids with lower probability.
+
+    Args:
+        keys (list[str]): List of keys for the demographic rapids to be shown. As an example: "age"
+        max_rapids (int): The maximum number of rapids to run.\n
+            Allows to provide more keys, in case some of the earlier ones are not selected because of high confidence.
+
+    Example:
+        ```python
+        DemographicSelection(["age", "gender"], 1)
+        ```
+        This will try to ask the user about their age, if that is not selected due to an already high confidence, it will try asking about their gender.
+        The gender question may also be skipped if the confidence is high enough.
+    """
+
+    def __init__(self, keys: list[str], max_rapids: int):
         self.keys = keys
-        self.maxRapids = maxRapids
+        self.max_rapids = max_rapids
+
+    def _to_model(self):
+        from rapidata.api_client.models.i_selection import ISelection
+        from rapidata.api_client.models.i_selection_demographic_selection import (
+            ISelectionDemographicSelection,
+        )
+
+        return ISelection(
+            actual_instance=ISelectionDemographicSelection(
+                _t="DemographicSelection", keys=self.keys, maxRapids=self.max_rapids
+            )
+        )
+
+    def __str__(self) -> str:
+        return f"DemographicSelection(keys={self.keys}, max_rapids={self.max_rapids})"
 
-    def to_model(self):
-        return DemographicSelectionModel(_t="DemographicSelection", keys=self.keys, maxRapids=self.maxRapids)
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/selection/effort_selection.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from rapidata.rapidata_client.selection._base_selection import RapidataSelection
+from rapidata.rapidata_client.selection.rapidata_retrieval_modes import (
+    RapidataRetrievalMode,
+)
+
+
+class EffortSelection(RapidataSelection):
+    """
+    With this selection you can define the effort budget you have for a task.
+    As an example, you have a task that takes 10 seconds to complete. The effort budget would be 10.
+
+    Args:
+        effort_budget (int): The effort budget for the task.
+        retrieval_mode (RetrievalMode): The retrieval mode for the task.
+        max_iterations (int | None): The maximum number of iterations for the task.
+    """
+
+    def __init__(
+        self,
+        effort_budget: int,
+        retrieval_mode: RapidataRetrievalMode = RapidataRetrievalMode.Shuffled,
+        max_iterations: int | None = None,
+    ):
+        self.effort_budget = effort_budget
+        self.retrieval_mode = retrieval_mode
+        self.max_iterations = max_iterations
+
+    def _to_model(self):
+        from rapidata.api_client.models.i_selection import ISelection
+        from rapidata.api_client.models.i_selection_effort_capped_selection import (
+            ISelectionEffortCappedSelection,
+        )
+
+        return ISelection(
+            actual_instance=ISelectionEffortCappedSelection(
+                _t="EffortCappedSelection",
+                effortBudget=self.effort_budget,
+                retrievalMode=self.retrieval_mode.value,
+                maxIterations=self.max_iterations,
+            )
+        )
+
+    def __str__(self) -> str:
+        return f"EffortSelection(effort_budget={self.effort_budget}, retrieval_mode={self.retrieval_mode}, max_iterations={self.max_iterations})"
+
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/selection/labeling_selection.py

@@ -1,14 +1,51 @@
-from typing import Any
-from rapidata.rapidata_client.selection.base_selection import Selection
-from rapidata.api_client.models.labeling_selection import (
-    LabelingSelection as LabelingSelectionModel,
+from __future__ import annotations
+
+from rapidata.rapidata_client.selection._base_selection import RapidataSelection
+from rapidata.rapidata_client.selection.rapidata_retrieval_modes import (
+    RapidataRetrievalMode,
 )
 
 
-class LabelingSelection(Selection):
+class LabelingSelection(RapidataSelection):
+    """Labeling selection class.
+
+    Decides how many actual datapoints you want to show per session.
+
+    Args:
+        amount (int): The amount of labeling rapids that will be shown per session.
+        retrieval_mode (RetrievalMode): The retrieval mode to use. Defaults to "Shuffled".
+        max_iterations (int | None): An annotator can answer the same task only once if the retrieval_mode is "Shuffled"
+            or "Sequential". max_iterations can increase the amount of responses an annotator can do
+            to the same task (datapoint).
+    """
 
-    def __init__(self, amount: int):
+    def __init__(
+        self,
+        amount: int,
+        retrieval_mode: RapidataRetrievalMode = RapidataRetrievalMode.Shuffled,
+        max_iterations: int | None = None,
+    ):
         self.amount = amount
+        self.retrieval_mode = retrieval_mode
+        self.max_iterations = max_iterations
+
+    def _to_model(self):
+        from rapidata.api_client.models.i_selection import ISelection
+        from rapidata.api_client.models.i_selection_labeling_selection import (
+            ISelectionLabelingSelection,
+        )
+
+        return ISelection(
+            actual_instance=ISelectionLabelingSelection(
+                _t="LabelingSelection",
+                amount=self.amount,
+                retrievalMode=self.retrieval_mode.value,
+                maxIterations=self.max_iterations,
+            )
+        )
+
+    def __str__(self) -> str:
+        return f"LabelingSelection(amount={self.amount}, retrieval_mode={self.retrieval_mode}, max_iterations={self.max_iterations})"
 
-    def to_model(self) -> Any:
-        return LabelingSelectionModel(_t="LabelingSelection", amount=self.amount)
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/selection/rapidata_retrieval_modes.py

@@ -0,0 +1,21 @@
+from enum import Enum
+from rapidata.api_client.models.retrieval_mode import RetrievalMode
+
+
+class RapidataRetrievalMode(Enum):
+    """
+    Enum for defining retrieval modes for datapoints.
+    """
+
+    Shuffled = RetrievalMode.SHUFFLED
+    """
+    Will shuffle the datapoints randomly for each user. The user will then see the datapoints in that order. This will take into account the "max_iterations" parameter.
+    """
+    Sequential = RetrievalMode.SEQUENTIAL
+    """
+    Will show the datapoints in the order they are in the dataset. This will take into account the "max_iterations" parameter.
+    """
+    Random = RetrievalMode.RANDOM
+    """
+    Will just randomly feed the datapoints to the annotators. This will NOT take into account the "max_iterations" parameter.
+    """

rapidata/rapidata_client/selection/rapidata_selections.py

@@ -0,0 +1,47 @@
+from rapidata.rapidata_client.selection import (
+    DemographicSelection,
+    LabelingSelection,
+    ValidationSelection,
+    ConditionalValidationSelection,
+    CappedSelection,
+    ShufflingSelection,
+)
+
+
+class RapidataSelections:
+    """RapidataSelections Classes
+
+    Selections are used to define what type of tasks and in what order they are shown to the user.
+    All selections combined are called a "Session". A session can contain multiple tasks of different types of tasks.
+    As an example, a session might be 1 validation task, 2 labeling tasks.
+
+    Attributes:
+        labeling (LabelingSelection): Decides how many actual datapoints you want to show per session.
+        validation (ValidationSelection): Decides how many validation rapids you want to show per session.
+        conditional_validation (ConditionalValidationSelection): Probabilistically decides how many validation rapids you want to show per session based on the user score.
+        demographic (DemographicSelection): Decides if and how many demographic questions you want to show per session.
+        capped (CappedSelection): Takes in different selections and caps the amount of rapids that can be shown.
+        shuffling (ShufflingSelection): Shuffles the selections provided in the list.
+
+    Example:
+        ```python
+        from rapidata import LabelingSelection, ValidationSelection
+        selections=[ValidationSelection("your-validation-set-id", 1),
+                    LabelingSelection(2)]
+        ```
+
+        This will require annotators to complete one validation task followed by two labeling tasks.
+    """
+
+    labeling = LabelingSelection
+    validation = ValidationSelection
+    conditional_validation = ConditionalValidationSelection
+    demographic = DemographicSelection
+    capped = CappedSelection
+    shuffling = ShufflingSelection
+
+    def __str__(self) -> str:
+        return f"RapidataSelections(labeling={self.labeling}, validation={self.validation}, conditional_validation={self.conditional_validation}, demographic={self.demographic}, capped={self.capped}, shuffling={self.shuffling})"
+
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/selection/shuffling_selection.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import Sequence
+
+from rapidata.rapidata_client.selection._base_selection import RapidataSelection
+
+
+class ShufflingSelection(RapidataSelection):
+    """ShufflingSelection Class
+
+    Shuffles the selections provided in the list.
+
+    Args:
+        selections (Sequence[RapidataSelection]): List of selections to shuffle.
+
+    Example:
+        ```python
+        selection = ShufflingSelection(
+                    [ValidSelections("validation_id", 1), LabelingSelection(2)])
+        ```
+        This means that the users will get 1 validation task and 2 labeling tasks in a shuffled order.
+    """
+
+    def __init__(self, selections: Sequence[RapidataSelection]):
+        self.selections = selections
+
+    def _to_model(self):
+        from rapidata.api_client.models.i_selection import ISelection
+        from rapidata.api_client.models.i_selection_shuffling_selection import (
+            ISelectionShufflingSelection,
+        )
+
+        return ISelection(
+            actual_instance=ISelectionShufflingSelection(
+                _t="ShufflingSelection",
+                selections=[selection._to_model() for selection in self.selections],
+            )
+        )
+
+    def __str__(self) -> str:
+        return f"ShufflingSelection(selections={self.selections})"
+
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/selection/static_selection.py

@@ -0,0 +1,33 @@
+from rapidata.rapidata_client.selection._base_selection import RapidataSelection
+
+
+class StaticSelection(RapidataSelection):
+    """StaticSelection Class
+
+    Given a list of RapidIds, theses specific rapids will be shown in order for every session.
+
+    Args:
+        rapid_ids (list[str]): List of rapid ids to show.
+    """
+
+    def __init__(self, rapid_ids: list[str]):
+        self.rapid_ids = rapid_ids
+
+    def _to_model(self):
+        from rapidata.api_client.models.i_selection import ISelection
+        from rapidata.api_client.models.i_selection_static_selection import (
+            ISelectionStaticSelection,
+        )
+
+        return ISelection(
+            actual_instance=ISelectionStaticSelection(
+                _t="StaticSelection",
+                rapidIds=self.rapid_ids,
+            )
+        )
+
+    def __str__(self) -> str:
+        return f"StaticSelection(rapid_ids={self.rapid_ids})"
+
+    def __repr__(self) -> str:
+        return self.__str__()