rapidata 2.23.2__py3-none-any.whl → 3.0.1__py3-none-any.whl

rapidata/rapidata_client/validation/validation_set_manager.py

@@ -1,23 +1,43 @@
+from __future__ import annotations
+
+import time
+import urllib.parse
+import webbrowser
+from colorama import Fore
+from typing import Literal, TYPE_CHECKING
 from rapidata.api_client import QueryModel
-from rapidata.rapidata_client.validation.rapidata_validation_set import RapidataValidationSet
-from rapidata.api_client.models.create_validation_set_model import CreateValidationSetModel
+from rapidata.rapidata_client.validation.rapidata_validation_set import (
+    RapidataValidationSet,
+)
+from rapidata.api_client.models.create_validation_set_model import (
+    CreateValidationSetModel,
+)
 from rapidata.service.openapi_service import OpenAPIService
-from rapidata.rapidata_client.assets.data_type_enum import RapidataDataTypes
 from rapidata.rapidata_client.validation.rapids.rapids_manager import RapidsManager
-from rapidata.rapidata_client.validation.rapids.rapids import Rapid
-from rapidata.rapidata_client.metadata import PromptMetadata
 
 from rapidata.api_client.models.page_info import PageInfo
 from rapidata.api_client.models.root_filter import RootFilter
 from rapidata.api_client.models.filter import Filter
 from rapidata.api_client.models.sort_criterion import SortCriterion
-from rapidata.api_client.exceptions import BadRequestException
-from urllib3._collections import HTTPHeaderDict # type: ignore[import]
+from rapidata.api_client.models.sort_direction import SortDirection
+from rapidata.api_client.models.filter_operator import FilterOperator
 
 from rapidata.rapidata_client.validation.rapids.box import Box
 
-from rapidata.rapidata_client.logging import logger, managed_print
+from rapidata.rapidata_client.config import (
+    logger,
+    managed_print,
+    rapidata_config,
+    tracer,
+)
 from tqdm import tqdm
+from rapidata.rapidata_client.validation.rapids.rapids import Rapid
+
+if TYPE_CHECKING:
+    from rapidata.rapidata_client.workflow import Workflow
+    from rapidata.rapidata_client.datapoints._datapoint import Datapoint
+    from rapidata.rapidata_client.settings._rapidata_setting import RapidataSetting
+    from typing import Sequence
 
 
 class ValidationSetManager:
@@ -27,24 +47,35 @@ class ValidationSetManager:
     Attributes:
         rapid (RapidsManager): The RapidsManager instance.
     """
+
     def __init__(self, openapi_service: OpenAPIService) -> None:
-        self.__openapi_service = openapi_service
-        self.rapid = RapidsManager()
+        self._openapi_service = openapi_service
+        self.rapid = RapidsManager(openapi_service)
         logger.debug("ValidationSetManager initialized")
 
-    def create_classification_set(self,
+    def _get_total_and_labeled_rapids_count(
+        self, validation_set_id: str
+    ) -> tuple[int, int]:
+        uploaded_rapids = self._openapi_service.validation_api.validation_set_validation_set_id_rapids_get(
+            validation_set_id=validation_set_id
+        ).items
+        return len(uploaded_rapids), sum(1 for rapid in uploaded_rapids if rapid.truth)
+
+    def create_classification_set(
+        self,
         name: str,
         instruction: str,
         answer_options: list[str],
         datapoints: list[str],
         truths: list[list[str]],
-        data_type: str = RapidataDataTypes.MEDIA,
+        data_type: Literal["media", "text"] = "media",
         contexts: list[str] | None = None,
+        media_contexts: list[str] | None = None,
         explanations: list[str | None] | None = None,
         dimensions: list[str] = [],
     ) -> RapidataValidationSet:
         """Create a classification validation set.
-        
+
         Args:
             name (str): The name of the validation set. (will not be shown to the labeler)
             instruction (str): The instruction by which the labeler will answer.
@@ -55,10 +86,13 @@ class ValidationSetManager:
                     options: ["yes", "no", "maybe"]
                     datapoints: ["datapoint1", "datapoint2"]
                     truths: [["yes"], ["no", "maybe"]] -> first datapoint correct answer is "yes", second datapoint is "no" or "maybe"
-            data_type (str, optional): The type of data. Defaults to RapidataDataTypes.MEDIA. Other option: RapidataDataTypes.TEXT ("text").
+            data_type (str, optional): The type of data. Defaults to "media" (any form of image, video or audio). Other option: "text".
             contexts (list[str], optional): The contexts for each datapoint. Defaults to None.\n
                 If provided has to be the same length as datapoints and will be shown in addition to the instruction and answer options. (Therefore will be different for each datapoint)
                 Will be match up with the datapoints using the list index.
+            media_contexts (list[str], optional): The list of media contexts i.e. links to the images / videos for the comparison. Defaults to None.\n
+                If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint)
+                Will be matched up with the datapoints using the list index.
             explanations (list[str | None], optional): The explanations for each datapoint. Will be given to the annotators in case the answer is wrong. Defaults to None.
             dimensions (list[str], optional): The dimensions to add to the validation set accross which users will be tracked. Defaults to [] which is the default dimension.
 
@@ -70,44 +104,61 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint correct answer is "yes", second datapoint is "no" or "maybe"
         """
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-        
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
-
-        if(explanations and len(explanations) != len(datapoints)):
-            raise ValueError("The numeber of reasons and datapoints must be equal, the index must align, but can be padded with None")
-       
-        logger.debug("Creating classification rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapids.append(
-                self.rapid.classification_rapid(
-                    instruction=instruction,
-                    answer_options=answer_options,
-                    datapoint=datapoints[i],
-                    truths=truths[i],
-                    data_type=data_type,
-                    metadata=[PromptMetadata(contexts[i])] if contexts else [],
-                    explanation=explanations[i] if explanations != None else None
+        with tracer.start_as_current_span(
+            "ValidationSetManager.create_classification_set"
+        ):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
+
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
+
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
+
+            if explanations and len(explanations) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating classification rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.classification_rapid(
+                        instruction=instruction,
+                        answer_options=answer_options,
+                        datapoint=datapoints[i],
+                        truths=truths[i],
+                        data_type=data_type,
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanations[i] if explanations != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting classification rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting classification rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
-    def create_compare_set(self,
+    def create_compare_set(
+        self,
         name: str,
         instruction: str,
         datapoints: list[list[str]],
         truths: list[str],
-        data_type: str = RapidataDataTypes.MEDIA,
+        data_type: Literal["media", "text"] = "media",
         contexts: list[str] | None = None,
+        media_contexts: list[str] | None = None,
         explanation: list[str | None] | None = None,
         dimensions: list[str] = [],
     ) -> RapidataValidationSet:
@@ -121,12 +172,15 @@ class ValidationSetManager:
                     instruction: "Which image has a cat?"
                     datapoints = [["image1.jpg", "image2.jpg"], ["image3.jpg", "image4.jpg"]]
                     truths: ["image1.jpg", "image4.jpg"] -> first comparison image1.jpg has a cat, second comparison image4.jpg has a cat
-            datapoints (list[list[str]]): The compare datapoints to create the validation set with. 
+            datapoints (list[list[str]]): The compare datapoints to create the validation set with.
                 Outer list is for each comparison, inner list the two images/texts that will be compared.
-            data_type (str, optional): The type of data. Defaults to RapidataDataTypes.MEDIA. Other option: RapidataDataTypes.TEXT ("text").
+            data_type (str, optional): The type of data. Defaults to "media" (any form of image, video or audio). Other option: "text".
             contexts (list[str], optional): The contexts for each datapoint. Defaults to None.\n
                 If provided has to be the same length as datapoints and will be shown in addition to the instruction and truth. (Therefore will be different for each datapoint)
                 Will be match up with the datapoints using the list index.
+            media_contexts (list[str], optional): The list of media contexts i.e. links to the images / videos for the comparison. Defaults to None.\n
+                If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint)
+                Will be matched up with the datapoints using the list index.
             explanation (list[str | None], optional): The explanations for each datapoint. Will be given to the annotators in case the answer is wrong. Defaults to None.
             dimensions (list[str], optional): The dimensions to add to the validation set accross which users will be tracked. Defaults to [] which is the default dimension.
 
@@ -138,37 +192,51 @@ class ValidationSetManager:
             ```
             This would mean: first comparison image1.jpg has a cat, second comparison image4.jpg has a cat
         """
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, str) for truth in truths]):
-            raise ValueError("Truths must be a list of strings")
-
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
- 
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The numeber of reasons and datapoints must be equal, the index must align, but can be padded with None")
-        
-        logger.debug("Creating comparison rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapids.append(
-                self.rapid.compare_rapid(
-                    instruction=instruction,
-                    truth=truths[i],
-                    datapoint=datapoints[i],
-                    data_type=data_type,
-                    metadata=[PromptMetadata(contexts[i])] if contexts else [],
-                    explanation=explanation[i] if explanation != None else None
+        with tracer.start_as_current_span("ValidationSetManager.create_compare_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, str) for truth in truths]):
+                raise ValueError("Truths must be a list of strings")
+
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
+
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
                 )
-            )
-        
-        logger.debug("Submitting comparison rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
-  
-    def create_select_words_set(self,
+
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating comparison rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.compare_rapid(
+                        instruction=instruction,
+                        truth=truths[i],
+                        datapoint=datapoints[i],
+                        data_type=data_type,
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
+                )
+
+            logger.debug("Submitting comparison rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+
+    def create_select_words_set(
+        self,
         name: str,
         instruction: str,
         truths: list[list[int]],
@@ -204,41 +272,52 @@ class ValidationSetManager:
             truths: [[0, 1], [2]]
             ```
             This would mean: first datapoint the correct words are "this" and "example", second datapoint is "with"
-            """
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-
-        if len(datapoints) != len(truths) or len(datapoints) != len(sentences):
-            raise ValueError("The number of datapoints, truths, and sentences must be equal")
- 
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The numeber of reasons and datapoints must be equal, the index must align, but can be padded with None")
-
-        logger.debug("Creating select words rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapids.append(
-                self.rapid.select_words_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    sentence=sentences[i],
-                    required_precision=required_precision,
-                    required_completeness=required_completeness,
-                    explanation=explanation[i] if explanation != None else None
+        """
+        with tracer.start_as_current_span(
+            "ValidationSetManager.create_select_words_set"
+        ):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
+
+            if len(datapoints) != len(truths) or len(datapoints) != len(sentences):
+                raise ValueError(
+                    "The number of datapoints, truths, and sentences must be equal"
+                )
+
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating select words rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.select_words_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        sentence=sentences[i],
+                        required_precision=required_precision,
+                        required_completeness=required_completeness,
+                        explanation=explanation[i] if explanation != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting select words rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting select words rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
-    def create_locate_set(self,
+    def create_locate_set(
+        self,
         name: str,
         instruction: str,
         truths: list[list[Box]],
         datapoints: list[str],
         contexts: list[str] | None = None,
+        media_contexts: list[str] | None = None,
         explanation: list[str | None] | None = None,
         dimensions: list[str] = [],
     ) -> RapidataValidationSet:
@@ -253,6 +332,9 @@ class ValidationSetManager:
                     truths: [[Box(0, 0, 100, 100)], [Box(50, 50, 150, 150)]] -> first datapoint the object is in the top left corner, second datapoint the object is in the center
             datapoints (list[str]): The datapoints that will be used for validation.
             contexts (list[str], optional): The contexts for each datapoint. Defaults to None.
+            media_contexts (list[str], optional): The list of media contexts i.e. links to the images / videos for the comparison. Defaults to None.\n
+                If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint)
+                Will be matched up with the datapoints using the list index.
             explanation (list[str | None], optional): The explanations for each datapoint. Will be given to the annotators in case the answer is wrong. Defaults to None.
             dimensions (list[str], optional): The dimensions to add to the validation set accross which users will be tracked. Defaults to [] which is the default dimension.
 
@@ -263,43 +345,56 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the object is in the top left corner, second datapoint the object is in the center
         """
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-        
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
- 
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The numeber of reasons and datapoints must be equal, the index must align, but can be padded with None")
-        
-        logger.debug("Creating locate rapids")
-        rapids = []
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapids.append(
-                self.rapid.locate_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    metadata=[PromptMetadata(contexts[i])] if contexts else [],
-                    explanation=explanation[i] if explanation != None else None
+        with tracer.start_as_current_span("ValidationSetManager.create_locate_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
 
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
+
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
+
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
                 )
-            )
-        
-        logger.debug("Submitting locate rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
-    
-    def create_draw_set(self,
+
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating locate rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.locate_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
+                )
+
+            logger.debug("Submitting locate rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+
+    def create_draw_set(
+        self,
         name: str,
         instruction: str,
         truths: list[list[Box]],
         datapoints: list[str],
         contexts: list[str] | None = None,
+        media_contexts: list[str] | None = None,
         explanation: list[str | None] | None = None,
         dimensions: list[str] = [],
     ) -> RapidataValidationSet:
@@ -314,6 +409,9 @@ class ValidationSetManager:
                     truths: [[Box(0, 0, 100, 100)], [Box(50, 50, 150, 150)]] -> first datapoint the object is in the top left corner, second datapoint the object is in the center
             datapoints (list[str]): The datapoints that will be used for validation.
             contexts (list[str], optional): The contexts for each datapoint. Defaults to None.
+            media_contexts (list[str], optional): The list of media contexts i.e. links to the images / videos for the comparison. Defaults to None.\n
+                If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint)
+                Will be matched up with the datapoints using the list index.
             explanation (list[str | None], optional): The explanations for each datapoint. Will be given to the annotators in case the answer is wrong. Defaults to None.
             dimensions (list[str], optional): The dimensions to add to the validation set accross which users will be tracked. Defaults to [] which is the default dimension.
 
@@ -324,42 +422,56 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the object is in the top left corner, second datapoint the object is in the center
         """
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-        
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
- 
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The numeber of reasons and datapoints must be equal, the index must align, but can be padded with None")
-
-        logger.debug("Creating draw rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapids.append(
-                self.rapid.draw_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    metadata=[PromptMetadata(contexts[i])] if contexts else [],
-                    explanation=explanation[i] if explanation != None else None
+        with tracer.start_as_current_span("ValidationSetManager.create_draw_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
 
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
+
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
+
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating draw rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.draw_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting draw rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting draw rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
-    def create_timestamp_set(self,
+    def create_timestamp_set(
+        self,
         name: str,
         instruction: str,
         truths: list[list[tuple[int, int]]],
         datapoints: list[str],
         contexts: list[str] | None = None,
+        media_contexts: list[str] | None = None,
         explanation: list[str | None] | None = None,
         dimensions: list[str] = [],
     ) -> RapidataValidationSet:
@@ -368,13 +480,16 @@ class ValidationSetManager:
         Args:
             name (str): The name of the validation set. (will not be shown to the labeler)
             instruction (str): The instruction to show to the labeler.
-            truths (list[list[tuple[int, int]]]): The truths for each datapoint defined as start and endpoint based on miliseconds. 
+            truths (list[list[tuple[int, int]]]): The truths for each datapoint defined as start and endpoint based on miliseconds.
                 Outer list is for each datapoint, inner list is for each truth.\n
                 example:
                     datapoints: ["datapoint1", "datapoint2"]
                     truths: [[(0, 10)], [(20, 30)]] -> first datapoint the correct interval is from 0 to 10, second datapoint the correct interval is from 20 to 30
             datapoints (list[str]): The datapoints that will be used for validation.
             contexts (list[str], optional): The contexts for each datapoint. Defaults to None.
+            media_contexts (list[str], optional): The list of media contexts i.e. links to the images / videos for the comparison. Defaults to None.\n
+                If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint)
+                Will be matched up with the datapoints using the list index.
             explanation (list[str | None], optional): The explanations for each datapoint. Will be given to the annotators in case the answer is wrong. Defaults to None.
             dimensions (list[str], optional): The dimensions to add to the validation set accross which users will be tracked. Defaults to [] which is the default dimension.
 
@@ -385,36 +500,50 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the correct interval is from 0 to 10, second datapoint the correct interval is from 20 to 30
         """
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-        
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
- 
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The numeber of reasons and datapoints must be equal, the index must align, but can be padded with None")
-              
-        logger.debug("Creating timestamp rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapids.append(
-                self.rapid.timestamp_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    metadata=[PromptMetadata(contexts[i])] if contexts else [],
-                    explanation=explanation[i] if explanation != None else None
+        with tracer.start_as_current_span("ValidationSetManager.create_timestamp_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
+
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
+
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
+
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
                 )
-            )
 
-        logger.debug("Submitting timestamp rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
-    
-    def create_mixed_set(self,
+            logger.debug("Creating timestamp rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.timestamp_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
+                )
+
+            logger.debug("Submitting timestamp rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+
+    def create_mixed_set(
+        self,
         name: str,
         rapids: list[Rapid],
         dimensions: list[str] = [],
@@ -426,20 +555,23 @@ class ValidationSetManager:
             rapids (list[Rapid]): The list of rapids to add to the validation set.
             dimensions (list[str], optional): The dimensions to add to the validation set accross which users will be tracked. Defaults to [] which is the default dimension.
         """
+        with tracer.start_as_current_span("ValidationSetManager.create_mixed_set"):
+            if not rapids:
+                raise ValueError("Rapids cannot be empty")
 
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
-    def _submit(self, name: str, rapids: list[Rapid], dimensions: list[str] | None) -> RapidataValidationSet:
+    def _submit(
+        self, name: str, rapids: list[Rapid], dimensions: list[str]
+    ) -> RapidataValidationSet:
         logger.debug("Creating validation set")
         validation_set_id = (
-            self.__openapi_service.validation_api.validation_set_post(
-                create_validation_set_model=CreateValidationSetModel(
-                    name=name
-                )
+            self._openapi_service.validation_api.validation_set_post(
+                create_validation_set_model=CreateValidationSetModel(name=name)
             )
         ).validation_set_id
 
-        logger.debug(f"Validation set created with ID: {validation_set_id}")
+        logger.debug("Validation set created with ID: %s", validation_set_id)
 
         if validation_set_id is None:
             raise ValueError("Failed to create validation set")
@@ -449,23 +581,159 @@ class ValidationSetManager:
         validation_set = RapidataValidationSet(
             name=name,
             validation_set_id=validation_set_id,
-            openapi_service=self.__openapi_service
+            dimensions=dimensions,
+            openapi_service=self._openapi_service,
         )
+        with tracer.start_as_current_span("Adding rapids to validation set"):
+            logger.debug("Adding rapids to validation set")
+            failed_rapids = []
+
+            progress_bar = tqdm(
+                total=len(rapids),
+                desc="Uploading validation tasks",
+                disable=rapidata_config.logging.silent_mode,
+            )
+
+            for rapid in rapids:
+                try:
+                    validation_set.add_rapid(rapid)
+                    progress_bar.update(1)
+                except Exception as e:
+                    logger.error(
+                        "Failed to add rapid %s to validation set.\nError: %s",
+                        rapid.asset,
+                        str(e),
+                    )
+                    failed_rapids.append(rapid.asset)
+
+            progress_bar.close()
+
+            if failed_rapids:
+                logger.error(
+                    "Failed to add %s datapoints to validation set: %s",
+                    len(failed_rapids),
+                    failed_rapids,
+                )
+                raise RuntimeError(
+                    f"Failed to add {len(failed_rapids)} datapoints to validation set: {failed_rapids}"
+                )
 
-        logger.debug("Adding rapids to validation set")
-        for rapid in tqdm(rapids, desc="Uploading validation tasks"):
-            validation_set.add_rapid(rapid)
-        
         managed_print()
-        managed_print(f"Validation set '{name}' created with ID {validation_set_id}\n",
-                f"Now viewable under: https://app.{self.__openapi_service.environment}/validation-set/detail/{validation_set_id}",
-                sep="")
-        
+        managed_print(
+            f"Validation set '{name}' created with ID {validation_set_id}\n",
+            f"Now viewable under: {validation_set.validation_set_details_page}",
+            sep="",
+        )
+
         if dimensions:
             validation_set.update_dimensions(dimensions)
-        
+
         return validation_set
-    
+
+    def _create_order_validation_set(
+        self,
+        workflow: Workflow,
+        name: str,
+        datapoints: list[Datapoint],
+        required_amount: int,
+        settings: Sequence[RapidataSetting] | None = None,
+        dimensions: list[str] = [],
+    ) -> RapidataValidationSet:
+        with tracer.start_as_current_span(
+            "ValidationSetManager._create_order_validation_set"
+        ):
+            rapids: list[Rapid] = []
+            for datapoint in workflow._format_datapoints(datapoints):
+                rapids.append(
+                    Rapid(
+                        asset=datapoint.asset,
+                        payload=workflow._to_payload(datapoint),
+                        context=datapoint.context,
+                        media_context=datapoint.media_context,
+                        data_type=datapoint.data_type,
+                        settings=settings,
+                    )
+                )
+            validation_set = RapidataValidationSet(
+                validation_set_id=self._openapi_service.validation_api.validation_set_post(
+                    create_validation_set_model=CreateValidationSetModel(name=name)
+                ).validation_set_id,
+                name=name,
+                dimensions=dimensions,
+                openapi_service=self._openapi_service,
+            )
+
+            managed_print()
+            managed_print(
+                Fore.YELLOW
+                + f"A new validation set was created. Please annotate {required_amount} datapoint{('s' if required_amount != 1 else '')} before the order can run."
+                + Fore.RESET
+            )
+
+            link = f"https://app.{self._openapi_service.environment}/validation-set/detail/{validation_set.id}/annotate?maxSize={len(datapoints)}&required={required_amount}"
+            could_open_browser = webbrowser.open(link)
+            if not could_open_browser:
+                encoded_url = urllib.parse.quote(link, safe="%/:=&?~#+!$,;'@()*[]")
+                managed_print(
+                    Fore.RED
+                    + f"Please open this URL in your browser to annotate the validation set: '{encoded_url}'"
+                    + Fore.RESET
+                )
+            else:
+                managed_print(
+                    Fore.YELLOW
+                    + f"Please annotate the validation set. \n'{link}'"
+                    + Fore.RESET
+                )
+
+            with tracer.start_as_current_span("Annotating validation set"):
+                progress_bar = tqdm(
+                    total=required_amount,
+                    desc="Annotate the validation set",
+                    disable=rapidata_config.logging.silent_mode,
+                )
+
+                rapid_index = 0
+                while True:
+                    total_rapids, labeled_rapids = (
+                        self._get_total_and_labeled_rapids_count(validation_set.id)
+                    )
+
+                    progress_bar.n = labeled_rapids
+                    progress_bar.refresh()
+
+                    if labeled_rapids >= required_amount:
+                        break
+
+                    if total_rapids < required_amount and rapid_index >= len(rapids):
+                        managed_print(
+                            Fore.RED
+                            + f"""Warning: An order can only be started with at least {required_amount} annotated validation tasks. But only {labeled_rapids}/{required_amount} were annotated.
+Either add clearer examples or turn off the 'autoValidationSetCreation' with:
+
+from rapidata import rapidata_config
+rapidata_config.order.autoValidationSetCreation = False"""
+                            + Fore.RESET
+                        )
+                        raise RuntimeError(
+                            f"Not enough rapids annotated. Required: {required_amount}, Annotated: {labeled_rapids}"
+                        )
+
+                    if (
+                        rapid_index < len(rapids)
+                        and total_rapids - labeled_rapids <= required_amount * 2
+                    ):
+                        validation_set.add_rapid(rapids[rapid_index])
+                        rapid_index += 1
+
+                    time.sleep(2)
+
+                progress_bar.close()
+
+            validation_set.update_dimensions(dimensions)
+
+            return validation_set
+
     def get_validation_set_by_id(self, validation_set_id: str) -> RapidataValidationSet:
         """Get a validation set by ID.
 
@@ -475,37 +743,71 @@ class ValidationSetManager:
         Returns:
             RapidataValidationSet: The ValidationSet instance.
         """
-        try:
-            validation_set = self.__openapi_service.validation_api.validation_set_validation_set_id_get(validation_set_id=validation_set_id)
-        except Exception:
-            raise ValueError(f"ValidationSet with ID {validation_set_id} not found.")
-        
-        return RapidataValidationSet(validation_set_id, str(validation_set.name), self.__openapi_service)
 
+        with tracer.start_as_current_span(
+            "ValidationSetManager.get_validation_set_by_id"
+        ):
+            logger.debug("Getting validation set by ID: %s", validation_set_id)
+            validation_set = self._openapi_service.validation_api.validation_set_validation_set_id_get(
+                validation_set_id=validation_set_id
+            )
+
+            return RapidataValidationSet(
+                validation_set_id,
+                str(validation_set.name),
+                validation_set.dimensions,
+                self._openapi_service,
+            )
 
-    def find_validation_sets(self, name: str = "", amount: int = 1) -> list[RapidataValidationSet]:
+    def find_validation_sets(
+        self, name: str = "", amount: int = 10
+    ) -> list[RapidataValidationSet]:
         """Find validation sets by name.
 
         Args:
             name (str, optional): The name to search for. Defaults to "" to match with any set.
-            amount (int, optional): The amount of validation sets to return. Defaults to 1.
+            amount (int, optional): The amount of validation sets to return. Defaults to 10.
 
         Returns:
             list[RapidataValidationSet]: The list of validation sets.
         """
-        try:
-            validation_page_result = self.__openapi_service.validation_api.validation_sets_get(QueryModel(
-                page=PageInfo(index=1, size=amount),
-                filter=RootFilter(filters=[Filter(field="Name", operator="Contains", value=name)]),
-                sortCriteria=[SortCriterion(direction="Desc", propertyName="CreatedAt")]
-                ))
+        with tracer.start_as_current_span("ValidationSetManager.find_validation_sets"):
+            logger.debug(
+                "Finding validation sets with name: %s and amount: %s", name, amount
+            )
+
+            validation_page_result = (
+                self._openapi_service.validation_api.validation_sets_get(
+                    QueryModel(
+                        page=PageInfo(index=1, size=amount),
+                        filter=RootFilter(
+                            filters=[
+                                Filter(
+                                    field="Name",
+                                    operator=FilterOperator.CONTAINS,
+                                    value=name,
+                                )
+                            ]
+                        ),
+                        sortCriteria=[
+                            SortCriterion(
+                                direction=SortDirection.DESC, propertyName="CreatedAt"
+                            )
+                        ],
+                    )
+                )
+            )
 
-        except BadRequestException as e:
-            raise ValueError(f"Error occured during request. \nError: {e.body} \nTraceid: {e.headers.get('X-Trace-Id') if isinstance(e.headers, HTTPHeaderDict) else 'Unknown'}")
+            logger.debug("Validation sets found: %s", validation_page_result.items)
 
-        except Exception as e:
-            raise ValueError(f"Unknown error occured: {e}")
+            validation_sets = [
+                self.get_validation_set_by_id(str(validation_set.id))
+                for validation_set in validation_page_result.items
+            ]
+            return validation_sets
 
-        validation_sets = [self.get_validation_set_by_id(str(validation_set.id)) for validation_set in validation_page_result.items]
-        return validation_sets
+    def __str__(self) -> str:
+        return "ValidationSetManager"
 
+    def __repr__(self) -> str:
+        return self.__str__()