rapidata 2.35.1__py3-none-any.whl → 3.0.0a8__py3-none-any.whl

rapidata/rapidata_client/validation/validation_set_manager.py

@@ -1,21 +1,43 @@
-from typing import Literal
+from __future__ import annotations
+
+import time
+import urllib.parse
+import webbrowser
+from colorama import Fore
+from typing import Literal, TYPE_CHECKING
 from rapidata.api_client import QueryModel
-from rapidata.rapidata_client.validation.rapidata_validation_set import RapidataValidationSet
-from rapidata.api_client.models.create_validation_set_model import CreateValidationSetModel
+from rapidata.rapidata_client.validation.rapidata_validation_set import (
+    RapidataValidationSet,
+)
+from rapidata.api_client.models.create_validation_set_model import (
+    CreateValidationSetModel,
+)
 from rapidata.service.openapi_service import OpenAPIService
 from rapidata.rapidata_client.validation.rapids.rapids_manager import RapidsManager
-from rapidata.rapidata_client.validation.rapids.rapids import Rapid
-from rapidata.rapidata_client.datapoints.metadata import PromptMetadata, MediaAssetMetadata
 
 from rapidata.api_client.models.page_info import PageInfo
 from rapidata.api_client.models.root_filter import RootFilter
 from rapidata.api_client.models.filter import Filter
 from rapidata.api_client.models.sort_criterion import SortCriterion
+from rapidata.api_client.models.sort_direction import SortDirection
+from rapidata.api_client.models.filter_operator import FilterOperator
 
 from rapidata.rapidata_client.validation.rapids.box import Box
 
-from rapidata.rapidata_client.logging import logger, managed_print, RapidataOutputManager
+from rapidata.rapidata_client.config import (
+    logger,
+    managed_print,
+    rapidata_config,
+    tracer,
+)
 from tqdm import tqdm
+from rapidata.rapidata_client.validation.rapids.rapids import Rapid
+
+if TYPE_CHECKING:
+    from rapidata.rapidata_client.workflow import Workflow
+    from rapidata.rapidata_client.datapoints._datapoint import Datapoint
+    from rapidata.rapidata_client.settings._rapidata_setting import RapidataSetting
+    from typing import Sequence
 
 
 class ValidationSetManager:
@@ -25,12 +47,22 @@ class ValidationSetManager:
     Attributes:
         rapid (RapidsManager): The RapidsManager instance.
     """
+
     def __init__(self, openapi_service: OpenAPIService) -> None:
-        self.__openapi_service = openapi_service
-        self.rapid = RapidsManager()
+        self._openapi_service = openapi_service
+        self.rapid = RapidsManager(openapi_service)
         logger.debug("ValidationSetManager initialized")
 
-    def create_classification_set(self,
+    def _get_total_and_labeled_rapids_count(
+        self, validation_set_id: str
+    ) -> tuple[int, int]:
+        uploaded_rapids = self._openapi_service.validation_api.validation_set_validation_set_id_rapids_get(
+            validation_set_id=validation_set_id
+        ).items
+        return len(uploaded_rapids), sum(1 for rapid in uploaded_rapids if rapid.truth)
+
+    def create_classification_set(
+        self,
         name: str,
         instruction: str,
         answer_options: list[str],
@@ -43,7 +75,7 @@ class ValidationSetManager:
         dimensions: list[str] = [],
     ) -> RapidataValidationSet:
         """Create a classification validation set.
-        
+
         Args:
             name (str): The name of the validation set. (will not be shown to the labeler)
             instruction (str): The instruction by which the labeler will answer.
@@ -72,49 +104,54 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint correct answer is "yes", second datapoint is "no" or "maybe"
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-        
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
-
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError("The number of media contexts and datapoints must be equal")
-
-        if(explanations and len(explanations) != len(datapoints)):
-            raise ValueError("The number of explanations and datapoints must be equal, the index must align, but can be padded with None")
-
-       
-        logger.debug("Creating classification rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.classification_rapid(
-                    instruction=instruction,
-                    answer_options=answer_options,
-                    datapoint=datapoints[i],
-                    truths=truths[i],
-                    data_type=data_type,
-                    metadata=rapid_metadata,
-                    explanation=explanations[i] if explanations != None else None
+        with tracer.start_as_current_span(
+            "ValidationSetManager.create_classification_set"
+        ):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
+
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
+
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
+
+            if explanations and len(explanations) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating classification rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.classification_rapid(
+                        instruction=instruction,
+                        answer_options=answer_options,
+                        datapoint=datapoints[i],
+                        truths=truths[i],
+                        data_type=data_type,
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanations[i] if explanations != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting classification rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting classification rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
-    def create_compare_set(self,
+    def create_compare_set(
+        self,
         name: str,
         instruction: str,
         datapoints: list[list[str]],
@@ -135,7 +172,7 @@ class ValidationSetManager:
                     instruction: "Which image has a cat?"
                     datapoints = [["image1.jpg", "image2.jpg"], ["image3.jpg", "image4.jpg"]]
                     truths: ["image1.jpg", "image4.jpg"] -> first comparison image1.jpg has a cat, second comparison image4.jpg has a cat
-            datapoints (list[list[str]]): The compare datapoints to create the validation set with. 
+            datapoints (list[list[str]]): The compare datapoints to create the validation set with.
                 Outer list is for each comparison, inner list the two images/texts that will be compared.
             data_type (str, optional): The type of data. Defaults to "media" (any form of image, video or audio). Other option: "text".
             contexts (list[str], optional): The contexts for each datapoint. Defaults to None.\n
@@ -155,47 +192,51 @@ class ValidationSetManager:
             ```
             This would mean: first comparison image1.jpg has a cat, second comparison image4.jpg has a cat
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, str) for truth in truths]):
-            raise ValueError("Truths must be a list of strings")
-
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
-
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError("The number of media contexts and datapoints must be equal")
- 
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The number of explanations and datapoints must be equal, the index must align, but can be padded with None")
-        
-        logger.debug("Creating comparison rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.compare_rapid(
-                    instruction=instruction,
-                    truth=truths[i],
-                    datapoint=datapoints[i],
-                    data_type=data_type,
-                    metadata=rapid_metadata,
-                    explanation=explanation[i] if explanation != None else None
+        with tracer.start_as_current_span("ValidationSetManager.create_compare_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, str) for truth in truths]):
+                raise ValueError("Truths must be a list of strings")
+
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
+
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
                 )
-            )
-        
-        logger.debug("Submitting comparison rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
-  
-    def create_select_words_set(self,
+
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating comparison rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.compare_rapid(
+                        instruction=instruction,
+                        truth=truths[i],
+                        datapoint=datapoints[i],
+                        data_type=data_type,
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
+                )
+
+            logger.debug("Submitting comparison rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+
+    def create_select_words_set(
+        self,
         name: str,
         instruction: str,
         truths: list[list[int]],
@@ -231,38 +272,46 @@ class ValidationSetManager:
             truths: [[0, 1], [2]]
             ```
             This would mean: first datapoint the correct words are "this" and "example", second datapoint is "with"
-            """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-
-        if len(datapoints) != len(truths) or len(datapoints) != len(sentences):
-            raise ValueError("The number of datapoints, truths, and sentences must be equal")
- 
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The number of explanations and datapoints must be equal, the index must align, but can be padded with None")
-
-        logger.debug("Creating select words rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapids.append(
-                self.rapid.select_words_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    sentence=sentences[i],
-                    required_precision=required_precision,
-                    required_completeness=required_completeness,
-                    explanation=explanation[i] if explanation != None else None
+        """
+        with tracer.start_as_current_span(
+            "ValidationSetManager.create_select_words_set"
+        ):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
+
+            if len(datapoints) != len(truths) or len(datapoints) != len(sentences):
+                raise ValueError(
+                    "The number of datapoints, truths, and sentences must be equal"
+                )
+
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating select words rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.select_words_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        sentence=sentences[i],
+                        required_precision=required_precision,
+                        required_completeness=required_completeness,
+                        explanation=explanation[i] if explanation != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting select words rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting select words rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
-    def create_locate_set(self,
+    def create_locate_set(
+        self,
         name: str,
         instruction: str,
         truths: list[list[Box]],
@@ -296,48 +345,50 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the object is in the top left corner, second datapoint the object is in the center
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-        
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
- 
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError("The number of media contexts and datapoints must be equal")
-
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The number of explanations and datapoints must be equal, the index must align, but can be padded with None")
-        
-        logger.debug("Creating locate rapids")
-        rapids = []
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.locate_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    metadata=rapid_metadata,
-                    explanation=explanation[i] if explanation != None else None
+        with tracer.start_as_current_span("ValidationSetManager.create_locate_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
 
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
+
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
+
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
                 )
-            )
-        
-        logger.debug("Submitting locate rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
-    
-    def create_draw_set(self,
+
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating locate rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.locate_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
+                )
+
+            logger.debug("Submitting locate rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+
+    def create_draw_set(
+        self,
         name: str,
         instruction: str,
         truths: list[list[Box]],
@@ -371,47 +422,50 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the object is in the top left corner, second datapoint the object is in the center
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-        
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
- 
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError("The number of media contexts and datapoints must be equal")
-
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The number of explanations and datapoints must be equal, the index must align, but can be padded with None")
-
-        logger.debug("Creating draw rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.draw_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    metadata=rapid_metadata,
-                    explanation=explanation[i] if explanation != None else None
+        with tracer.start_as_current_span("ValidationSetManager.create_draw_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
+
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
 
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
+
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
                 )
-            )
 
-        logger.debug("Submitting draw rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Creating draw rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.draw_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
+                )
+
+            logger.debug("Submitting draw rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
-    def create_timestamp_set(self,
+    def create_timestamp_set(
+        self,
         name: str,
         instruction: str,
         truths: list[list[tuple[int, int]]],
@@ -426,7 +480,7 @@ class ValidationSetManager:
         Args:
             name (str): The name of the validation set. (will not be shown to the labeler)
             instruction (str): The instruction to show to the labeler.
-            truths (list[list[tuple[int, int]]]): The truths for each datapoint defined as start and endpoint based on miliseconds. 
+            truths (list[list[tuple[int, int]]]): The truths for each datapoint defined as start and endpoint based on miliseconds.
                 Outer list is for each datapoint, inner list is for each truth.\n
                 example:
                     datapoints: ["datapoint1", "datapoint2"]
@@ -446,46 +500,50 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the correct interval is from 0 to 10, second datapoint the correct interval is from 20 to 30
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
-        
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
-        
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
-        
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
- 
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError("The number of media contexts and datapoints must be equal")
-
-        if(explanation and len(explanation) != len(datapoints)):
-            raise ValueError("The number of explanations and datapoints must be equal, the index must align, but can be padded with None")
-              
-        logger.debug("Creating timestamp rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.timestamp_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    metadata=rapid_metadata,
-                    explanation=explanation[i] if explanation != None else None
+        with tracer.start_as_current_span("ValidationSetManager.create_timestamp_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
+
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
+
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
+
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
+
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
                 )
-            )
 
-        logger.debug("Submitting timestamp rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
-    
-    def create_mixed_set(self,
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
+
+            logger.debug("Creating timestamp rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.timestamp_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
+                )
+
+            logger.debug("Submitting timestamp rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+
+    def create_mixed_set(
+        self,
         name: str,
         rapids: list[Rapid],
         dimensions: list[str] = [],
@@ -497,22 +555,23 @@ class ValidationSetManager:
             rapids (list[Rapid]): The list of rapids to add to the validation set.
             dimensions (list[str], optional): The dimensions to add to the validation set accross which users will be tracked. Defaults to [] which is the default dimension.
         """
-        if not rapids:
-            raise ValueError("Rapids cannot be empty")
+        with tracer.start_as_current_span("ValidationSetManager.create_mixed_set"):
+            if not rapids:
+                raise ValueError("Rapids cannot be empty")
 
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
-    def _submit(self, name: str, rapids: list[Rapid], dimensions: list[str] | None) -> RapidataValidationSet:
+    def _submit(
+        self, name: str, rapids: list[Rapid], dimensions: list[str]
+    ) -> RapidataValidationSet:
         logger.debug("Creating validation set")
         validation_set_id = (
-            self.__openapi_service.validation_api.validation_set_post(
-                create_validation_set_model=CreateValidationSetModel(
-                    name=name
-                )
+            self._openapi_service.validation_api.validation_set_post(
+                create_validation_set_model=CreateValidationSetModel(name=name)
             )
         ).validation_set_id
 
-        logger.debug(f"Validation set created with ID: {validation_set_id}")
+        logger.debug("Validation set created with ID: %s", validation_set_id)
 
         if validation_set_id is None:
             raise ValueError("Failed to create validation set")
@@ -522,31 +581,159 @@ class ValidationSetManager:
         validation_set = RapidataValidationSet(
             name=name,
             validation_set_id=validation_set_id,
-            openapi_service=self.__openapi_service
+            dimensions=dimensions,
+            openapi_service=self._openapi_service,
         )
+        with tracer.start_as_current_span("Adding rapids to validation set"):
+            logger.debug("Adding rapids to validation set")
+            failed_rapids = []
+
+            progress_bar = tqdm(
+                total=len(rapids),
+                desc="Uploading validation tasks",
+                disable=rapidata_config.logging.silent_mode,
+            )
 
-        logger.debug("Adding rapids to validation set")
-        failed_rapids = []
-        for rapid in tqdm(rapids, desc="Uploading validation tasks", disable=RapidataOutputManager.silent_mode):
-            try: 
-                validation_set.add_rapid(rapid)
-            except Exception:
-                failed_rapids.append(rapid.asset)
-
-        if failed_rapids:
-            logger.error(f"Failed to add {len(failed_rapids)} datapoints to validation set: {failed_rapids}")
-            raise RuntimeError(f"Failed to add {len(failed_rapids)} datapoints to validation set: {failed_rapids}")
+            for rapid in rapids:
+                try:
+                    validation_set.add_rapid(rapid)
+                    progress_bar.update(1)
+                except Exception as e:
+                    logger.error(
+                        "Failed to add rapid %s to validation set.\nError: %s",
+                        rapid.asset,
+                        str(e),
+                    )
+                    failed_rapids.append(rapid.asset)
+
+            progress_bar.close()
+
+            if failed_rapids:
+                logger.error(
+                    "Failed to add %s datapoints to validation set: %s",
+                    len(failed_rapids),
+                    failed_rapids,
+                )
+                raise RuntimeError(
+                    f"Failed to add {len(failed_rapids)} datapoints to validation set: {failed_rapids}"
+                )
 
         managed_print()
-        managed_print(f"Validation set '{name}' created with ID {validation_set_id}\n",
-                f"Now viewable under: https://app.{self.__openapi_service.environment}/validation-set/detail/{validation_set_id}",
-                sep="")
-        
+        managed_print(
+            f"Validation set '{name}' created with ID {validation_set_id}\n",
+            f"Now viewable under: {validation_set.validation_set_details_page}",
+            sep="",
+        )
+
         if dimensions:
             validation_set.update_dimensions(dimensions)
-        
+
         return validation_set
-    
+
+    def _create_order_validation_set(
+        self,
+        workflow: Workflow,
+        name: str,
+        datapoints: list[Datapoint],
+        required_amount: int,
+        settings: Sequence[RapidataSetting] | None = None,
+        dimensions: list[str] = [],
+    ) -> RapidataValidationSet:
+        with tracer.start_as_current_span(
+            "ValidationSetManager._create_order_validation_set"
+        ):
+            rapids: list[Rapid] = []
+            for datapoint in workflow._format_datapoints(datapoints):
+                rapids.append(
+                    Rapid(
+                        asset=datapoint.asset,
+                        payload=workflow._to_payload(datapoint),
+                        context=datapoint.context,
+                        media_context=datapoint.media_context,
+                        data_type=datapoint.data_type,
+                        settings=settings,
+                    )
+                )
+            validation_set = RapidataValidationSet(
+                validation_set_id=self._openapi_service.validation_api.validation_set_post(
+                    create_validation_set_model=CreateValidationSetModel(name=name)
+                ).validation_set_id,
+                name=name,
+                dimensions=dimensions,
+                openapi_service=self._openapi_service,
+            )
+
+            managed_print()
+            managed_print(
+                Fore.YELLOW
+                + f"A new validation set was created. Please annotate {required_amount} datapoint{('s' if required_amount != 1 else '')} before the order can run."
+                + Fore.RESET
+            )
+
+            link = f"https://app.{self._openapi_service.environment}/validation-set/detail/{validation_set.id}/annotate?maxSize={len(datapoints)}&required={required_amount}"
+            could_open_browser = webbrowser.open(link)
+            if not could_open_browser:
+                encoded_url = urllib.parse.quote(link, safe="%/:=&?~#+!$,;'@()*[]")
+                managed_print(
+                    Fore.RED
+                    + f"Please open this URL in your browser to annotate the validation set: '{encoded_url}'"
+                    + Fore.RESET
+                )
+            else:
+                managed_print(
+                    Fore.YELLOW
+                    + f"Please annotate the validation set. \n'{link}'"
+                    + Fore.RESET
+                )
+
+            with tracer.start_as_current_span("Annotating validation set"):
+                progress_bar = tqdm(
+                    total=required_amount,
+                    desc="Annotate the validation set",
+                    disable=rapidata_config.logging.silent_mode,
+                )
+
+                rapid_index = 0
+                while True:
+                    total_rapids, labeled_rapids = (
+                        self._get_total_and_labeled_rapids_count(validation_set.id)
+                    )
+
+                    progress_bar.n = labeled_rapids
+                    progress_bar.refresh()
+
+                    if labeled_rapids >= required_amount:
+                        break
+
+                    if total_rapids < required_amount and rapid_index >= len(rapids):
+                        managed_print(
+                            Fore.RED
+                            + f"""Warning: An order can only be started with at least {required_amount} annotated validation tasks. But only {labeled_rapids}/{required_amount} were annotated.
+Either add clearer examples or turn off the 'autoValidationSetCreation' with:
+
+from rapidata import rapidata_config
+rapidata_config.order.autoValidationSetCreation = False"""
+                            + Fore.RESET
+                        )
+                        raise RuntimeError(
+                            f"Not enough rapids annotated. Required: {required_amount}, Annotated: {labeled_rapids}"
+                        )
+
+                    if (
+                        rapid_index < len(rapids)
+                        and total_rapids - labeled_rapids <= required_amount * 2
+                    ):
+                        validation_set.add_rapid(rapids[rapid_index])
+                        rapid_index += 1
+
+                    time.sleep(2)
+
+                progress_bar.close()
+
+            validation_set.update_dimensions(dimensions)
+
+            return validation_set
+
     def get_validation_set_by_id(self, validation_set_id: str) -> RapidataValidationSet:
         """Get a validation set by ID.
 
@@ -556,29 +743,71 @@ class ValidationSetManager:
         Returns:
             RapidataValidationSet: The ValidationSet instance.
         """
-        
-        validation_set = self.__openapi_service.validation_api.validation_set_validation_set_id_get(validation_set_id=validation_set_id)
-        
-        return RapidataValidationSet(validation_set_id, str(validation_set.name), self.__openapi_service)
 
+        with tracer.start_as_current_span(
+            "ValidationSetManager.get_validation_set_by_id"
+        ):
+            logger.debug("Getting validation set by ID: %s", validation_set_id)
+            validation_set = self._openapi_service.validation_api.validation_set_validation_set_id_get(
+                validation_set_id=validation_set_id
+            )
+
+            return RapidataValidationSet(
+                validation_set_id,
+                str(validation_set.name),
+                validation_set.dimensions,
+                self._openapi_service,
+            )
 
-    def find_validation_sets(self, name: str = "", amount: int = 1) -> list[RapidataValidationSet]:
+    def find_validation_sets(
+        self, name: str = "", amount: int = 10
+    ) -> list[RapidataValidationSet]:
         """Find validation sets by name.
 
         Args:
             name (str, optional): The name to search for. Defaults to "" to match with any set.
-            amount (int, optional): The amount of validation sets to return. Defaults to 1.
+            amount (int, optional): The amount of validation sets to return. Defaults to 10.
 
         Returns:
             list[RapidataValidationSet]: The list of validation sets.
         """
+        with tracer.start_as_current_span("ValidationSetManager.find_validation_sets"):
+            logger.debug(
+                "Finding validation sets with name: %s and amount: %s", name, amount
+            )
+
+            validation_page_result = (
+                self._openapi_service.validation_api.validation_sets_get(
+                    QueryModel(
+                        page=PageInfo(index=1, size=amount),
+                        filter=RootFilter(
+                            filters=[
+                                Filter(
+                                    field="Name",
+                                    operator=FilterOperator.CONTAINS,
+                                    value=name,
+                                )
+                            ]
+                        ),
+                        sortCriteria=[
+                            SortCriterion(
+                                direction=SortDirection.DESC, propertyName="CreatedAt"
+                            )
+                        ],
+                    )
+                )
+            )
+
+            logger.debug("Validation sets found: %s", validation_page_result.items)
 
-        validation_page_result = self.__openapi_service.validation_api.validation_sets_get(QueryModel(
-            page=PageInfo(index=1, size=amount),
-            filter=RootFilter(filters=[Filter(field="Name", operator="Contains", value=name)]),
-            sortCriteria=[SortCriterion(direction="Desc", propertyName="CreatedAt")]
-            ))
+            validation_sets = [
+                self.get_validation_set_by_id(str(validation_set.id))
+                for validation_set in validation_page_result.items
+            ]
+            return validation_sets
 
-        validation_sets = [self.get_validation_set_by_id(str(validation_set.id)) for validation_set in validation_page_result.items]
-        return validation_sets
+    def __str__(self) -> str:
+        return "ValidationSetManager"
 
+    def __repr__(self) -> str:
+        return self.__str__()