rapidata 2.37.0__py3-none-any.whl → 3.0.0a8__py3-none-any.whl

rapidata/rapidata_client/validation/validation_set_manager.py

@@ -1,4 +1,10 @@
-from typing import Literal
+from __future__ import annotations
+
+import time
+import urllib.parse
+import webbrowser
+from colorama import Fore
+from typing import Literal, TYPE_CHECKING
 from rapidata.api_client import QueryModel
 from rapidata.rapidata_client.validation.rapidata_validation_set import (
     RapidataValidationSet,
@@ -8,11 +14,6 @@ from rapidata.api_client.models.create_validation_set_model import (
 )
 from rapidata.service.openapi_service import OpenAPIService
 from rapidata.rapidata_client.validation.rapids.rapids_manager import RapidsManager
-from rapidata.rapidata_client.validation.rapids.rapids import Rapid
-from rapidata.rapidata_client.datapoints.metadata import (
-    PromptMetadata,
-    MediaAssetMetadata,
-)
 
 from rapidata.api_client.models.page_info import PageInfo
 from rapidata.api_client.models.root_filter import RootFilter
@@ -23,17 +24,20 @@ from rapidata.api_client.models.filter_operator import FilterOperator
 
 from rapidata.rapidata_client.validation.rapids.box import Box
 
-from rapidata.rapidata_client.logging import (
+from rapidata.rapidata_client.config import (
     logger,
     managed_print,
-    RapidataOutputManager,
+    rapidata_config,
+    tracer,
 )
 from tqdm import tqdm
-from rapidata.rapidata_client.workflow import Workflow
-from rapidata.rapidata_client.datapoints._datapoint import Datapoint
 from rapidata.rapidata_client.validation.rapids.rapids import Rapid
-from rapidata.rapidata_client.settings._rapidata_setting import RapidataSetting
-from typing import Sequence
+
+if TYPE_CHECKING:
+    from rapidata.rapidata_client.workflow import Workflow
+    from rapidata.rapidata_client.datapoints._datapoint import Datapoint
+    from rapidata.rapidata_client.settings._rapidata_setting import RapidataSetting
+    from typing import Sequence
 
 
 class ValidationSetManager:
@@ -45,28 +49,17 @@ class ValidationSetManager:
     """
 
     def __init__(self, openapi_service: OpenAPIService) -> None:
-        self.__openapi_service = openapi_service
-        self.rapid = RapidsManager()
+        self._openapi_service = openapi_service
+        self.rapid = RapidsManager(openapi_service)
         logger.debug("ValidationSetManager initialized")
 
-    def _create_order_validation_set(
-        self,
-        workflow: Workflow,
-        order_name: str,
-        datapoints: list[Datapoint],
-        settings: Sequence[RapidataSetting] | None = None,
-    ) -> RapidataValidationSet:
-        rapids: list[Rapid] = []
-        for datapoint in datapoints:
-            rapids.append(
-                Rapid(
-                    asset=datapoint.asset,
-                    payload=workflow._to_payload(datapoint),
-                    metadata=datapoint.metadata,
-                    settings=settings,
-                )
-            )
-        return self._submit(name=order_name, rapids=rapids, dimensions=[])
+    def _get_total_and_labeled_rapids_count(
+        self, validation_set_id: str
+    ) -> tuple[int, int]:
+        uploaded_rapids = self._openapi_service.validation_api.validation_set_validation_set_id_rapids_get(
+            validation_set_id=validation_set_id
+        ).items
+        return len(uploaded_rapids), sum(1 for rapid in uploaded_rapids if rapid.truth)
 
     def create_classification_set(
         self,
@@ -111,50 +104,51 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint correct answer is "yes", second datapoint is "no" or "maybe"
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
+        with tracer.start_as_current_span(
+            "ValidationSetManager.create_classification_set"
+        ):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
 
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
 
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
 
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
 
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError(
-                "The number of media contexts and datapoints must be equal"
-            )
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
 
-        if explanations and len(explanations) != len(datapoints):
-            raise ValueError(
-                "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
-            )
+            if explanations and len(explanations) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
 
-        logger.debug("Creating classification rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.classification_rapid(
-                    instruction=instruction,
-                    answer_options=answer_options,
-                    datapoint=datapoints[i],
-                    truths=truths[i],
-                    data_type=data_type,
-                    metadata=rapid_metadata,
-                    explanation=explanations[i] if explanations != None else None,
+            logger.debug("Creating classification rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.classification_rapid(
+                        instruction=instruction,
+                        answer_options=answer_options,
+                        datapoint=datapoints[i],
+                        truths=truths[i],
+                        data_type=data_type,
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanations[i] if explanations != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting classification rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting classification rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
     def create_compare_set(
         self,
@@ -198,49 +192,48 @@ class ValidationSetManager:
             ```
             This would mean: first comparison image1.jpg has a cat, second comparison image4.jpg has a cat
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
+        with tracer.start_as_current_span("ValidationSetManager.create_compare_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
 
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
 
-        if not all([isinstance(truth, str) for truth in truths]):
-            raise ValueError("Truths must be a list of strings")
+            if not all([isinstance(truth, str) for truth in truths]):
+                raise ValueError("Truths must be a list of strings")
 
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
 
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError(
-                "The number of media contexts and datapoints must be equal"
-            )
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
 
-        if explanation and len(explanation) != len(datapoints):
-            raise ValueError(
-                "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
-            )
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
 
-        logger.debug("Creating comparison rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.compare_rapid(
-                    instruction=instruction,
-                    truth=truths[i],
-                    datapoint=datapoints[i],
-                    data_type=data_type,
-                    metadata=rapid_metadata,
-                    explanation=explanation[i] if explanation != None else None,
+            logger.debug("Creating comparison rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.compare_rapid(
+                        instruction=instruction,
+                        truth=truths[i],
+                        datapoint=datapoints[i],
+                        data_type=data_type,
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting comparison rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting comparison rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
     def create_select_words_set(
         self,
@@ -280,39 +273,42 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the correct words are "this" and "example", second datapoint is "with"
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
+        with tracer.start_as_current_span(
+            "ValidationSetManager.create_select_words_set"
+        ):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
 
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
 
-        if len(datapoints) != len(truths) or len(datapoints) != len(sentences):
-            raise ValueError(
-                "The number of datapoints, truths, and sentences must be equal"
-            )
+            if len(datapoints) != len(truths) or len(datapoints) != len(sentences):
+                raise ValueError(
+                    "The number of datapoints, truths, and sentences must be equal"
+                )
 
-        if explanation and len(explanation) != len(datapoints):
-            raise ValueError(
-                "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
-            )
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
 
-        logger.debug("Creating select words rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapids.append(
-                self.rapid.select_words_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    sentence=sentences[i],
-                    required_precision=required_precision,
-                    required_completeness=required_completeness,
-                    explanation=explanation[i] if explanation != None else None,
+            logger.debug("Creating select words rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.select_words_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        sentence=sentences[i],
+                        required_precision=required_precision,
+                        required_completeness=required_completeness,
+                        explanation=explanation[i] if explanation != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting select words rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting select words rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
     def create_locate_set(
         self,
@@ -349,49 +345,47 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the object is in the top left corner, second datapoint the object is in the center
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
+        with tracer.start_as_current_span("ValidationSetManager.create_locate_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
 
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
 
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
 
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
 
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError(
-                "The number of media contexts and datapoints must be equal"
-            )
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
 
-        if explanation and len(explanation) != len(datapoints):
-            raise ValueError(
-                "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
-            )
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
 
-        logger.debug("Creating locate rapids")
-        rapids = []
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.locate_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    metadata=rapid_metadata,
-                    explanation=explanation[i] if explanation != None else None,
+            logger.debug("Creating locate rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.locate_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting locate rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting locate rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
     def create_draw_set(
         self,
@@ -428,48 +422,47 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the object is in the top left corner, second datapoint the object is in the center
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
+        with tracer.start_as_current_span("ValidationSetManager.create_draw_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
 
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
 
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
 
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
 
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError(
-                "The number of media contexts and datapoints must be equal"
-            )
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
 
-        if explanation and len(explanation) != len(datapoints):
-            raise ValueError(
-                "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
-            )
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
 
-        logger.debug("Creating draw rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.draw_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    metadata=rapid_metadata,
-                    explanation=explanation[i] if explanation != None else None,
+            logger.debug("Creating draw rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.draw_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting draw rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting draw rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
     def create_timestamp_set(
         self,
@@ -507,48 +500,47 @@ class ValidationSetManager:
             ```
             This would mean: first datapoint the correct interval is from 0 to 10, second datapoint the correct interval is from 20 to 30
         """
-        if not datapoints:
-            raise ValueError("Datapoints cannot be empty")
+        with tracer.start_as_current_span("ValidationSetManager.create_timestamp_set"):
+            if not datapoints:
+                raise ValueError("Datapoints cannot be empty")
 
-        if len(datapoints) != len(truths):
-            raise ValueError("The number of datapoints and truths must be equal")
+            if len(datapoints) != len(truths):
+                raise ValueError("The number of datapoints and truths must be equal")
 
-        if not all([isinstance(truth, (list, tuple)) for truth in truths]):
-            raise ValueError("Truths must be a list of lists or tuples")
+            if not all([isinstance(truth, (list, tuple)) for truth in truths]):
+                raise ValueError("Truths must be a list of lists or tuples")
 
-        if contexts and len(contexts) != len(datapoints):
-            raise ValueError("The number of contexts and datapoints must be equal")
+            if contexts and len(contexts) != len(datapoints):
+                raise ValueError("The number of contexts and datapoints must be equal")
 
-        if media_contexts and len(media_contexts) != len(datapoints):
-            raise ValueError(
-                "The number of media contexts and datapoints must be equal"
-            )
+            if media_contexts and len(media_contexts) != len(datapoints):
+                raise ValueError(
+                    "The number of media contexts and datapoints must be equal"
+                )
 
-        if explanation and len(explanation) != len(datapoints):
-            raise ValueError(
-                "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
-            )
+            if explanation and len(explanation) != len(datapoints):
+                raise ValueError(
+                    "The number of explanations and datapoints must be equal, the index must align, but can be padded with None"
+                )
 
-        logger.debug("Creating timestamp rapids")
-        rapids: list[Rapid] = []
-        for i in range(len(datapoints)):
-            rapid_metadata = []
-            if contexts:
-                rapid_metadata.append(PromptMetadata(contexts[i]))
-            if media_contexts:
-                rapid_metadata.append(MediaAssetMetadata(media_contexts[i]))
-            rapids.append(
-                self.rapid.timestamp_rapid(
-                    instruction=instruction,
-                    truths=truths[i],
-                    datapoint=datapoints[i],
-                    metadata=rapid_metadata,
-                    explanation=explanation[i] if explanation != None else None,
+            logger.debug("Creating timestamp rapids")
+            rapids: list[Rapid] = []
+            for i in range(len(datapoints)):
+                rapids.append(
+                    self.rapid.timestamp_rapid(
+                        instruction=instruction,
+                        truths=truths[i],
+                        datapoint=datapoints[i],
+                        context=contexts[i] if contexts != None else None,
+                        media_context=(
+                            media_contexts[i] if media_contexts != None else None
+                        ),
+                        explanation=explanation[i] if explanation != None else None,
+                    )
                 )
-            )
 
-        logger.debug("Submitting timestamp rapids")
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            logger.debug("Submitting timestamp rapids")
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
     def create_mixed_set(
         self,
@@ -563,20 +555,18 @@ class ValidationSetManager:
             rapids (list[Rapid]): The list of rapids to add to the validation set.
             dimensions (list[str], optional): The dimensions to add to the validation set accross which users will be tracked. Defaults to [] which is the default dimension.
         """
-        if not rapids:
-            raise ValueError("Rapids cannot be empty")
+        with tracer.start_as_current_span("ValidationSetManager.create_mixed_set"):
+            if not rapids:
+                raise ValueError("Rapids cannot be empty")
 
-        return self._submit(name=name, rapids=rapids, dimensions=dimensions)
+            return self._submit(name=name, rapids=rapids, dimensions=dimensions)
 
     def _submit(
-        self,
-        name: str,
-        rapids: list[Rapid],
-        dimensions: list[str] | None,
+        self, name: str, rapids: list[Rapid], dimensions: list[str]
     ) -> RapidataValidationSet:
         logger.debug("Creating validation set")
         validation_set_id = (
-            self.__openapi_service.validation_api.validation_set_post(
+            self._openapi_service.validation_api.validation_set_post(
                 create_validation_set_model=CreateValidationSetModel(name=name)
             )
         ).validation_set_id
@@ -591,31 +581,43 @@ class ValidationSetManager:
         validation_set = RapidataValidationSet(
             name=name,
             validation_set_id=validation_set_id,
-            openapi_service=self.__openapi_service,
+            dimensions=dimensions,
+            openapi_service=self._openapi_service,
         )
-
-        logger.debug("Adding rapids to validation set")
-        failed_rapids = []
-        for rapid in tqdm(
-            rapids,
-            desc="Uploading validation tasks",
-            disable=RapidataOutputManager.silent_mode,
-        ):
-            try:
-                validation_set.add_rapid(rapid)
-            except Exception:
-                failed_rapids.append(rapid.asset)
-
-        if failed_rapids:
-            logger.error(
-                "Failed to add %s datapoints to validation set: %s",
-                len(failed_rapids),
-                failed_rapids,
-            )
-            raise RuntimeError(
-                f"Failed to add {len(failed_rapids)} datapoints to validation set: {failed_rapids}"
+        with tracer.start_as_current_span("Adding rapids to validation set"):
+            logger.debug("Adding rapids to validation set")
+            failed_rapids = []
+
+            progress_bar = tqdm(
+                total=len(rapids),
+                desc="Uploading validation tasks",
+                disable=rapidata_config.logging.silent_mode,
             )
 
+            for rapid in rapids:
+                try:
+                    validation_set.add_rapid(rapid)
+                    progress_bar.update(1)
+                except Exception as e:
+                    logger.error(
+                        "Failed to add rapid %s to validation set.\nError: %s",
+                        rapid.asset,
+                        str(e),
+                    )
+                    failed_rapids.append(rapid.asset)
+
+            progress_bar.close()
+
+            if failed_rapids:
+                logger.error(
+                    "Failed to add %s datapoints to validation set: %s",
+                    len(failed_rapids),
+                    failed_rapids,
+                )
+                raise RuntimeError(
+                    f"Failed to add {len(failed_rapids)} datapoints to validation set: {failed_rapids}"
+                )
+
         managed_print()
         managed_print(
             f"Validation set '{name}' created with ID {validation_set_id}\n",
@@ -628,6 +630,110 @@ class ValidationSetManager:
 
         return validation_set
 
+    def _create_order_validation_set(
+        self,
+        workflow: Workflow,
+        name: str,
+        datapoints: list[Datapoint],
+        required_amount: int,
+        settings: Sequence[RapidataSetting] | None = None,
+        dimensions: list[str] = [],
+    ) -> RapidataValidationSet:
+        with tracer.start_as_current_span(
+            "ValidationSetManager._create_order_validation_set"
+        ):
+            rapids: list[Rapid] = []
+            for datapoint in workflow._format_datapoints(datapoints):
+                rapids.append(
+                    Rapid(
+                        asset=datapoint.asset,
+                        payload=workflow._to_payload(datapoint),
+                        context=datapoint.context,
+                        media_context=datapoint.media_context,
+                        data_type=datapoint.data_type,
+                        settings=settings,
+                    )
+                )
+            validation_set = RapidataValidationSet(
+                validation_set_id=self._openapi_service.validation_api.validation_set_post(
+                    create_validation_set_model=CreateValidationSetModel(name=name)
+                ).validation_set_id,
+                name=name,
+                dimensions=dimensions,
+                openapi_service=self._openapi_service,
+            )
+
+            managed_print()
+            managed_print(
+                Fore.YELLOW
+                + f"A new validation set was created. Please annotate {required_amount} datapoint{('s' if required_amount != 1 else '')} before the order can run."
+                + Fore.RESET
+            )
+
+            link = f"https://app.{self._openapi_service.environment}/validation-set/detail/{validation_set.id}/annotate?maxSize={len(datapoints)}&required={required_amount}"
+            could_open_browser = webbrowser.open(link)
+            if not could_open_browser:
+                encoded_url = urllib.parse.quote(link, safe="%/:=&?~#+!$,;'@()*[]")
+                managed_print(
+                    Fore.RED
+                    + f"Please open this URL in your browser to annotate the validation set: '{encoded_url}'"
+                    + Fore.RESET
+                )
+            else:
+                managed_print(
+                    Fore.YELLOW
+                    + f"Please annotate the validation set. \n'{link}'"
+                    + Fore.RESET
+                )
+
+            with tracer.start_as_current_span("Annotating validation set"):
+                progress_bar = tqdm(
+                    total=required_amount,
+                    desc="Annotate the validation set",
+                    disable=rapidata_config.logging.silent_mode,
+                )
+
+                rapid_index = 0
+                while True:
+                    total_rapids, labeled_rapids = (
+                        self._get_total_and_labeled_rapids_count(validation_set.id)
+                    )
+
+                    progress_bar.n = labeled_rapids
+                    progress_bar.refresh()
+
+                    if labeled_rapids >= required_amount:
+                        break
+
+                    if total_rapids < required_amount and rapid_index >= len(rapids):
+                        managed_print(
+                            Fore.RED
+                            + f"""Warning: An order can only be started with at least {required_amount} annotated validation tasks. But only {labeled_rapids}/{required_amount} were annotated.
+Either add clearer examples or turn off the 'autoValidationSetCreation' with:
+
+from rapidata import rapidata_config
+rapidata_config.order.autoValidationSetCreation = False"""
+                            + Fore.RESET
+                        )
+                        raise RuntimeError(
+                            f"Not enough rapids annotated. Required: {required_amount}, Annotated: {labeled_rapids}"
+                        )
+
+                    if (
+                        rapid_index < len(rapids)
+                        and total_rapids - labeled_rapids <= required_amount * 2
+                    ):
+                        validation_set.add_rapid(rapids[rapid_index])
+                        rapid_index += 1
+
+                    time.sleep(2)
+
+                progress_bar.close()
+
+            validation_set.update_dimensions(dimensions)
+
+            return validation_set
+
     def get_validation_set_by_id(self, validation_set_id: str) -> RapidataValidationSet:
         """Get a validation set by ID.
 
@@ -638,53 +744,70 @@ class ValidationSetManager:
             RapidataValidationSet: The ValidationSet instance.
         """
 
-        validation_set = (
-            self.__openapi_service.validation_api.validation_set_validation_set_id_get(
+        with tracer.start_as_current_span(
+            "ValidationSetManager.get_validation_set_by_id"
+        ):
+            logger.debug("Getting validation set by ID: %s", validation_set_id)
+            validation_set = self._openapi_service.validation_api.validation_set_validation_set_id_get(
                 validation_set_id=validation_set_id
             )
-        )
 
-        return RapidataValidationSet(
-            validation_set_id, str(validation_set.name), self.__openapi_service
-        )
+            return RapidataValidationSet(
+                validation_set_id,
+                str(validation_set.name),
+                validation_set.dimensions,
+                self._openapi_service,
+            )
 
     def find_validation_sets(
-        self, name: str = "", amount: int = 1
+        self, name: str = "", amount: int = 10
     ) -> list[RapidataValidationSet]:
         """Find validation sets by name.
 
         Args:
             name (str, optional): The name to search for. Defaults to "" to match with any set.
-            amount (int, optional): The amount of validation sets to return. Defaults to 1.
+            amount (int, optional): The amount of validation sets to return. Defaults to 10.
 
         Returns:
             list[RapidataValidationSet]: The list of validation sets.
         """
+        with tracer.start_as_current_span("ValidationSetManager.find_validation_sets"):
+            logger.debug(
+                "Finding validation sets with name: %s and amount: %s", name, amount
+            )
 
-        validation_page_result = (
-            self.__openapi_service.validation_api.validation_sets_get(
-                QueryModel(
-                    page=PageInfo(index=1, size=amount),
-                    filter=RootFilter(
-                        filters=[
-                            Filter(
-                                field="Name",
-                                operator=FilterOperator.CONTAINS,
-                                value=name,
+            validation_page_result = (
+                self._openapi_service.validation_api.validation_sets_get(
+                    QueryModel(
+                        page=PageInfo(index=1, size=amount),
+                        filter=RootFilter(
+                            filters=[
+                                Filter(
+                                    field="Name",
+                                    operator=FilterOperator.CONTAINS,
+                                    value=name,
+                                )
+                            ]
+                        ),
+                        sortCriteria=[
+                            SortCriterion(
+                                direction=SortDirection.DESC, propertyName="CreatedAt"
                             )
-                        ]
-                    ),
-                    sortCriteria=[
-                        SortCriterion(
-                            direction=SortDirection.DESC, propertyName="CreatedAt"
-                        )
-                    ],
+                        ],
+                    )
                 )
             )
-        )
 
-        validation_sets = [
-            self.get_validation_set_by_id(str(validation_set.id))
-            for validation_set in validation_page_result.items
-        ]
-        return validation_sets
+            logger.debug("Validation sets found: %s", validation_page_result.items)
+
+            validation_sets = [
+                self.get_validation_set_by_id(str(validation_set.id))
+                for validation_set in validation_page_result.items
+            ]
+            return validation_sets
+
+    def __str__(self) -> str:
+        return "ValidationSetManager"
+
+    def __repr__(self) -> str:
+        return self.__str__()