opik 1.8.101__tar.gz → 1.9.33__tar.gz

{opik-1.8.101/src/opik.egg-info → opik-1.9.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik
-Version: 1.8.101
+Version: 1.9.33
 Summary: Comet tool for logging and evaluating LLM traces
 Home-page: https://www.comet.com
 Author: Comet ML Inc.
@@ -27,7 +27,7 @@ Requires-Dist: boto3-stubs[bedrock-runtime]>=1.34.110
 Requires-Dist: click
 Requires-Dist: httpx
 Requires-Dist: rapidfuzz<4.0.0,>=3.0.0
-Requires-Dist: litellm!=1.75.0,!=1.75.1,!=1.75.2,!=1.75.3,!=1.75.4,!=1.75.5,<=1.77.1
+Requires-Dist: litellm!=1.75.0,!=1.75.1,!=1.75.2,!=1.75.3,!=1.75.4,!=1.75.5,!=1.77.3,!=1.77.4,!=1.77.5,!=1.77.7,!=1.78.0,!=1.78.2,!=1.78.3,!=1.78.4,!=1.78.5,!=1.78.6,!=1.78.7,!=1.79.0,!=1.79.1,>=1.79.2
 Requires-Dist: openai
 Requires-Dist: pydantic-settings!=2.9.0,<3.0.0,>=2.0.0
 Requires-Dist: pydantic<3.0.0,>=2.0.0
@@ -157,7 +157,7 @@ Deploy Opik in your own environment. Choose between Docker for local setups or K
 
 This is the simplest way to get a local Opik instance running. Note the new `./opik.sh` installation script:
 
-On Linux or Mac Enviroment:
+On Linux or Mac Environment:
 
 ```bash
 # Clone the Opik repository
@@ -170,7 +170,7 @@ cd opik
 ./opik.sh
 ```
 
-On Windows Enviroment:
+On Windows Environment:
 
 ```powershell
 # Clone the Opik repository

{opik-1.8.101 → opik-1.9.33}/setup.py

@@ -44,11 +44,12 @@ setup(
         "rapidfuzz>=3.0.0,<4.0.0",
         # LiteLLM dependency comments:
         # - Exclude litellm 1.75.0-1.75.5 (broken callbacks system)
-        # - Cap at 1.77.4: version 1.77.5+ removes trace_id/parent_span_id passthrough
-        #   Can be removed after this PR is released: https://github.com/BerriAI/litellm/pull/15529
-        # - Cap at 1.77.1: version 1.77.2+ introduces C++ compiler dependency. It's already fixed in litellm
-        #   but we can't remove the cap because we still have to cap at 1.77.4.
-        "litellm<=1.77.1,!=1.75.0,!=1.75.1,!=1.75.2,!=1.75.3,!=1.75.4,!=1.75.5",
+        # - Exclude versions 1.77.2-1.77.4: introduce C++ compiler dependency (madoka), fixed in 1.77.5
+        #   See: https://github.com/BerriAI/litellm/issues/14762
+        # - Exclude versions 1.77.5-1.79.1: remove trace_id/parent_span_id passthrough, fixed in 1.79.2+
+        #   See: https://github.com/BerriAI/litellm/pull/15529
+        # Please keep this list in sync with the one in sdks/opik_optimizer/pyproject.toml
+        "litellm>=1.79.2,!=1.75.0,!=1.75.1,!=1.75.2,!=1.75.3,!=1.75.4,!=1.75.5,!=1.77.3,!=1.77.4,!=1.77.5,!=1.77.7,!=1.78.0,!=1.78.2,!=1.78.3,!=1.78.4,!=1.78.5,!=1.78.6,!=1.78.7,!=1.79.0,!=1.79.1",
         "openai",
         "pydantic-settings>=2.0.0,<3.0.0,!=2.9.0",
         "pydantic>=2.0.0,<3.0.0",

{opik-1.8.101 → opik-1.9.33}/src/opik/__init__.py

@@ -12,12 +12,17 @@ from .api_objects.span import Span
 from .api_objects.trace import Trace
 from .configurator.configure import configure
 from .decorator.tracker import flush_tracker, track
-from .evaluation import evaluate, evaluate_experiment, evaluate_prompt
+from .evaluation import (
+    evaluate,
+    evaluate_experiment,
+    evaluate_on_dict_items,
+    evaluate_prompt,
+)
 from .integrations.sagemaker import auth as sagemaker_auth
 from .plugins.pytest.decorator import llm_unit
 from .types import LLMProvider
 from . import opik_context
-from .decorator.tracing_runtime_config import (
+from .tracing_runtime_config import (
     is_tracing_active,
     reset_tracing_to_config_default,
     set_tracing_active,
@@ -37,6 +42,7 @@ __all__ = [
     "evaluate",
     "evaluate_prompt",
     "evaluate_experiment",
+    "evaluate_on_dict_items",
     "ExperimentItemContent",
     "ExperimentItemReferences",
     "track",

opik-1.9.33/src/opik/anonymizer/__init__.py

@@ -0,0 +1,5 @@
+from .anonymizer import Anonymizer
+from .factory import create_anonymizer
+from .recursive_anonymizer import RecursiveAnonymizer
+
+__all__ = ["Anonymizer", "create_anonymizer", "RecursiveAnonymizer"]

opik-1.9.33/src/opik/anonymizer/anonymizer.py

@@ -0,0 +1,12 @@
+import abc
+from typing import Dict, Any, Union, List
+
+AnonymizerDataType = Union[Dict[str, Any], str, List[Any]]
+
+
+class Anonymizer(abc.ABC):
+    """Abstract base class for anonymizing sensitive data in various data structures."""
+
+    @abc.abstractmethod
+    def anonymize(self, data: AnonymizerDataType, **kwargs: Any) -> AnonymizerDataType:
+        pass

opik-1.9.33/src/opik/anonymizer/factory.py

@@ -0,0 +1,80 @@
+from typing import Union, List, Dict, Callable, Tuple
+
+from . import anonymizer, rules_anonymizer, rules
+
+RulesType = Union[
+    List[Dict[str, str]],
+    List[Tuple[str, str]],
+    List[Callable[[str], str]],
+    List[Union[Dict[str, str], Tuple[str, str], Callable[[str], str]]],
+    Dict[str, str],
+    Tuple[str, str],
+    Callable[[str], str],
+]
+
+
+def create_anonymizer(
+    anonymizer_rules: RulesType, max_depth: int = 10
+) -> anonymizer.Anonymizer:
+    """Create an anonymizer with the specified rules.
+
+    Args:
+        anonymizer_rules: Anonymizer rules specification in various formats:
+            - Dict with "regex" and "replace" keys for a single regex rule
+            - Tuple with (regex, replacement) for a single regex rule
+            - Callable that takes a string and returns anonymized string
+            - List of any of the above for multiple rules
+        max_depth: Maximum recursion depth for nested data structures.
+
+    Returns:
+        An Anonymizer instance configured with the specified rules.
+
+    Raises:
+        ValueError: If a rule format is invalid.
+    """
+    rule_objects: List[rules.Rule] = []
+
+    if callable(anonymizer_rules):
+        # Single function rule
+        rule_objects.append(rules.FunctionRule(anonymizer_rules))
+    elif isinstance(anonymizer_rules, dict):
+        # Single dictionary rule
+        _check_dictionary_rule(anonymizer_rules)
+        rule_objects.append(
+            rules.RegexRule(anonymizer_rules["regex"], anonymizer_rules["replace"])
+        )
+    elif isinstance(anonymizer_rules, tuple):
+        # Single tuple rule
+        _check_tuple_rule(anonymizer_rules)
+        regex_pattern, replacement = anonymizer_rules
+        rule_objects.append(rules.RegexRule(regex_pattern, replacement))
+    elif isinstance(anonymizer_rules, list):
+        # List of rules
+        for rule in anonymizer_rules:
+            if callable(rule) and not isinstance(rule, (dict, tuple)):
+                rule_objects.append(rules.FunctionRule(rule))
+            elif isinstance(rule, dict):
+                _check_dictionary_rule(rule)
+                rule_objects.append(rules.RegexRule(rule["regex"], rule["replace"]))
+            elif isinstance(rule, tuple):
+                _check_tuple_rule(rule)
+                regex_pattern, replacement = rule
+                rule_objects.append(rules.RegexRule(regex_pattern, replacement))
+            else:
+                raise ValueError(f"Unsupported rule type in list: {type(rule)}")
+    else:
+        raise ValueError(f"Unsupported rules type: {type(anonymizer_rules)}")
+
+    return rules_anonymizer.RulesAnonymizer(rule_objects, max_depth=max_depth)
+
+
+def _check_dictionary_rule(rule: Dict[str, str]) -> None:
+    if "regex" not in rule or "replace" not in rule:
+        raise ValueError("Dictionary rule must have 'regex' and 'replace' keys")
+
+
+def _check_tuple_rule(rule: Tuple[str, str]) -> None:
+    if len(rule) != 2:
+        raise ValueError(
+            "Tuple rule must have exactly 2 elements: (regex, replacement)"
+        )

opik-1.9.33/src/opik/anonymizer/recursive_anonymizer.py

@@ -0,0 +1,64 @@
+import abc
+from typing import Any, Optional
+
+from . import anonymizer
+
+
+class RecursiveAnonymizer(anonymizer.Anonymizer):
+    """Abstract base class for anonymizing sensitive data in various data structures.
+
+    This class provides a framework for recursively anonymizing text content within
+    nested data structures such as dictionaries, lists, and strings. Subclasses must
+    implement the anonymize_text() method to define the specific anonymization logic.
+    """
+
+    def __init__(self, max_depth: int = 10):
+        """Initialize the Anonymizer with depth limiting.
+
+        Args:
+            max_depth: Maximum recursion depth to prevent infinite loops when
+                      processing deeply nested or circular data structures.
+                      Defaults to 10.
+        """
+        self.max_depth = max_depth
+
+    def anonymize(
+        self, data: anonymizer.AnonymizerDataType, **kwargs: Any
+    ) -> anonymizer.AnonymizerDataType:
+        return self._recursive_anonymize(data, **kwargs)
+
+    @abc.abstractmethod
+    def anonymize_text(self, data: str, **kwargs: Any) -> str:
+        pass
+
+    def _recursive_anonymize(
+        self,
+        data: anonymizer.AnonymizerDataType,
+        depth: int = 0,
+        field_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> anonymizer.AnonymizerDataType:
+        if depth >= self.max_depth:
+            return data
+
+        if field_name is None:
+            field_name = ""
+
+        if isinstance(data, str):
+            return self.anonymize_text(data, field_name=field_name, **kwargs)
+        elif isinstance(data, dict):
+            return {
+                key: self._recursive_anonymize(
+                    value, depth + 1, field_name=f"{field_name}.{key}", **kwargs
+                )
+                for key, value in data.items()
+            }
+        elif isinstance(data, list):
+            return [
+                self._recursive_anonymize(
+                    item, depth + 1, field_name=f"{field_name}.{i}", **kwargs
+                )
+                for i, item in enumerate(data)
+            ]
+        else:
+            return data

opik-1.9.33/src/opik/anonymizer/rules.py

@@ -0,0 +1,56 @@
+import abc
+import re
+from typing import Callable
+
+
+class Rule(abc.ABC):
+    """Abstract base class for text anonymization rules.
+
+    Rules define specific patterns or conditions for anonymizing sensitive
+    information in text. Subclasses must implement the apply() method to
+    define the anonymization logic.
+    """
+
+    @abc.abstractmethod
+    def apply(self, text: str) -> str:
+        pass
+
+
+class RegexRule(Rule):
+    """A rule that uses regular expressions to find and replace patterns in text.
+
+    This rule compiles a regular expression pattern and applies it to input text,
+    replacing all matches with a specified replacement string.
+    """
+
+    def __init__(self, regex: str, replacement: str):
+        """Initialize the regex rule with a pattern and replacement.
+
+        Args:
+            regex: Regular expression pattern to match sensitive data.
+            replacement: String to replace matched patterns with.
+        """
+        self.pattern = re.compile(regex)
+        self.replacement = replacement
+
+    def apply(self, text: str) -> str:
+        return self.pattern.sub(self.replacement, text)
+
+
+class FunctionRule(Rule):
+    """A rule that applies a custom function to anonymize text.
+
+    This rule allows for flexible anonymization by accepting any callable
+    that takes a string as input and returns an anonymized string.
+    """
+
+    def __init__(self, func: Callable[[str], str]):
+        """Initialize the function rule with a custom anonymization function.
+
+        Args:
+            func: A callable that takes a string and returns an anonymized version.
+        """
+        self.func = func
+
+    def apply(self, text: str) -> str:
+        return self.func(text)

opik-1.9.33/src/opik/anonymizer/rules_anonymizer.py

@@ -0,0 +1,35 @@
+from typing import List, Any
+
+from . import recursive_anonymizer, rules
+
+
+class RulesAnonymizer(recursive_anonymizer.RecursiveAnonymizer):
+    """An anonymizer that applies a list of rules sequentially to text data.
+
+    This class takes a list of Rule objects and applies them to
+    anonymize sensitive information in text.
+    """
+
+    def __init__(self, anonymizer_rules: List[rules.Rule], max_depth: int = 10):
+        """Initialize the RulesAnonymizer with a list of rules.
+
+        Args:
+            anonymizer_rules: List of Rule objects to apply for anonymization.
+            max_depth: Maximum recursion depth for nested data structures.
+        """
+        super().__init__(max_depth)
+        self.rules = anonymizer_rules
+
+    def anonymize_text(self, data: str, **kwargs: Any) -> str:
+        """Apply all rules sequentially to the input text.
+
+        Args:
+            data: The text to anonymize.
+
+        Returns:
+            The anonymized text after applying all rules.
+        """
+        result = data
+        for rule in self.rules:
+            result = rule.apply(result)
+        return result

{opik-1.8.101 → opik-1.9.33}/src/opik/api_objects/dataset/rest_operations.py

@@ -4,6 +4,7 @@ import opik.exceptions as exceptions
 from opik.message_processing import streamer
 from . import dataset
 from .. import experiment
+from ..experiment import experiments_client
 from ...rest_api.core.api_error import ApiError
 
 
@@ -60,6 +61,7 @@ def get_dataset_experiments(
     dataset_id: str,
     max_results: int,
     streamer: streamer.Streamer,
+    experiments_client: experiments_client.ExperimentsClient,
 ) -> List[experiment.Experiment]:
     page_size = 100
     experiments: List[experiment.Experiment] = []
@@ -83,6 +85,7 @@ def get_dataset_experiments(
                     dataset_name=experiment_.dataset_name,
                     rest_client=rest_client,
                     streamer=streamer,
+                    experiments_client=experiments_client,
                 )
             )
 

{opik-1.8.101 → opik-1.9.33}/src/opik/api_objects/experiment/experiment.py

@@ -2,13 +2,12 @@ import functools
 import logging
 from typing import List, Optional
 
-import opik.rest_api
 from opik.message_processing.batching import sequence_splitter
 from opik.message_processing import messages, streamer
 from opik.rest_api import client as rest_api_client
 from opik.rest_api.types import experiment_public
-from . import experiment_item
-from .. import constants, helpers, rest_stream_parser
+from . import experiment_item, experiments_client
+from .. import constants, helpers
 from ...api_objects.prompt import Prompt
 
 LOGGER = logging.getLogger(__name__)
@@ -22,6 +21,7 @@ class Experiment:
         dataset_name: str,
         rest_client: rest_api_client.OpikApi,
         streamer: streamer.Streamer,
+        experiments_client: experiments_client.ExperimentsClient,
         prompts: Optional[List[Prompt]] = None,
     ) -> None:
         self._id = id
@@ -30,6 +30,7 @@ class Experiment:
         self._rest_client = rest_client
         self._prompts = prompts
         self._streamer = streamer
+        self._experiments_client = experiments_client
 
     @property
     def id(self) -> str:
@@ -100,55 +101,25 @@ class Experiment:
 
     def get_items(
         self,
-        max_results: Optional[int] = None,
+        max_results: Optional[int] = 10000,
         truncate: bool = False,
     ) -> List[experiment_item.ExperimentItemContent]:
         """
-        Retrieves and returns a list of experiment items by streaming from the backend in batches, with an option to
-        truncate the results for each batch.
-
-        This method streams experiment items from a backend service in chunks up to the specified `max_results`
-        or until the available items are exhausted. It handles batch-wise retrieval and parsing, ensuring the client
-        receives a list of `ExperimentItemContent` objects, while respecting the constraints on maximum retrieval size
-        from the backend. If truncation is enabled, the backend may return truncated details for each item.
+        Retrieves and returns a list of experiment items for this experiment.
 
         Args:
-            max_results: Maximum number of experiment items to retrieve.
-            truncate: Whether to truncate the items returned by the backend.
+            max_results: Maximum number of experiment items to retrieve. Defaults to 10000 if not specified.
+            truncate: Whether to truncate the items returned by the backend. Defaults to False.
 
+        Returns:
+            List of ExperimentItemContent objects for this experiment.
         """
-        result: List[experiment_item.ExperimentItemContent] = []
-        max_endpoint_batch_size = rest_stream_parser.MAX_ENDPOINT_BATCH_SIZE
-
-        while True:
-            if max_results is None:
-                current_batch_size = max_endpoint_batch_size
-            else:
-                current_batch_size = min(
-                    max_results - len(result), max_endpoint_batch_size
-                )
-
-            items_stream = self._rest_client.experiments.stream_experiment_items(
-                experiment_name=self.name,
-                limit=current_batch_size,
-                last_retrieved_id=result[-1].id if len(result) > 0 else None,
-                truncate=truncate,
-            )
-
-            experiment_item_compare_current_batch = (
-                rest_stream_parser.read_and_parse_stream(
-                    stream=items_stream,
-                    item_class=opik.rest_api.ExperimentItemCompare,
-                )
-            )
-
-            for item in experiment_item_compare_current_batch:
-                converted_item = experiment_item.ExperimentItemContent.from_rest_experiment_item_compare(
-                    value=item
-                )
-                result.append(converted_item)
-
-            if current_batch_size > len(experiment_item_compare_current_batch):
-                break
-
-        return result
+        if max_results is None:
+            max_results = 10000  # TODO: remove this once we have a proper way to get all experiment items
+
+        return self._experiments_client.find_experiment_items_for_dataset(
+            dataset_name=self.dataset_name,
+            experiment_ids=[self.id],
+            truncate=truncate,
+            max_results=max_results,
+        )

{opik-1.8.101 → opik-1.9.33}/src/opik/api_objects/experiment/experiment_item.py

@@ -24,6 +24,7 @@ class ExperimentItemContent:
     def from_rest_experiment_item_compare(
         cls,
         value: experiment_item_compare.ExperimentItemCompare,
+        dataset_item_data: Optional[Dict[str, Any]] = None,
     ) -> "ExperimentItemContent":
         if value.feedback_scores is None:
             feedback_scores: List[FeedbackScoreDict] = []
@@ -42,7 +43,7 @@ class ExperimentItemContent:
             id=value.id,
             trace_id=value.trace_id,
             dataset_item_id=value.dataset_item_id,
-            dataset_item_data=value.input,
+            dataset_item_data=dataset_item_data if dataset_item_data else value.input,
             evaluation_task_output=value.output,
             feedback_scores=feedback_scores,
         )

opik-1.9.33/src/opik/api_objects/experiment/experiments_client.py

@@ -0,0 +1,64 @@
+import json
+from typing import List, Optional
+
+from . import rest_operations, experiment_item
+from .. import opik_query_language
+from ...rest_api import client as rest_api_client
+
+
+class ExperimentsClient:
+    """Client for managing and querying experiments in bulk."""
+
+    def __init__(self, rest_client: rest_api_client.OpikApi):
+        self._rest_client = rest_client
+
+    def find_experiment_items_for_dataset(
+        self,
+        dataset_name: str,
+        experiment_ids: List[str],
+        truncate: bool = True,
+        max_results: int = 1000,
+        filter_string: Optional[str] = None,
+    ) -> List[experiment_item.ExperimentItemContent]:
+        """
+        Find experiment items associated with a specific dataset among a list of experiments.
+
+        This method queries the dataset for experiment items matching the
+        criteria provided by the input parameters. It leverages the
+        ExperimentsClient to perform the underlying operation.
+
+        Args:
+            dataset_name: Name of the dataset to query for experiment items.
+            experiment_ids: List of experiment IDs to filter the results.
+            filter_string: Optional filter string to refine the
+                query based on additional criteria (dataset fields, feedback scores, etc.).
+            truncate: Whether to truncate image data stored in input, output,
+                or metadata. Defaults to True.
+            max_results: Maximum number of results to return. Defaults to 1000.
+
+        Returns:
+            A list of experiment item content objects that match the criteria.
+        """
+        # prepare filter expression
+        if filter_string is not None:
+            filter_expression = json.dumps(
+                opik_query_language.OpikQueryLanguage(
+                    filter_string
+                ).get_filter_expressions()
+            )
+        else:
+            filter_expression = None
+
+        # get dataset id
+        dataset_id = self._rest_client.datasets.get_dataset_by_identifier(
+            dataset_name=dataset_name
+        ).id
+
+        return rest_operations.find_experiment_items_for_dataset(
+            dataset_id=dataset_id,
+            experiment_ids=experiment_ids,
+            rest_client=self._rest_client,
+            max_results=max_results,
+            truncate=truncate,
+            filter_expression=filter_expression,
+        )

opik-1.9.33/src/opik/api_objects/experiment/rest_operations.py

@@ -0,0 +1,105 @@
+import json
+from typing import List, Optional
+
+from . import experiment_item
+from .. import rest_stream_parser
+from ... import exceptions, rest_api
+from ...rest_api.types import experiment_public
+
+
+def get_experiment_data_by_name(
+    rest_client: rest_api.OpikApi, name: str
+) -> experiment_public.ExperimentPublic:
+    # TODO: this method is deprecated and should be removed after
+    #  deprecated Opik.get_experiment_by_name() will be removed.
+    #  This function should not be used anywhere else except for deprecated logic as it is confusing and misleading
+
+    experiments = get_experiments_data_by_name(rest_client, name)
+    for experiment in experiments:
+        if experiment.name == name:
+            return experiment
+
+    raise exceptions.ExperimentNotFound(f"No experiment found with the name '{name}'.")
+
+
+def get_experiments_data_by_name(
+    rest_client: rest_api.OpikApi,
+    name: str,
+    max_results: Optional[int] = None,
+) -> List[experiment_public.ExperimentPublic]:
+    experiments = rest_stream_parser.read_and_parse_full_stream(
+        read_source=lambda current_batch_size,
+        last_retrieved_id: rest_client.experiments.stream_experiments(
+            name=name,
+            limit=current_batch_size,
+            last_retrieved_id=last_retrieved_id,
+        ),
+        max_results=max_results,
+        parsed_item_class=experiment_public.ExperimentPublic,
+    )
+
+    if len(experiments) == 0:
+        raise exceptions.ExperimentNotFound(
+            f"No experiment(s) found with the name '{name}'."
+        )
+
+    return experiments
+
+
+def find_experiment_items_for_dataset(
+    rest_client: rest_api.OpikApi,
+    dataset_id: str,
+    experiment_ids: List[str],
+    max_results: int,
+    truncate: bool,
+    filter_expression: Optional[str] = None,
+) -> List[experiment_item.ExperimentItemContent]:
+    PAGE_SIZE = 100
+
+    collected_items: List[experiment_item.ExperimentItemContent] = []
+    experiment_ids_json = json.dumps(experiment_ids)
+
+    page_number = 1
+    while len(collected_items) < max_results:
+        page_dataset_items = (
+            rest_client.datasets.find_dataset_items_with_experiment_items(
+                id=dataset_id,
+                page=page_number,
+                size=PAGE_SIZE,
+                experiment_ids=experiment_ids_json,
+                truncate=truncate,
+                filters=filter_expression,
+            )
+        )
+
+        if not page_dataset_items.content:
+            break
+
+        # Build a flat list of experiment-item compares for this page in a readable way
+        experiment_items = []
+        for dataset_item in page_dataset_items.content:
+            # Guard if dataset_item.experiment_items might be None
+            if dataset_item.experiment_items is not None:
+                for experiment_item_compare in dataset_item.experiment_items:
+                    # Convert to domain objects
+                    dataset_item_data = dataset_item.data
+                    if dataset_item_data is not None:
+                        dataset_item_data.update({"id": dataset_item.id})
+                    experiment_item_content = experiment_item.ExperimentItemContent.from_rest_experiment_item_compare(
+                        value=experiment_item_compare,
+                        dataset_item_data=dataset_item_data,
+                    )
+
+                    experiment_items.append(experiment_item_content)
+
+        if not experiment_items:
+            page_number += 1
+            continue
+
+        remaining = max_results - len(collected_items)
+
+        collected_items.extend(experiment_items[:remaining])
+
+        page_number += 1
+
+    return collected_items