mlrun 1.8.0rc5__py3-none-any.whl → 1.8.0rc7__py3-none-any.whl

mlrun/db/base.py

@@ -370,7 +370,9 @@ class RunDBInterface(ABC):
         pass
 
     @abstractmethod
-    def get_project(self, name: str) -> mlrun.common.schemas.Project:
+    def get_project(
+        self, name: str
+    ) -> Union[mlrun.common.schemas.Project, "mlrun.MlrunProject"]:
         pass
 
     @abstractmethod
@@ -663,16 +665,16 @@ class RunDBInterface(ABC):
     @abstractmethod
     def create_model_endpoint(
         self,
-        project: str,
-        endpoint_id: str,
-        model_endpoint: Union[mlrun.model_monitoring.ModelEndpoint, dict],
-    ):
+        model_endpoint: mlrun.common.schemas.ModelEndpoint,
+    ) -> mlrun.common.schemas.ModelEndpoint:
         pass
 
     @abstractmethod
     def delete_model_endpoint(
         self,
+        name: str,
         project: str,
+        function_name: str,
         endpoint_id: str,
     ):
         pass
@@ -681,36 +683,40 @@ class RunDBInterface(ABC):
     def list_model_endpoints(
         self,
         project: str,
-        model: Optional[str] = None,
-        function: Optional[str] = None,
+        name: Optional[str] = None,
+        function_name: Optional[str] = None,
+        model_name: Optional[str] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-        start: str = "now-1h",
-        end: str = "now",
-        metrics: Optional[list[str]] = None,
+        start: Optional[datetime.datetime] = None,
+        end: Optional[datetime.datetime] = None,
+        tsdb_metrics: bool = True,
         top_level: bool = False,
         uids: Optional[list[str]] = None,
-    ) -> list[mlrun.model_monitoring.model_endpoint.ModelEndpoint]:
+        latest_only: bool = False,
+    ) -> mlrun.common.schemas.ModelEndpointList:
         pass
 
     @abstractmethod
     def get_model_endpoint(
         self,
+        name: str,
         project: str,
-        endpoint_id: str,
-        start: Optional[str] = None,
-        end: Optional[str] = None,
-        metrics: Optional[list[str]] = None,
-        features: bool = False,
-    ) -> mlrun.model_monitoring.ModelEndpoint:
+        function_name: Optional[str] = None,
+        endpoint_id: Optional[str] = None,
+        tsdb_metrics: bool = True,
+        feature_analysis: bool = False,
+    ) -> mlrun.common.schemas.ModelEndpoint:
         pass
 
     @abstractmethod
     def patch_model_endpoint(
         self,
+        name: str,
         project: str,
-        endpoint_id: str,
         attributes: dict,
-    ):
+        function_name: Optional[str] = None,
+        endpoint_id: Optional[str] = None,
+    ) -> mlrun.common.schemas.ModelEndpoint:
         pass
 
     @abstractmethod
@@ -1051,3 +1057,7 @@ class RunDBInterface(ABC):
         replace_creds: bool,
     ) -> None:
         pass
+
+    @abstractmethod
+    def get_project_summary(self, project: str) -> mlrun.common.schemas.ProjectSummary:
+        pass

mlrun/db/httpdb.py

@@ -28,7 +28,7 @@ from urllib.parse import urlparse
 import pydantic.v1
 import requests
 import semver
-from pydantic import parse_obj_as
+from pydantic.v1 import parse_obj_as
 
 import mlrun
 import mlrun.common.constants
@@ -37,7 +37,6 @@ import mlrun.common.runtimes
 import mlrun.common.schemas
 import mlrun.common.schemas.model_monitoring.model_endpoints as mm_endpoints
 import mlrun.common.types
-import mlrun.model_monitoring.model_endpoint
 import mlrun.platforms
 import mlrun.projects
 import mlrun.runtimes.nuclio.api_gateway
@@ -49,6 +48,7 @@ from mlrun.errors import MLRunInvalidArgumentError, err_to_str
 from mlrun_pipelines.utils import compile_pipeline
 
 from ..artifacts import Artifact
+from ..common.schemas import AlertActivations
 from ..config import config
 from ..datastore.datastore_profile import DatastoreProfile2Json
 from ..feature_store import FeatureSet, FeatureVector
@@ -558,10 +558,6 @@ class HTTPRunDB(RunDBInterface):
                 server_cfg.get("external_platform_tracking")
                 or config.external_platform_tracking
             )
-            config.model_endpoint_monitoring.endpoint_store_connection = (
-                server_cfg.get("model_endpoint_monitoring_endpoint_store_connection")
-                or config.model_endpoint_monitoring.endpoint_store_connection
-            )
             config.model_endpoint_monitoring.tsdb_connection = (
                 server_cfg.get("model_monitoring_tsdb_connection")
                 or config.model_endpoint_monitoring.tsdb_connection
@@ -1701,6 +1697,8 @@ class HTTPRunDB(RunDBInterface):
         name: Optional[str] = None,
         kind: mlrun.common.schemas.ScheduleKinds = None,
         include_last_run: bool = False,
+        next_run_time_since: Optional[datetime] = None,
+        next_run_time_until: Optional[datetime] = None,
     ) -> mlrun.common.schemas.SchedulesOutput:
         """Retrieve list of schedules of specific name or kind.
 
@@ -1709,10 +1707,18 @@ class HTTPRunDB(RunDBInterface):
         :param kind: Kind of schedule objects to retrieve, can be either ``job`` or ``pipeline``.
         :param include_last_run: Whether to return for each schedule returned also the results of the last run of
             that schedule.
+        :param next_run_time_since: Return only schedules with next run time after this date.
+        :param next_run_time_until: Return only schedules with next run time before this date.
         """
 
         project = project or config.default_project
-        params = {"kind": kind, "name": name, "include_last_run": include_last_run}
+        params = {
+            "kind": kind,
+            "name": name,
+            "include_last_run": include_last_run,
+            "next_run_time_since": datetime_to_iso(next_run_time_since),
+            "next_run_time_until": datetime_to_iso(next_run_time_until),
+        }
         path = f"projects/{project}/schedules"
         error_message = f"Failed listing schedules for {project} ? {kind} {name}"
         resp = self.api_call("GET", path, error_message, params=params)
@@ -2260,6 +2266,75 @@ class HTTPRunDB(RunDBInterface):
 
         return resp.json()
 
+    def retry_pipeline(
+        self,
+        run_id: str,
+        namespace: Optional[str] = None,
+        timeout: int = 30,
+        project: Optional[str] = None,
+    ):
+        """
+        Retry a specific pipeline run using its run ID. This function sends an API request
+        to retry a pipeline run. If a project is specified, the run must belong to that
+        project; otherwise, all projects are queried.
+
+        :param run_id: The unique ID of the pipeline run to retry.
+        :param namespace: Kubernetes namespace where the pipeline is running. Optional.
+        :param timeout: Timeout (in seconds) for the API call. Defaults to 30 seconds.
+        :param project: Name of the MLRun project associated with the pipeline. Can be
+            ``*`` to query across all projects. Optional.
+
+        :raises ValueError: Raised if the API response is not successful or contains an
+            error.
+
+        :return: JSON response containing details of the retried pipeline run.
+        """
+
+        params = {}
+        if namespace:
+            params["namespace"] = namespace
+        project_path = project if project else "*"
+
+        resp_text = ""
+        resp_code = None
+        try:
+            resp = self.api_call(
+                "POST",
+                f"projects/{project_path}/pipelines/{run_id}/retry",
+                params=params,
+                timeout=timeout,
+            )
+            resp_code = resp.status_code
+            resp_text = resp.text
+            if not resp.ok:
+                raise mlrun.errors.MLRunHTTPError(
+                    f"Failed to retry pipeline run '{run_id}'. "
+                    f"HTTP {resp_code}: {resp_text}"
+                )
+        except Exception as exc:
+            logger.error(
+                "Retry pipeline API call encountered an error.",
+                run_id=run_id,
+                project=project_path,
+                namespace=namespace,
+                response_code=resp_code,
+                response_text=resp_text,
+                error=str(exc),
+            )
+            if isinstance(exc, mlrun.errors.MLRunHTTPError):
+                raise exc  # Re-raise known HTTP errors
+            raise mlrun.errors.MLRunRuntimeError(
+                f"Unexpected error while retrying pipeline run '{run_id}'."
+            ) from exc
+
+        logger.info(
+            "Successfully retried pipeline run",
+            run_id=run_id,
+            project=project_path,
+            namespace=namespace,
+        )
+        return resp.json()
+
     @staticmethod
     def _resolve_reference(tag, uid):
         if uid and tag:
@@ -3048,7 +3123,7 @@ class HTTPRunDB(RunDBInterface):
             for project_dict in response.json()["projects"]
         ]
 
-    def get_project(self, name: str) -> mlrun.projects.MlrunProject:
+    def get_project(self, name: str) -> "mlrun.MlrunProject":
         """Get details for a specific project."""
 
         if not name:
@@ -3057,7 +3132,7 @@ class HTTPRunDB(RunDBInterface):
         path = f"projects/{name}"
         error_message = f"Failed retrieving project {name}"
         response = self.api_call("GET", path, error_message)
-        return mlrun.projects.MlrunProject.from_dict(response.json())
+        return mlrun.MlrunProject.from_dict(response.json())
 
     def delete_project(
         self,
@@ -3504,217 +3579,180 @@ class HTTPRunDB(RunDBInterface):
 
     def create_model_endpoint(
         self,
-        project: str,
-        endpoint_id: str,
-        model_endpoint: Union[
-            mlrun.model_monitoring.model_endpoint.ModelEndpoint, dict
-        ],
-    ):
+        model_endpoint: mlrun.common.schemas.ModelEndpoint,
+    ) -> mlrun.common.schemas.ModelEndpoint:
         """
         Creates a DB record with the given model_endpoint record.
 
-        :param project: The name of the project.
-        :param endpoint_id: The id of the endpoint.
         :param model_endpoint: An object representing the model endpoint.
-        """
 
-        if isinstance(
-            model_endpoint, mlrun.model_monitoring.model_endpoint.ModelEndpoint
-        ):
-            model_endpoint = model_endpoint.to_dict()
+        :return: The created model endpoint object.
+        """
 
-        path = f"projects/{project}/model-endpoints/{endpoint_id}"
-        self.api_call(
-            method="POST",
+        path = f"projects/{model_endpoint.metadata.project}/model-endpoints"
+        response = self.api_call(
+            method=mlrun.common.types.HTTPMethod.POST,
             path=path,
-            body=dict_to_json(model_endpoint),
+            body=model_endpoint.json(),
         )
+        return mlrun.common.schemas.ModelEndpoint(**response.json())
 
     def delete_model_endpoint(
         self,
+        name: str,
         project: str,
+        function_name: str,
         endpoint_id: str,
     ):
         """
         Deletes the DB record of a given model endpoint, project and endpoint_id are used for lookup
 
+        :param name: The name of the model endpoint
         :param project: The name of the project
+        :param function_name: The name of the function
         :param endpoint_id: The id of the endpoint
         """
 
-        path = f"projects/{project}/model-endpoints/{endpoint_id}"
+        path = f"projects/{project}/model-endpoints/{name}"
         self.api_call(
-            method="DELETE",
+            method=mlrun.common.types.HTTPMethod.DELETE,
             path=path,
+            params={
+                "function_name": function_name,
+                "endpoint_id": endpoint_id,
+            },
         )
 
     def list_model_endpoints(
         self,
         project: str,
-        model: Optional[str] = None,
-        function: Optional[str] = None,
+        name: Optional[str] = None,
+        function_name: Optional[str] = None,
+        model_name: Optional[str] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-        start: str = "now-1h",
-        end: str = "now",
-        metrics: Optional[list[str]] = None,
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+        tsdb_metrics: bool = True,
         top_level: bool = False,
         uids: Optional[list[str]] = None,
-    ) -> list[mlrun.model_monitoring.model_endpoint.ModelEndpoint]:
-        """
-        Returns a list of `ModelEndpoint` objects. Each `ModelEndpoint` object represents the current state of a
-        model endpoint. This functions supports filtering by the following parameters:
-        1) model
-        2) function
-        3) labels
-        4) top level
-        5) uids
-        By default, when no filters are applied, all available endpoints for the given project will be listed.
-
-        In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
-        and depends on the 'start' and 'end' parameters. By default, when the metrics parameter is None, no metrics are
-        added to the output of this function.
-
-        :param project: The name of the project
-        :param model: The name of the model to filter by
-        :param function: The name of the function to filter by
-        :param labels: Filter model endpoints by label key-value pairs or key existence. This can be provided as:
-            - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
-            or `{"label": None}` to check for key existence.
-            - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
-            or just `"label"` for key existence.
-            - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
-            the specified key-value pairs or key existence.
-        :param metrics: A list of metrics to return for each endpoint, read more in 'TimeMetric'
-        :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339 time, a
-                      Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
-                      `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
-        :param end: The end time of the metrics. Can be represented by a string containing an RFC 3339 time, a
-                      Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
-                      `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
-        :param top_level: if true will return only routers and endpoint that are NOT children of any router
-        :param uids: if passed will return a list `ModelEndpoint` object with uid in uids
-
-        :returns: Returns a list of `ModelEndpoint` objects.
+        latest_only: bool = False,
+    ) -> mlrun.common.schemas.ModelEndpointList:
+        """
+        List model endpoints with optional filtering by name, function name, model name, labels, and time range.
+
+        :param project:         The name of the project
+        :param name:            The name of the model endpoint
+        :param function_name:   The name of the function
+        :param model_name:      The name of the model
+        :param labels:          A list of labels to filter by. (see mlrun.common.schemas.LabelsModel)
+        :param start:           The start time to filter by.Corresponding to the `created` field.
+        :param end:             The end time to filter by. Corresponding to the `created` field.
+        :param tsdb_metrics:    Whether to include metrics from the time series DB.
+        :param top_level:       Whether to return only top level model endpoints.
+        :param uids:            A list of unique ids to filter by.
+        :param latest_only:     Whether to return only the latest model endpoint version.
+        :return:                A list of model endpoints.
         """
-
         path = f"projects/{project}/model-endpoints"
         labels = self._parse_labels(labels)
 
         response = self.api_call(
-            method="GET",
+            method=mlrun.common.types.HTTPMethod.GET,
             path=path,
             params={
-                "model": model,
-                "function": function,
+                "name": name,
+                "model_name": model_name,
+                "function_name": function_name,
                 "label": labels,
-                "start": start,
-                "end": end,
-                "metric": metrics or [],
+                "start": datetime_to_iso(start),
+                "end": datetime_to_iso(end),
+                "tsdb_metrics": tsdb_metrics,
                 "top-level": top_level,
                 "uid": uids,
+                "latest_only": latest_only,
             },
         )
 
-        # Generate a list of a model endpoint dictionaries
-        model_endpoints = response.json()["endpoints"]
-        if model_endpoints:
-            return [
-                mlrun.model_monitoring.model_endpoint.ModelEndpoint.from_dict(obj)
-                for obj in model_endpoints
-            ]
-        return []
+        return mlrun.common.schemas.ModelEndpointList(**response.json())
 
     def get_model_endpoint(
         self,
+        name: str,
         project: str,
-        endpoint_id: str,
-        start: Optional[str] = None,
-        end: Optional[str] = None,
-        metrics: Optional[list[str]] = None,
+        function_name: Optional[str] = None,
+        # TODO: function_tag
+        endpoint_id: Optional[str] = None,
+        tsdb_metrics: bool = True,
         feature_analysis: bool = False,
-    ) -> mlrun.model_monitoring.model_endpoint.ModelEndpoint:
+    ) -> mlrun.common.schemas.ModelEndpoint:
         """
         Returns a single `ModelEndpoint` object with additional metrics and feature related data.
 
+        :param name:                       The name of the model endpoint
         :param project:                    The name of the project
-        :param endpoint_id:                The unique id of the model endpoint.
-        :param start:                      The start time of the metrics. Can be represented by a string containing an
-                                           RFC 3339 time, a  Unix timestamp in milliseconds, a relative time
-                                           (`'now'` or `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours,
-                                           `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
-        :param end:                        The end time of the metrics. Can be represented by a string containing an
-                                           RFC 3339 time, a  Unix timestamp in milliseconds, a relative time
-                                           (`'now'` or `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours,
-                                           `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
-        :param metrics:                    A list of metrics to return for the model endpoint. There are pre-defined
-                                           metrics for model endpoints such as predictions_per_second and
-                                           latency_avg_5m but also custom metrics defined by the user. Please note that
-                                           these metrics are stored in the time series DB and the results will be
-                                           appeared under model_endpoint.spec.metrics.
-        :param feature_analysis:           When True, the base feature statistics and current feature statistics will
-                                           be added to the output of the resulting object.
-
-        :returns: A `ModelEndpoint` object.
-        """
-
-        path = f"projects/{project}/model-endpoints/{endpoint_id}"
+        :param function_name:              The name of the function
+        :param endpoint_id:                The id of the endpoint
+        :param tsdb_metrics:               Whether to include metrics from the time series DB.
+        :param feature_analysis:           Whether to include feature analysis data (feature_stats,
+                                            current_stats & drift_measures).
+
+        :return:                          A `ModelEndpoint` object.
+        """
+
+        path = f"projects/{project}/model-endpoints/{name}"
         response = self.api_call(
-            method="GET",
+            method=mlrun.common.types.HTTPMethod.GET,
             path=path,
             params={
-                "start": start,
-                "end": end,
-                "metric": metrics or [],
+                "function_name": function_name,
+                "endpoint_id": endpoint_id,
+                "tsdb_metrics": tsdb_metrics,
                 "feature_analysis": feature_analysis,
             },
         )
 
-        return mlrun.model_monitoring.model_endpoint.ModelEndpoint.from_dict(
-            response.json()
-        )
+        return mlrun.common.schemas.ModelEndpoint(**response.json())
 
     def patch_model_endpoint(
         self,
+        name: str,
         project: str,
-        endpoint_id: str,
         attributes: dict,
-    ):
+        function_name: Optional[str] = None,
+        endpoint_id: Optional[str] = None,
+    ) -> mlrun.common.schemas.ModelEndpoint:
         """
-        Updates model endpoint with provided attributes.
-
-        :param project: The name of the project.
-        :param endpoint_id: The id of the endpoint.
-        :param attributes: Dictionary of attributes that will be used for update the model endpoint. The keys
-            of this dictionary should exist in the target table. Note that the values should be from type string or from
-            a valid numerical type such as int or float. More details about the model endpoint available attributes can
-            be found under :py:class:`~mlrun.common.schemas.ModelEndpoint`.
-
-        Example::
+        Updates a model endpoint with the given attributes.
 
-            # Generate current stats for two features
-            current_stats = {'tvd_sum': 2.2,
-                             'tvd_mean': 0.5,
-                             'hellinger_sum': 3.6,
-                             'hellinger_mean': 0.9,
-                             'kld_sum': 24.2,
-                             'kld_mean': 6.0,
-                             'f1': {'tvd': 0.5, 'hellinger': 1.0, 'kld': 6.4},
-                             'f2': {'tvd': 0.5, 'hellinger': 1.0, 'kld': 6.5}}
-
-            # Create attributes dictionary according to the required format
-            attributes = {`current_stats`: json.dumps(current_stats),
-                          `drift_status`: "DRIFT_DETECTED"}
-
-        """
-
-        attributes = {"attributes": _as_json(attributes)}
-        path = f"projects/{project}/model-endpoints/{endpoint_id}"
-        self.api_call(
-            method="PATCH",
+        :param name:                       The name of the model endpoint
+        :param project:                    The name of the project
+        :param attributes:                 The attributes to update
+        :param function_name:              The name of the function
+        :param endpoint_id:                The id of the endpoint
+        :return:                          The updated `ModelEndpoint` object.
+        """
+        attributes_keys = list(attributes.keys())
+        attributes["name"] = name
+        attributes["project"] = project
+        attributes["uid"] = endpoint_id or ""
+        model_endpoint = mlrun.common.schemas.ModelEndpoint.from_flat_dict(attributes)
+        path = f"projects/{project}/model-endpoints"
+        logger.info(
+            "Patching model endpoint",
+            attributes_keys=attributes_keys,
+            model_endpoint=model_endpoint,
+        )
+        response = self.api_call(
+            method=mlrun.common.types.HTTPMethod.PATCH,
             path=path,
-            params=attributes,
+            params={
+                "attribute-key": attributes_keys,
+            },
+            body=model_endpoint.json(),
         )
 
+        return mlrun.common.schemas.ModelEndpoint(**response.json())
+
     def update_model_monitoring_controller(
         self,
         project: str,
@@ -4707,7 +4745,7 @@ class HTTPRunDB(RunDBInterface):
             Union[mlrun.common.schemas.alert.EventEntityKind, str]
         ] = None,
         event_kind: Optional[Union[mlrun.common.schemas.alert.EventKind, str]] = None,
-    ) -> list[mlrun.common.schemas.AlertActivation]:
+    ) -> mlrun.common.schemas.AlertActivations:
         """
         Retrieve a list of all alert activations.
 
@@ -4743,7 +4781,7 @@ class HTTPRunDB(RunDBInterface):
         page_size: Optional[int] = None,
         page_token: Optional[str] = None,
         **kwargs,
-    ) -> tuple[list, Optional[str]]:
+    ) -> tuple[AlertActivations, Optional[str]]:
         """List alerts activations with support for pagination and various filtering options.
 
         This method retrieves a paginated list of alert activations based on the specified filter parameters.
@@ -4799,6 +4837,22 @@ class HTTPRunDB(RunDBInterface):
             **kwargs,
         )
 
+    def get_project_summary(
+        self, project: Optional[str] = None
+    ) -> mlrun.common.schemas.ProjectSummary:
+        """
+        Retrieve the summary of a project.
+
+        :param project: Project name for which the summary belongs.
+        :returns: A summary of the project.
+        """
+        project = project or config.default_project
+
+        endpoint_path = f"project-summaries/{project}"
+        error_message = f"Failed retrieving project summary for {project}"
+        response = self.api_call("GET", endpoint_path, error_message)
+        return mlrun.common.schemas.ProjectSummary(**response.json())
+
     @staticmethod
     def _parse_labels(
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]],
@@ -5031,9 +5085,11 @@ class HTTPRunDB(RunDBInterface):
             "name": name,
             "uid": uid,
             "label": labels,
-            "state": mlrun.utils.helpers.as_list(state)
-            if state is not None
-            else states or None,
+            "state": (
+                mlrun.utils.helpers.as_list(state)
+                if state is not None
+                else states or None
+            ),
             "sort": bool2str(sort),
             "iter": bool2str(iter),
             "start_time_from": datetime_to_iso(start_time_from),
@@ -5088,7 +5144,7 @@ class HTTPRunDB(RunDBInterface):
         page_size: Optional[int] = None,
         page_token: Optional[str] = None,
         return_all: bool = False,
-    ) -> tuple[list[mlrun.common.schemas.AlertActivation], Optional[str]]:
+    ) -> tuple[mlrun.common.schemas.AlertActivations, Optional[str]]:
         project = project or config.default_project
         params = {
             "name": name,
@@ -5112,9 +5168,12 @@ class HTTPRunDB(RunDBInterface):
         paginated_responses, token = self.process_paginated_responses(
             responses, "activations"
         )
-        paginated_results = [
-            mlrun.common.schemas.AlertActivation(**item) for item in paginated_responses
-        ]
+        paginated_results = mlrun.common.schemas.AlertActivations(
+            activations=[
+                mlrun.common.schemas.AlertActivation(**item)
+                for item in paginated_responses
+            ]
+        )
 
         return paginated_results, token