fiddler-evals 0.1.1.dev11__tar.gz → 0.1.1.dev13__tar.gz

{fiddler_evals-0.1.1.dev11/fiddler_evals.egg-info → fiddler_evals-0.1.1.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fiddler-evals
-Version: 0.1.1.dev11
+Version: 0.1.1.dev13
 Summary: Python SDK for evaluating LLM Applications
 Author-email: Fiddler AI <support@fiddler.ai>
 Maintainer-email: Fiddler AI <support@fiddler.ai>

fiddler_evals-0.1.1.dev13/fiddler_evals/VERSION

@@ -0,0 +1 @@
+0.1.1.dev13

{fiddler_evals-0.1.1.dev11 → fiddler_evals-0.1.1.dev13}/fiddler_evals/entities/dataset.py

@@ -732,7 +732,10 @@ class Dataset(BaseEntity):
             with list(dataset.get_items()) if you need to iterate multiple times or get
             the total count. The iterator fetches items lazily from the API.
         """
-        for item in self._paginate(url=f"{self._get_url(self.id)}/items"):
+        # Read upto 1K dataset items in a call to reduce network calls and latency
+        for item in self._paginate(
+            url=f"{self._get_url(self.id)}/items", page_size=1000
+        ):
             yield DatasetItem(**item)
 
     @handle_api_error

{fiddler_evals-0.1.1.dev11 → fiddler_evals-0.1.1.dev13}/fiddler_evals/runner/experiment_result_publisher.py

@@ -61,7 +61,7 @@ class ExperimentResultPublisher:
 
             # Auto-flush when batch size is reached
             if len(self._buffer) >= self._batch_size:
-                logger.info(
+                logger.debug(
                     "Flushing experiment result buffer of size %d", len(self._buffer)
                 )
                 self._flush_internal()

{fiddler_evals-0.1.1.dev11 → fiddler_evals-0.1.1.dev13}/fiddler_evals/runner/experiment_runner.py

@@ -656,6 +656,11 @@ class ExperimentRunner:
             self._dataset.name,
         )
 
+        # Mark the experiment as in progress
+        self._experiment.update(
+            status=ExperimentStatus.IN_PROGRESS,
+        )
+
         # Use a thread pool to process the dataset items when max_workers > 1
         if self._max_workers > 1:
             results = thread_map(

{fiddler_evals-0.1.1.dev11 → fiddler_evals-0.1.1.dev13}/fiddler_evals/runner/tests/test_evaluate.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 from typing import Any
 from unittest import mock
@@ -147,7 +148,23 @@ def test_evaluate_with_regex_with_kwargs_mapping() -> None:
         json=EXPERIMENT_API_RESPONSE,
     )
 
-    # Mock experiment updation
+    # Mock experiment status update to IN_PROGRESS with request body verification
+    in_progress_response = EXPERIMENT_API_RESPONSE.copy()
+    in_progress_response["data"]["status"] = ExperimentStatus.IN_PROGRESS
+
+    def in_progress_callback(request):
+        # Verify that the request body contains the correct status
+        request_body = json.loads(request.body)
+        assert request_body["status"] == ExperimentStatus.IN_PROGRESS
+        return (200, {}, json.dumps(in_progress_response))
+
+    responses.add_callback(
+        responses.PATCH,
+        url=f"{URL}/v3/evals/experiments/{EXPERIMENT_ID}",
+        callback=in_progress_callback,
+    )
+
+    # Mock experiment updation to COMPLETED
     updated_response = EXPERIMENT_API_RESPONSE.copy()
     updated_response["data"]["status"] = ExperimentStatus.COMPLETED
     responses.patch(

{fiddler_evals-0.1.1.dev11 → fiddler_evals-0.1.1.dev13/fiddler_evals.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fiddler-evals
-Version: 0.1.1.dev11
+Version: 0.1.1.dev13
 Summary: Python SDK for evaluating LLM Applications
 Author-email: Fiddler AI <support@fiddler.ai>
 Maintainer-email: Fiddler AI <support@fiddler.ai>

{fiddler_evals-0.1.1.dev11 → fiddler_evals-0.1.1.dev13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fiddler-evals"
-version = "0.1.1.dev11"
+version = "0.1.1.dev13"
 authors = [
   { name = "Fiddler AI", email = "support@fiddler.ai" },
 ]

fiddler_evals-0.1.1.dev11/fiddler_evals/VERSION

@@ -1 +0,0 @@
-0.1.1.dev11