python-flexeval 0.2.0__tar.gz → 0.3.0__tar.gz

{python_flexeval-0.2.0 → python_flexeval-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-flexeval
-Version: 0.2.0
+Version: 0.3.0
 Summary: FlexEval is a tool for designing custom metrics, completion functions, and LLM-graded rubrics for evaluating the behavior of LLM-powered systems.
 Project-URL: Homepage, https://digitalharborfoundation.github.io/FlexEval/
 Project-URL: GitHub, https://github.com/DigitalHarborFoundation/FlexEval

python_flexeval-0.3.0/src/flexeval/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

python_flexeval-0.3.0/src/flexeval/classes/jsonview.py

@@ -0,0 +1,107 @@
+import json
+from collections import UserDict
+
+
+class JsonViewDict(UserDict):
+    """Dictionary that syncs changes back to the model field."""
+
+    def __init__(
+        self,
+        model_instance,
+        text_field_attr_name,
+        json_dumps_fn=json.dumps,
+        json_loads_fn=json.loads,
+    ):
+        self.model_instance = model_instance
+        self.text_field_attr_name = text_field_attr_name
+        self.json_dumps_fn = json_dumps_fn
+        self.json_loads_fn = json_loads_fn
+
+        text_value = getattr(model_instance, text_field_attr_name)
+        initial_data = self.json_loads_fn(text_value)
+        super().__init__(initial_data)
+
+    def _sync_to_model(self):
+        """Sync the current data back to the model field."""
+        json_str = self.json_loads_fn(self.data)
+        setattr(self.model_instance, self.text_field_attr_name, json_str)
+
+    # Override mutating methods to trigger sync
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+        self._sync_to_model()
+
+    def __delitem__(self, key):
+        super().__delitem__(key)
+        self._sync_to_model()
+
+    def clear(self):
+        super().clear()
+        self._sync_to_model()
+
+    def pop(self, key, *args):
+        result = super().pop(key, *args)
+        self._sync_to_model()
+        return result
+
+    def popitem(self):
+        result = super().popitem()
+        self._sync_to_model()
+        return result
+
+    def setdefault(self, key, default=None):
+        result = super().setdefault(key, default)
+        self._sync_to_model()
+        return result
+
+    def update(self, *args, **kwargs):
+        super().update(*args, **kwargs)
+        self._sync_to_model()
+
+
+class JsonView:
+    """Descriptor that provides dict-like access to a JSON text field.
+
+    Example:
+    class SomeModel(pw.Model):
+        some_field = pw.TextField(default="{}")
+        some_field_dict = JsonView(text_field_attr_name="some_field")
+
+    m = SomeModel()
+    m.some_field_dict["chosen_mistake"] = "whatever"
+    """
+
+    def __init__(self, text_field_attr_name):
+        self.text_field_attr_name = text_field_attr_name
+        self.attr_name = None
+
+    def __set_name__(self, owner, name):
+        """Called when the descriptor is assigned to a class attribute."""
+        self.attr_name = f"_{name}_dict"
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+
+        # Check if we already have a cached JsonViewDict
+        if not hasattr(instance, self.attr_name):
+            if not hasattr(instance, self.text_field_attr_name):
+                raise ValueError(
+                    f"Failed to link this JsonView to field '{self.text_field_attr_name}' because it doesn't exist on this model instance."
+                )
+            # Cache a new JsonViewDict
+            json_dict = JsonViewDict(instance, self.text_field_attr_name)
+            setattr(instance, self.attr_name, json_dict)
+
+        return getattr(instance, self.attr_name)
+
+    def __set__(self, instance, value):
+        """Allow setting the entire dict."""
+        if isinstance(value, dict):
+            json_dict = JsonViewDict(instance, self.text_field_attr_name)
+            json_dict.update(value)
+            setattr(instance, self.attr_name, json_dict)
+        else:
+            raise ValueError(
+                f"This JsonView must be a dictionary to set linked field '{self.text_field_attr_name}' correctly."
+            )

{python_flexeval-0.2.0 → python_flexeval-0.3.0}/src/flexeval/classes/message.py

@@ -10,6 +10,7 @@ from flexeval.classes.dataset import Dataset
 from flexeval.classes.eval_set_run import EvalSetRun
 from flexeval.classes.thread import Thread
 from flexeval.classes.turn import Turn
+from flexeval.classes.jsonview import JsonView
 from flexeval.configuration import completion_functions
 
 logger = logging.getLogger(__name__)
@@ -34,6 +35,10 @@ class Message(BaseModel):
     content = pw.TextField()
     context = pw.TextField(null=True)  # Previous messages
 
+    # metadata
+    metadata = pw.TextField(default="{}", null=False)
+    metadata_dict = JsonView("metadata")
+
     # helpers
     system_prompt = pw.TextField(null=True)
     is_flexeval_completion = pw.BooleanField(null=True)

{python_flexeval-0.2.0 → python_flexeval-0.3.0}/src/flexeval/classes/thread.py

@@ -3,6 +3,7 @@ import peewee as pw
 from flexeval.classes.base import BaseModel
 from flexeval.classes.dataset import Dataset
 from flexeval.classes.eval_set_run import EvalSetRun
+from flexeval.classes.jsonview import JsonView
 
 
 class Thread(BaseModel):
@@ -20,6 +21,9 @@ class Thread(BaseModel):
 
     system_prompt = pw.TextField(null=True)
 
+    metadata = pw.TextField(default="{}", null=False)
+    metadata_dict = JsonView("metadata")
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.metrics_to_evaluate = []

{python_flexeval-0.2.0 → python_flexeval-0.3.0}/src/flexeval/data_loader.py

@@ -54,18 +54,13 @@ def load_jsonl(
                 max(1, nb_evaluations_per_thread)
             ):  # duplicate stored threads for averaged evaluation results
                 if thread_id in selected_thread_ids:
-                    thread_object = Thread.create(
-                        evalsetrun=dataset.evalsetrun,
-                        dataset=dataset,
-                        jsonl_thread_id=thread_id,
-                        eval_run_thread_id=str(thread_id)
-                        + "_"
-                        + str(thread_eval_run_id),
-                    )
+                    thread_json = json.loads(thread)
+                    # extract any metadata
+                    thread_metadata = thread_json.copy()
+                    del thread_metadata["input"]
 
-                    # Context
                     context = []
-                    thread_input = json.loads(thread)["input"]
+                    thread_input = thread_json["input"]
 
                     # Get system prompt used in the thread - assuming only 1
                     for message in thread_input:
@@ -78,15 +73,35 @@ def load_jsonl(
                         # Add the system prompt as context
                         context.append({"role": "system", "content": system_prompt})
 
+                    thread_object: Thread = Thread.create(
+                        evalsetrun=dataset.evalsetrun,
+                        dataset=dataset,
+                        jsonl_thread_id=thread_id,
+                        eval_run_thread_id=str(thread_id)
+                        + "_"
+                        + str(thread_eval_run_id),
+                        system_prompt=system_prompt,
+                        metadata=json.dumps(thread_metadata),
+                    )
+
                     # Create messages
                     index_in_thread = 0
                     for message in thread_input:
+                        if not isinstance(message, dict):
+                            raise ValueError(
+                                f"Can't load unknown object type; expected dict. Check JSONL format: {message}"
+                            )
                         role = message.get("role", None)
                         if role != "system":
                             # System message shouldn't be added as a separate message
                             system_prompt_for_this_message = ""
                             if role != "user":
                                 system_prompt_for_this_message = system_prompt
+                            message_metadata = message.copy()
+                            if "content" in message_metadata:
+                                del message_metadata["content"]
+                            if "role" in message_metadata:
+                                del message_metadata["role"]
                             Message.create(
                                 evalsetrun=dataset.evalsetrun,
                                 dataset=dataset,
@@ -95,9 +110,9 @@ def load_jsonl(
                                 role=role,
                                 content=message.get("content", None),
                                 context=json.dumps(context),
-                                metadata=message.get("metadata", None),
                                 is_flexeval_completion=False,
                                 system_prompt=system_prompt_for_this_message,
+                                metadata=json.dumps(message_metadata),
                             )
                             # Update context
                             context.append(

python_flexeval-0.3.0/tests/data/simple_metadata.jsonl

@@ -0,0 +1,2 @@
+{"key_1": "value_1", "key_2": {"nested_key": "nested_value"}, "input":[{ "role": "system", "content": "my system prompt" }, {"role":"user", "content": "I need help.", "index": 0}, {"role":"assistant", "content": "Help with what?", "index": 1}, {"role":"user", "content": "My homework.", "index": 2}]}
+{"input": [{ "role": "system", "content": "my system prompt" }, {"role": "user", "content": "Hi, Nice to meet you!"}, {"role": "assistant", "content": "Nice to meet you, too! How can I help you today?"}, {"role": "user", "content": "How do I find cube roots by hand?"}]}

{python_flexeval-0.2.0 → python_flexeval-0.3.0}/tests/unit/test_data_loader.py

@@ -59,6 +59,39 @@ class TestDataLoader(mixins.DotenvMixin, unittest.TestCase):
             # This is redundant, but just in case:
             data_loader.load_jsonl(dataset=dataset, filename=dataset_filepath)
 
+    def test_load_jsonl_metadata(self):
+        """Tests the inclusion of metadata in JSONL files."""
+        datasets = [
+            "tests/data/simple_metadata.jsonl",
+        ]
+        evalsetrun = EvalSetRun.create(
+            dataset_files=json.dumps(datasets),
+            metrics="",
+            metrics_graph_ordered_list="",
+            do_completion=False,
+        )
+        for dataset_filepath in evalsetrun.get_datasets():
+            dataset = Dataset.create(
+                evalsetrun=evalsetrun,
+                filename=dataset_filepath,
+            )
+            dataset.load_data()
+            # This is redundant, but just in case:
+            data_loader.load_jsonl(dataset=dataset, filename=dataset_filepath)
+
+            for thread in dataset.threads:
+                metadata = json.loads(thread.metadata)
+                assert "key_1" in metadata and metadata["key_1"] == "value_1"
+                assert (
+                    "key_2" in metadata
+                    and metadata["key_2"]["nested_key"] == "nested_value"
+                )
+                assert "input" not in metadata
+                for i, message in enumerate(thread.messages):
+                    metadata = json.loads(message.metadata)
+                    assert metadata["index"] == i
+                break
+
 
 class State(TypedDict):
     # TODO move this to some kind of langgraph utility file

python_flexeval-0.2.0/src/flexeval/__about__.py

@@ -1 +0,0 @@
-__version__ = "0.2.0"