rakam-eval-sdk 0.1.15__py3-none-any.whl → 0.1.16__py3-none-any.whl

rakam_eval_sdk/cli.py

@@ -0,0 +1,17 @@
+# cli.py
+import typer
+from pathlib import Path
+
+app = typer.Typer()
+
+
+@app.command()
+def read(file: Path):
+    """Read a Python file"""
+    if file.suffix != ".py":
+        raise typer.BadParameter("Must be a .py file")
+    typer.echo(file.read_text())
+
+
+def main():
+    app()

rakam_eval_sdk/client.py

@@ -1,6 +1,7 @@
 import os
 import random
-from typing import Any, List, Optional, cast
+from typing import Any, List, Optional, Union, cast
+
 import requests
 
 from .schema import (
@@ -79,10 +80,11 @@ class DeepEvalClient:
         metrics: List[MetricConfig],
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Run synchronous text evaluation."""
         payload = EvalConfig.model_construct(
-            data=data, metrics=metrics, component=component
+            data=data, metrics=metrics, component=component, version=version
         ).model_dump()
         return self._request("/deepeval/text-eval", payload, raise_exception)
 
@@ -92,10 +94,11 @@ class DeepEvalClient:
         metrics: List[MetricConfig],
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Run background text evaluation (async job)."""
         payload = EvalConfig.model_construct(
-            data=data, metrics=metrics, component=component
+            data=data, metrics=metrics, component=component, version=version
         ).model_dump()
         return self._request("/deepeval/text-eval/background", payload, raise_exception)
 
@@ -105,10 +108,11 @@ class DeepEvalClient:
         metrics: List[SchemaMetricConfig],
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Run synchronous schema evaluation."""
         payload = SchemaEvalConfig.model_construct(
-            data=data, metrics=metrics, component=component
+            data=data, metrics=metrics, component=component, version=version
         ).model_dump()
         return self._request("/deepeval/schema-eval", payload, raise_exception)
 
@@ -118,10 +122,11 @@ class DeepEvalClient:
         metrics: List[SchemaMetricConfig],
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Run background schema evaluation (async job)."""
         payload = SchemaEvalConfig.model_construct(
-            data=data, metrics=metrics, component=component
+            data=data, metrics=metrics, component=component, version=version
         ).model_dump()
         return self._request(
             "/deepeval/schema-eval/background", payload, raise_exception
@@ -134,11 +139,14 @@ class DeepEvalClient:
         chance: float,
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Randomly run text_eval based on a probability between 0 and 1."""
         self._validate_chance(chance)
         return (
-            self.text_eval(data, metrics, raise_exception, component=component)
+            self.text_eval(
+                data, metrics, raise_exception, component=component, version=version
+            )
             if random.random() <= chance
             else None
         )
@@ -150,12 +158,13 @@ class DeepEvalClient:
         chance: float,
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Randomly run text_eval_background based on a probability between 0 and 1."""
         self._validate_chance(chance)
         return (
             self.text_eval_background(
-                data, metrics, raise_exception, component=component
+                data, metrics, raise_exception, component=component, version=version
             )
             if random.random() <= chance
             else None
@@ -168,11 +177,14 @@ class DeepEvalClient:
         chance: float,
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Randomly run schema_eval based on a probability between 0 and 1."""
         self._validate_chance(chance)
         return (
-            self.schema_eval(data, metrics, raise_exception, component=component)
+            self.schema_eval(
+                data, metrics, raise_exception, component=component, version=version
+            )
             if random.random() <= chance
             else None
         )
@@ -184,12 +196,13 @@ class DeepEvalClient:
         chance: float,
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Randomly run text_eval_background based on a probability between 0 and 1."""
         self._validate_chance(chance)
         return (
             self.schema_eval_background(
-                data, metrics, raise_exception, component=component
+                data, metrics, raise_exception, component=component, version=version
             )
             if random.random() <= chance
             else None

rakam_eval_sdk/schema.py

@@ -39,7 +39,7 @@ class CorrectnessConfig(MetricConfigBase):
             "Minor formatting differences like '$1,250.00' vs '$1250.00' are acceptable."
         ]
     )
-    criteria: Optional[str] = None,
+    criteria: Optional[str] = (None,)
     params: List[Literal["actual_output", "expected_output"]] = Field(
         default=["actual_output", "expected_output"]
     )
@@ -94,8 +94,7 @@ MetricConfig = Annotated[
 ]
 
 SchemaMetricConfig = Annotated[
-    Union[JsonCorrectnessConfig, FieldsPresenceConfig], Field(
-        discriminator="type")
+    Union[JsonCorrectnessConfig, FieldsPresenceConfig], Field(discriminator="type")
 ]
 
 
@@ -118,11 +117,13 @@ class SchemaInputItem(InputItem):
 
 class EvalConfig(BaseModel):
     component: str = "unknown"
+    version: Union[str, None] = None
     data: List[TextInputItem]
     metrics: List[MetricConfig] = Field(default_factory=list)
 
 
 class SchemaEvalConfig(BaseModel):
     component: str = "unknown"
+    version: Union[str, None] = None
     data: List[SchemaInputItem]
     metrics: List[SchemaMetricConfig] = Field(default_factory=list)

{rakam_eval_sdk-0.1.15.dist-info → rakam_eval_sdk-0.1.16.dist-info}/METADATA

@@ -1,11 +1,12 @@
 Metadata-Version: 2.3
 Name: rakam-eval-sdk
-Version: 0.1.15
+Version: 0.1.16
 Summary: Evaluation Framework SDK
 Author: Mohamed Bachar Touil
 License: MIT
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: requests
+Requires-Dist: typer>=0.20.1
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 

rakam_eval_sdk-0.1.16.dist-info/RECORD

@@ -0,0 +1,8 @@
+rakam_eval_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rakam_eval_sdk/cli.py,sha256=dn1KXh-_VLpIvLFHnpHPfAL33ICoAJ9Y2iCOikkJcxY,277
+rakam_eval_sdk/client.py,sha256=q-Y11maLVKaEnq4OSyFCqrP3JgFS1xpyp9-bZhFssIA,7123
+rakam_eval_sdk/schema.py,sha256=MQfF0SEHf2wzeXJNTsMs-yDbN0vZJQbN_crfpPXsTk8,3467
+rakam_eval_sdk-0.1.16.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
+rakam_eval_sdk-0.1.16.dist-info/entry_points.txt,sha256=NzE2wDRB4Kt-TblkjSD37abcfP4B5STBOyygEhGTLdU,51
+rakam_eval_sdk-0.1.16.dist-info/METADATA,sha256=s7N_RsRR87-6aQhNmCnuDeMDrL6ZOm0vr7iR2cS5FwU,5959
+rakam_eval_sdk-0.1.16.dist-info/RECORD,,

rakam_eval_sdk-0.1.16.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+mycli = rakam_eval_sdk.cli:main
+

rakam_eval_sdk-0.1.15.dist-info/RECORD

@@ -1,6 +0,0 @@
-rakam_eval_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rakam_eval_sdk/client.py,sha256=EdYA8SFoq6PhO6JNxu_j2eJSd3g4I0rtUtGJmGgvfzA,6583
-rakam_eval_sdk/schema.py,sha256=FaY7nlcbzlFhH7lZl9iFfJ6T0wGVte7TYbt-w_wpFuI,3400
-rakam_eval_sdk-0.1.15.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
-rakam_eval_sdk-0.1.15.dist-info/METADATA,sha256=PhyFhXFiTeCt2KK_kBjGGXDI69q8qFmyg-aEiKh16OQ,5930
-rakam_eval_sdk-0.1.15.dist-info/RECORD,,