rakam-eval-sdk 0.2.0rc2__py3-none-any.whl → 0.2.1__py3-none-any.whl

rakam_eval_sdk/cli.py

@@ -6,7 +6,7 @@ import uuid
 from datetime import datetime
 from pathlib import Path
 from pprint import pprint
-from typing import Any, Optional
+from typing import Any, Dict, Optional, Sequence
 
 import typer
 from dotenv import load_dotenv
@@ -31,6 +31,60 @@ if PROJECT_ROOT not in sys.path:
     sys.path.insert(0, PROJECT_ROOT)
 list_app = typer.Typer(help="List resources")
 app.add_typer(list_app, name="list")
+metrics_app = typer.Typer(help="Metrics utilities")
+app.add_typer(metrics_app, name="metrics")
+
+
+@metrics_app.command("list")
+def list_metrics(
+    limit: int = typer.Option(
+        20,
+        "--limit",
+        help="Number of testcases to inspect for metrics",
+    ),
+):
+    """
+    List unique metric names found in evaluation testcases.
+    """
+    client = DeepEvalClient()
+
+    testcases = client.list_evaluation_testcases(
+        limit=limit,
+        offset=0,
+        raise_exception=True,
+    )
+
+    if not testcases:
+        typer.echo("No evaluation testcases found.")
+        return
+
+    metric_names: set[str] = set()
+
+    def collect_metrics(entries: Sequence[Dict] | None):
+        if not entries:
+            return
+        for entry in entries:
+            for metric in entry.get("metrics", []) or []:
+                name = metric.get("name")
+                if name:
+                    metric_names.add(name)
+
+    for tc in testcases:
+
+        collect_metrics(tc.get("result"))
+
+    if not metric_names:
+        typer.echo("No metrics found.")
+        return
+
+    typer.echo(
+        f"📊 Found {len(metric_names)} unique metrics "
+        f"(from latest {limit} testcases)\n"
+    )
+
+    for name in sorted(metric_names):
+        typer.echo(f"- {name}")
+
 
 @list_app.command("eval")
 def list(
@@ -66,16 +120,10 @@ def list(
         typer.echo(f"No @{TARGET_DECORATOR} functions found.")
 
 
-
-
-
 @list_app.command("runs")
 def list_runs(
     limit: int = typer.Option(20, help="Max number of runs"),
     offset: int = typer.Option(0, help="Pagination offset"),
-    status: Optional[str] = typer.Option(
-        None, help="Filter by status (running, completed, failed)"
-    ),
 ):
     """
     List evaluation runs (newest first).
@@ -92,19 +140,7 @@ def list_runs(
         typer.echo("No evaluation runs found.")
         return
 
-    # optional status filtering (client-side for now)
-    if status:
-        runs = [
-            r for r in runs
-            if r.get("result", {}).get("status") == status
-        ]
-
-    typer.echo(
-        f"[id] "
-        f"{'unique_id':<20}"
-        f"{'label':<20}"
-        f"created_at"
-    )
+    typer.echo(f"[id] " f"{'unique_id':<20}" f"{'label':<20}" f"created_at")
     # pretty CLI output
     for run in runs:
         run_id = run.get("id")
@@ -120,12 +156,7 @@ def list_runs(
             except ValueError:
                 pass
 
-        typer.echo(
-            f"[{run_id}] "
-            f"{uid:<20} "
-            f"{label:<20} "
-            f"{created_at}"
-        )
+        typer.echo(f"[{run_id}] " f"{uid:<20} " f"{label:<20} " f"{created_at}")
 
 
 @list_app.command("show")
@@ -345,8 +376,7 @@ def _print_and_save(
         return
 
     if out.exists() and not overwrite:
-        typer.echo(
-            f"❌ File already exists: {out} (use --overwrite to replace)")
+        typer.echo(f"❌ File already exists: {out} (use --overwrite to replace)")
         raise typer.Exit(code=1)
 
     out.parent.mkdir(parents=True, exist_ok=True)

rakam_eval_sdk/client.py

@@ -27,19 +27,19 @@ class DeepEvalClient:
         settings_module: Optional[Any] = None,  # optional external settings
         timeout: int = 30,
     ):
-        settings_url = getattr(settings_module, "EVALFRAMWORK_URL", None)
+        settings_url = getattr(settings_module, "EVALFRAMEWORK_URL", None)
         settings_token = getattr(settings_module, "EVALFRAMWORK_API_KEY", None)
 
         raw_url = (
             base_url
             or settings_url
-            or os.getenv("EVALFRAMWORK_URL")
+            or os.getenv("EVALFRAMEWORK_URL")
             or "http://localhost:8080"
         )
         self.base_url = raw_url.rstrip("/")
         self.api_token = (
             api_token or settings_token or os.getenv(
-                "EVALFRAMWORK_API_KEY", "")
+                "EVALFRAMEWORK_API_KEY", "")
         )
         self.timeout = timeout
 

rakam_eval_sdk/schema.py

@@ -94,8 +94,7 @@ MetricConfig = Annotated[
 ]
 
 SchemaMetricConfig = Annotated[
-    Union[JsonCorrectnessConfig, FieldsPresenceConfig], Field(
-        discriminator="type")
+    Union[JsonCorrectnessConfig, FieldsPresenceConfig], Field(discriminator="type")
 ]
 
 

{rakam_eval_sdk-0.2.0rc2.dist-info → rakam_eval_sdk-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rakam-eval-sdk
-Version: 0.2.0rc2
+Version: 0.2.1
 Summary: Evaluation Framework SDK
 Author: Mohamed Bachar Touil
 License: MIT
@@ -94,6 +94,7 @@ client = DeepEvalClient(
             )
 
 ```
+
 3. Schema Evaluation
 
 ```python
@@ -137,6 +138,7 @@ client = DeepEvalClient(
             )
 
 ```
+
 ## Configuration
 
 The client can be configured in multiple ways:
@@ -150,7 +152,7 @@ DeepEvalClient(base_url="http://api", api_token="123")
 ### Environment variables
 
 ```bash
-export EVALFRAMWORK_URL=http://api
+export EVALFRAMEWORK_URL=http://api
 export EVALFRAMWORK_API_KEY=123
 ```
 

rakam_eval_sdk-0.2.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+rakam_eval_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rakam_eval_sdk/cli.py,sha256=0Ym7C83xirGEf0YCh4-agwfjYPJ3yieeS02nXJ_iE-Y,14486
+rakam_eval_sdk/client.py,sha256=XExVLDzk9Ed3zW5uezAxs5KLROiTODrdmXEcw0g7IE4,13080
+rakam_eval_sdk/decorators.py,sha256=_9VFQmoYWd6cqnNryZJWEwYHQRxY7vIOam4z45zBk3c,1794
+rakam_eval_sdk/schema.py,sha256=P4LlnaInXWTq-ve6qPTTxPyzmj3j_1gcqV9i7CYRYec,3608
+rakam_eval_sdk/utils/decorator_utils.py,sha256=g0TjXtG9o4hwhUAFP8GJsXAkjhZhzeseTAg-YBFjj2g,1763
+rakam_eval_sdk-0.2.1.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
+rakam_eval_sdk-0.2.1.dist-info/entry_points.txt,sha256=tNhwmM_UGELb3h0zOfgCrtTheUkP-k8jGv0rTOfRSps,56
+rakam_eval_sdk-0.2.1.dist-info/METADATA,sha256=u-k-VOQD7BzOaWC0itLYUyQd-55ujT1IdJCma2Y4gf8,6019
+rakam_eval_sdk-0.2.1.dist-info/RECORD,,

rakam_eval_sdk-0.2.0rc2.dist-info/RECORD

@@ -1,10 +0,0 @@
-rakam_eval_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rakam_eval_sdk/cli.py,sha256=WNpfbUJvoAt4IuH9ZKSrzq4bqHa0QZir5g0M9FkJmw8,13601
-rakam_eval_sdk/client.py,sha256=4qUG8cLGqY8026s28uCHM3zFuGDzekLokZZDu7VRJ_8,13077
-rakam_eval_sdk/decorators.py,sha256=_9VFQmoYWd6cqnNryZJWEwYHQRxY7vIOam4z45zBk3c,1794
-rakam_eval_sdk/schema.py,sha256=ozNC56ygzR1G6UABjnqnJVAPVcF4rJMH1pUNH0a1K4M,3617
-rakam_eval_sdk/utils/decorator_utils.py,sha256=g0TjXtG9o4hwhUAFP8GJsXAkjhZhzeseTAg-YBFjj2g,1763
-rakam_eval_sdk-0.2.0rc2.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
-rakam_eval_sdk-0.2.0rc2.dist-info/entry_points.txt,sha256=tNhwmM_UGELb3h0zOfgCrtTheUkP-k8jGv0rTOfRSps,56
-rakam_eval_sdk-0.2.0rc2.dist-info/METADATA,sha256=IokdzEVaNyxqiXYAywxGAAolJA9JI1nIXQxSd7c-bEU,6019
-rakam_eval_sdk-0.2.0rc2.dist-info/RECORD,,