rakam-eval-sdk 0.1.15__py3-none-any.whl → 0.1.16rc1__py3-none-any.whl

rakam_eval_sdk/cli.py

@@ -0,0 +1,119 @@
+# cli.py
+from pathlib import Path
+
+import typer
+
+from rakam_eval_sdk.utils.decorator_utils import find_decorated_functions, load_module_from_path
+from rakam_eval_sdk.decorators import eval_run
+app = typer.Typer(help="CLI tools for evaluation utilities")
+
+
+@app.command()
+def find_eval_run_by_name(
+    directory: Path = typer.Argument(
+        Path("./eval"),
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+        help="Directory to scan (default: ./eval)",
+    ),
+    recursive: bool = typer.Option(
+        False,
+        "--recursive",
+        "-r",
+        help="Recursively search for Python files",
+    ),
+):
+    """
+    Find functions decorated with @track.
+    """
+    TARGET_DECORATOR = eval_run.__name__
+    files = (
+        directory.rglob("*.py")
+        if recursive
+        else directory.glob("*.py")
+    )
+
+    found = False
+
+    for file in sorted(files):
+        functions = find_decorated_functions(file, TARGET_DECORATOR)
+        for fn in functions:
+            found = True
+            typer.echo(f"{file}:{fn}")
+
+    if not found:
+        typer.echo(f"No @{TARGET_DECORATOR} functions found.")
+
+
+@app.command("run")
+def run_eval_runs(
+    directory: Path = typer.Argument(
+        Path("./eval"),
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+        help="Directory to scan (default: ./eval)",
+    ),
+    recursive: bool = typer.Option(
+        False,
+        "-r",
+        "--recursive",
+        help="Recursively search for Python files",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Only list functions without executing them",
+    ),
+):
+    """
+    Find and execute all functions decorated with @eval_run.
+    """
+    files = (
+        directory.rglob("*.py")
+        if recursive
+        else directory.glob("*.py")
+    )
+    TARGET_DECORATOR = eval_run.__name__
+
+    executed_any = False
+
+    for file in sorted(files):
+        functions = find_decorated_functions(file, TARGET_DECORATOR)
+        if not functions:
+            continue
+
+        typer.echo(f"\n📄 {file}")
+
+        module = None
+        if not dry_run:
+            try:
+                module = load_module_from_path(file)
+            except Exception as e:
+                typer.echo(f"  ❌ Failed to import module: {e}")
+                continue
+
+        for fn_name in functions:
+            typer.echo(f"  ▶ {fn_name}")
+
+            if dry_run:
+                continue
+
+            try:
+                func = getattr(module, fn_name)
+                func()  # <-- actual execution
+                executed_any = True
+            except Exception as e:
+                typer.echo(f"    ❌ Execution failed: {e}")
+
+    if not executed_any and not dry_run:
+        typer.echo("\nNo @eval_run functions executed.")
+
+
+def main():
+    app()
+
+
+if __name__ == "__main__":
+    main()

rakam_eval_sdk/client.py

@@ -1,6 +1,7 @@
 import os
 import random
-from typing import Any, List, Optional, cast
+from typing import Any, List, Optional, Union, cast
+
 import requests
 
 from .schema import (
@@ -79,10 +80,11 @@ class DeepEvalClient:
         metrics: List[MetricConfig],
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Run synchronous text evaluation."""
         payload = EvalConfig.model_construct(
-            data=data, metrics=metrics, component=component
+            data=data, metrics=metrics, component=component, version=version
         ).model_dump()
         return self._request("/deepeval/text-eval", payload, raise_exception)
 
@@ -92,10 +94,11 @@ class DeepEvalClient:
         metrics: List[MetricConfig],
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Run background text evaluation (async job)."""
         payload = EvalConfig.model_construct(
-            data=data, metrics=metrics, component=component
+            data=data, metrics=metrics, component=component, version=version
         ).model_dump()
         return self._request("/deepeval/text-eval/background", payload, raise_exception)
 
@@ -105,10 +108,11 @@ class DeepEvalClient:
         metrics: List[SchemaMetricConfig],
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Run synchronous schema evaluation."""
         payload = SchemaEvalConfig.model_construct(
-            data=data, metrics=metrics, component=component
+            data=data, metrics=metrics, component=component, version=version
         ).model_dump()
         return self._request("/deepeval/schema-eval", payload, raise_exception)
 
@@ -118,10 +122,11 @@ class DeepEvalClient:
         metrics: List[SchemaMetricConfig],
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Run background schema evaluation (async job)."""
         payload = SchemaEvalConfig.model_construct(
-            data=data, metrics=metrics, component=component
+            data=data, metrics=metrics, component=component, version=version
         ).model_dump()
         return self._request(
             "/deepeval/schema-eval/background", payload, raise_exception
@@ -134,11 +139,14 @@ class DeepEvalClient:
         chance: float,
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Randomly run text_eval based on a probability between 0 and 1."""
         self._validate_chance(chance)
         return (
-            self.text_eval(data, metrics, raise_exception, component=component)
+            self.text_eval(
+                data, metrics, raise_exception, component=component, version=version
+            )
             if random.random() <= chance
             else None
         )
@@ -150,12 +158,13 @@ class DeepEvalClient:
         chance: float,
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Randomly run text_eval_background based on a probability between 0 and 1."""
         self._validate_chance(chance)
         return (
             self.text_eval_background(
-                data, metrics, raise_exception, component=component
+                data, metrics, raise_exception, component=component, version=version
             )
             if random.random() <= chance
             else None
@@ -168,11 +177,14 @@ class DeepEvalClient:
         chance: float,
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Randomly run schema_eval based on a probability between 0 and 1."""
         self._validate_chance(chance)
         return (
-            self.schema_eval(data, metrics, raise_exception, component=component)
+            self.schema_eval(
+                data, metrics, raise_exception, component=component, version=version
+            )
             if random.random() <= chance
             else None
         )
@@ -184,12 +196,13 @@ class DeepEvalClient:
         chance: float,
         raise_exception: bool = False,
         component: str = "unknown",
+        version: Union[str, None] = None,
     ) -> Optional[dict]:
         """Randomly run text_eval_background based on a probability between 0 and 1."""
         self._validate_chance(chance)
         return (
             self.schema_eval_background(
-                data, metrics, raise_exception, component=component
+                data, metrics, raise_exception, component=component, version=version
             )
             if random.random() <= chance
             else None

rakam_eval_sdk/decorators.py

@@ -0,0 +1,44 @@
+
+import time
+import os
+import psutil
+import functools
+
+
+def eval_run(*dargs, **dkwargs):
+    def wrapper(func):
+        @functools.wraps(func)
+        def inner(*args, **kwargs):
+            process = psutil.Process(os.getpid())
+
+            # Start metrics
+            start_time = time.perf_counter()
+            start_cpu = process.cpu_times()
+            start_mem = process.memory_info().rss
+
+            try:
+                result = func(*args, **kwargs)
+                return result
+            finally:
+                # End metrics
+                end_time = time.perf_counter()
+                end_cpu = process.cpu_times()
+                end_mem = process.memory_info().rss
+
+                elapsed = end_time - start_time
+                cpu_used = (
+                    (end_cpu.user + end_cpu.system)
+                    - (start_cpu.user + start_cpu.system)
+                )
+                mem_diff_mb = (end_mem - start_mem) / (1024 * 1024)
+
+                print(
+                    f"[eval_run] {func.__module__}.{func.__name__} | "
+                    f"time={elapsed:.4f}s | "
+                    f"cpu={cpu_used:.4f}s | "
+                    f"mem_delta={mem_diff_mb:.2f}MB"
+                )
+
+        return inner
+
+    return wrapper

rakam_eval_sdk/schema.py

@@ -39,7 +39,7 @@ class CorrectnessConfig(MetricConfigBase):
             "Minor formatting differences like '$1,250.00' vs '$1250.00' are acceptable."
         ]
     )
-    criteria: Optional[str] = None,
+    criteria: Optional[str] = (None,)
     params: List[Literal["actual_output", "expected_output"]] = Field(
         default=["actual_output", "expected_output"]
     )
@@ -94,8 +94,7 @@ MetricConfig = Annotated[
 ]
 
 SchemaMetricConfig = Annotated[
-    Union[JsonCorrectnessConfig, FieldsPresenceConfig], Field(
-        discriminator="type")
+    Union[JsonCorrectnessConfig, FieldsPresenceConfig], Field(discriminator="type")
 ]
 
 
@@ -118,11 +117,13 @@ class SchemaInputItem(InputItem):
 
 class EvalConfig(BaseModel):
     component: str = "unknown"
+    version: Union[str, None] = None
     data: List[TextInputItem]
     metrics: List[MetricConfig] = Field(default_factory=list)
 
 
 class SchemaEvalConfig(BaseModel):
     component: str = "unknown"
+    version: Union[str, None] = None
     data: List[SchemaInputItem]
     metrics: List[SchemaMetricConfig] = Field(default_factory=list)

rakam_eval_sdk/utils/decorator_utils.py

@@ -0,0 +1,69 @@
+import ast
+import importlib
+import importlib.util
+from dataclasses import dataclass
+from pathlib import Path
+from types import ModuleType
+from typing import Callable, Iterable, List, Tuple
+
+
+class DecoratedFunctionVisitor(ast.NodeVisitor):
+    def __init__(self, decorator_name: str):
+        self.decorator_name = decorator_name
+        self.results: List[str] = []
+
+    def visit_FunctionDef(self, node: ast.FunctionDef):
+        for deco in node.decorator_list:
+            if self._matches(deco):
+                self.results.append(node.name)
+        self.generic_visit(node)
+
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef):
+        for deco in node.decorator_list:
+            if self._matches(deco):
+                self.results.append(node.name)
+        self.generic_visit(node)
+
+    def _matches(self, deco: ast.expr) -> bool:
+        # @deco
+        if isinstance(deco, ast.Name):
+            return deco.id == self.decorator_name
+
+        # @module.deco
+        if isinstance(deco, ast.Attribute):
+            return deco.attr == self.decorator_name
+
+        # @deco(...)
+        if isinstance(deco, ast.Call):
+            return self._matches(deco.func)
+
+        return False
+
+
+def find_decorated_functions(
+    file_path: Path,
+    decorator_name: str,
+) -> List[str]:
+    tree = ast.parse(file_path.read_text(encoding="utf-8"))
+    visitor = DecoratedFunctionVisitor(decorator_name)
+    visitor.visit(tree)
+    return visitor.results
+
+
+def load_module_from_path(file_path: Path) -> ModuleType:
+    spec = importlib.util.spec_from_file_location(file_path.stem, file_path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot import {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+
+
+
+def get_function(module: ModuleType, function_name: str) -> Callable:
+    func = getattr(module, function_name, None)
+    if func is None:
+        raise AttributeError(f"{function_name} not found in {module.__name__}")
+    return func

{rakam_eval_sdk-0.1.15.dist-info → rakam_eval_sdk-0.1.16rc1.dist-info}/METADATA

@@ -1,11 +1,13 @@
 Metadata-Version: 2.3
 Name: rakam-eval-sdk
-Version: 0.1.15
+Version: 0.1.16rc1
 Summary: Evaluation Framework SDK
 Author: Mohamed Bachar Touil
 License: MIT
+Requires-Dist: psutil>=7.2.1
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: requests
+Requires-Dist: typer>=0.20.1
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 

rakam_eval_sdk-0.1.16rc1.dist-info/RECORD

@@ -0,0 +1,10 @@
+rakam_eval_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rakam_eval_sdk/cli.py,sha256=9BHZte3cS1LWL0_dOVEtws9xIhdw0yORW93Dm1uDxDw,2876
+rakam_eval_sdk/client.py,sha256=q-Y11maLVKaEnq4OSyFCqrP3JgFS1xpyp9-bZhFssIA,7123
+rakam_eval_sdk/decorators.py,sha256=ZEcZb2KUsPrtx-Guc7tYN9MVCMxIQ83yhiJxKE1fjdw,1262
+rakam_eval_sdk/schema.py,sha256=MQfF0SEHf2wzeXJNTsMs-yDbN0vZJQbN_crfpPXsTk8,3467
+rakam_eval_sdk/utils/decorator_utils.py,sha256=hCC4F7v3KjGSDt2NUXfDsbBTMPzlG6wMzZVdR_wWn14,2048
+rakam_eval_sdk-0.1.16rc1.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
+rakam_eval_sdk-0.1.16rc1.dist-info/entry_points.txt,sha256=tNhwmM_UGELb3h0zOfgCrtTheUkP-k8jGv0rTOfRSps,56
+rakam_eval_sdk-0.1.16rc1.dist-info/METADATA,sha256=DRKzVNNF426R3ipnpG8Xr5LXKLTY4Ar9WdPIxe6hjzI,5991
+rakam_eval_sdk-0.1.16rc1.dist-info/RECORD,,

rakam_eval_sdk-0.1.16rc1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+rakam_eval = rakam_eval_sdk.cli:main
+

rakam_eval_sdk-0.1.15.dist-info/RECORD

@@ -1,6 +0,0 @@
-rakam_eval_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rakam_eval_sdk/client.py,sha256=EdYA8SFoq6PhO6JNxu_j2eJSd3g4I0rtUtGJmGgvfzA,6583
-rakam_eval_sdk/schema.py,sha256=FaY7nlcbzlFhH7lZl9iFfJ6T0wGVte7TYbt-w_wpFuI,3400
-rakam_eval_sdk-0.1.15.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
-rakam_eval_sdk-0.1.15.dist-info/METADATA,sha256=PhyFhXFiTeCt2KK_kBjGGXDI69q8qFmyg-aEiKh16OQ,5930
-rakam_eval_sdk-0.1.15.dist-info/RECORD,,