mwin 0.2.2__tar.gz → 0.2.4__tar.gz

{mwin-0.2.2 → mwin-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mwin
-Version: 0.2.2
+Version: 0.2.4
 Summary: Track OpenAI, Claude, Gemini and OpenAI-compatible models then give solutions to improve your agent system.
 Author-email: yanghui <dasss90ovo@gmail.com>
 Project-URL: Homepage, https://github.com/yanghui1-arch/mwin.git
@@ -73,14 +73,14 @@ It's the most recommended method to use mwin to track the trace in a project. mw
 
 ### Demo
 ```python
-from mwin import track, StepType, start_trace
+from mwin import track, start_trace
 from openai import OpenAI
 
 openai_apikey = "<llm_api_key>"
 openai_base_url = "<llm_base_url>"
 model = "<llm_model>"
 
-@track(step_type=StepType.TOOL)
+@track(step_type="tool")
 def execute_bash(command: str):
     # assume execute bash and get a stdout
     return "<bash_stdout>"

{mwin-0.2.2 → mwin-0.2.4}/README.md

@@ -56,14 +56,14 @@ It's the most recommended method to use mwin to track the trace in a project. mw
 
 ### Demo
 ```python
-from mwin import track, StepType, start_trace
+from mwin import track, start_trace
 from openai import OpenAI
 
 openai_apikey = "<llm_api_key>"
 openai_base_url = "<llm_base_url>"
 model = "<llm_model>"
 
-@track(step_type=StepType.TOOL)
+@track(step_type="tool")
 def execute_bash(command: str):
     # assume execute bash and get a stdout
     return "<bash_stdout>"

{mwin-0.2.2 → mwin-0.2.4}/mwin/__init__.py

@@ -1,16 +1,15 @@
-from .track import track
-from .models import LLMProvider, StepType, Step, Trace, Conversation
-from .context import start_trace, start_trace_async
-from .prompt.wrapper import template_prompt
-
-__all__ = [
-    "track",
-    "start_trace",
-    "start_trace_async",
-    "template_prompt",
-    "LLMProvider",
-    "StepType",
-    "Step",
-    "Trace",
-    "Conversation",
+from .track import track
+from .models import LLMProvider, Step, Trace, Conversation
+from .context import start_trace, start_trace_async
+from . import eval
+
+__all__ = [
+    "track",
+    "start_trace",
+    "start_trace_async",
+    "LLMProvider",
+    "Step",
+    "Trace",
+    "Conversation",
+    "eval",
 ]

{mwin-0.2.2 → mwin-0.2.4}/mwin/client/schemas/request/log_request.py

@@ -62,9 +62,6 @@ class LogStepRequest(BaseModel):
     prompt_version: str | None = None
     """Prompt version"""
 
-    system_prompt: str | None = None
-    """Raw system prompt string captured via template_prompt() inside the step"""
-
     @field_serializer('input', 'output')
     def serialize_any_field(self, value: Any):
         return serialize_helper.safe_serialize(value)
@@ -93,4 +90,5 @@ class LogTraceRequest(BaseModel):
     
     @field_serializer("start_time", "last_update_timestamp")
     def serialize_datetime(self, value: datetime):
-        return value.strftime("%Y-%m-%d %H:%M:%S.%f")
+        return value.strftime("%Y-%m-%d %H:%M:%S.%f")
+    

{mwin-0.2.2 → mwin-0.2.4}/mwin/client/sync_client.py

@@ -58,7 +58,6 @@ class SyncClient:
         llm_provider: LLMProvider,
         pipeline: str | None,
         prompt_name: str | None,
-        system_prompt: str | None,
         prompt_version: str | None,
     ) -> LogStepResponse:
         """Create a step and log it in server."""
@@ -68,12 +67,6 @@ class SyncClient:
             parent_step_id = None
 
         llm_provider = llm_provider.value
-        # prompt_version not set -> don't pass system_prompt.
-        # TODO: Give a warn
-        if prompt_version is None:
-            pipeline = None
-            system_prompt = None
-            prompt_name = None
 
         log_step_req = LogStepRequest(
             project_name=self._project_name,
@@ -94,7 +87,6 @@ class SyncClient:
             llm_provider=llm_provider,
             prompt_pipeline=pipeline,
             prompt_name=prompt_name,
-            system_prompt=system_prompt,
             prompt_version=prompt_version,
         )
 

{mwin-0.2.2 → mwin-0.2.4}/mwin/context/runner.py

@@ -22,13 +22,13 @@ It works well at beginning of project but not recommend it for developer. it's v
 Please use following demo to develop with mwin. It's more safe, easier and more controllable.
 
 ```python
-from mwin import track, StepType
+from mwin import track
 from mwin.runner import start_trace
 @track()
 def call_openai() -> Dict:
     ...
 
-@track(StepType.TOOL)
+@track(step_type="tool")
 def execute_bash(*args, **kwargs)
     ...
 

mwin-0.2.4/mwin/eval/__init__.py

@@ -0,0 +1,26 @@
+from .llm_as_judge import JudgeResult, llm_as_judge
+from .metric import (
+    accuracy,
+    evaluate,
+    f1,
+    get_metric,
+    list_metrics,
+    recall,
+    register_metric,
+    unregister_metric,
+)
+from . import metric
+
+__all__ = [
+    "llm_as_judge",
+    "JudgeResult",
+    "accuracy",
+    "f1",
+    "recall",
+    "evaluate",
+    "register_metric",
+    "unregister_metric",
+    "get_metric",
+    "list_metrics",
+    "metric",
+]

mwin-0.2.4/mwin/eval/llm_as_judge.py

@@ -0,0 +1,148 @@
+"""Simple LLM-as-judge utility."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from openai import OpenAI
+
+from ..helper import serialize_helper
+from ..models.common import LLMProvider
+from ..track import track
+
+
+@dataclass(frozen=True)
+class JudgeResult:
+    score: float
+    reasoning: str
+    raw_response: str
+
+
+def llm_as_judge(
+    *,
+    input_data: Any,
+    output_data: Any,
+    metric_name: str,
+    judge_prompt: str,
+    model: str = "gpt-4o-mini",
+    client: Any | None = None,
+    api_key: str | None = None,
+    base_url: str | None = None,
+    score_range_min: float = 0.0,
+    score_range_max: float = 1.0,
+    track_judge_call: bool = True,
+    judge_project_name: str = "mwin_judge",
+    llm_provider: LLMProvider = LLMProvider.OPENAI,
+) -> JudgeResult:
+    """Evaluate an output with an LLM judge and return score/reasoning."""
+    if len(metric_name.strip()) == 0:
+        raise ValueError("metric_name must be non-empty.")
+    if len(judge_prompt.strip()) == 0:
+        raise ValueError("judge_prompt must be non-empty.")
+    if score_range_min >= score_range_max:
+        raise ValueError(
+            "score_range_min must be smaller than score_range_max. "
+            f"Got {score_range_min} and {score_range_max}."
+        )
+
+    judge_client = client or OpenAI(api_key=api_key, base_url=base_url)
+
+    prompt = (
+        "You are evaluating an LLM output.\n\n"
+        f"Evaluation criterion: {metric_name}\n"
+        "Scoring guide:\n"
+        f"{judge_prompt}\n\n"
+        f"Score range: {score_range_min} (worst) to {score_range_max} (best)\n\n"
+        "=== Input ===\n"
+        f"{_dump_text(input_data)}\n\n"
+        "=== Output ===\n"
+        f"{_dump_text(output_data)}\n\n"
+        "Reply with ONLY valid JSON, no markdown fences:\n"
+        '{"score": <number within range>, "reasoning": "<one concise sentence>"}'
+    )
+
+    def _judge_call() -> Any:
+        return judge_client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0,
+            max_tokens=256,
+        )
+
+    if track_judge_call:
+        response = track(
+            project_name=judge_project_name,
+            tags=["eval", "llm_as_judge"],
+            step_type="llm",
+            model=model,
+            llm_provider=llm_provider,
+            description="mwin.eval.llm_as_judge",
+        )(_judge_call)()
+    else:
+        response = _judge_call()
+
+    raw_text = _extract_response_text(response)
+    payload = _parse_payload(raw_text)
+
+    if "score" not in payload:
+        raise ValueError("Judge response JSON must include 'score'.")
+    if "reasoning" not in payload:
+        raise ValueError("Judge response JSON must include 'reasoning'.")
+
+    score = float(payload["score"])
+    if score < score_range_min or score > score_range_max:
+        raise ValueError(
+            f"Judge score out of range: {score}. "
+            f"Expected within [{score_range_min}, {score_range_max}]."
+        )
+
+    return JudgeResult(
+        score=score,
+        reasoning=str(payload["reasoning"]).strip(),
+        raw_response=raw_text,
+    )
+
+
+def _dump_text(value: Any) -> str:
+    serialized = serialize_helper.safe_serialize(value)
+    if isinstance(serialized, str):
+        return serialized
+    return json.dumps(serialized, ensure_ascii=False, indent=2)
+
+
+def _extract_response_text(response: Any) -> str:
+    try:
+        content = response.choices[0].message.content
+    except Exception as exc:
+        raise ValueError("Unable to read judge response content.") from exc
+
+    if content is None:
+        raise ValueError("Judge response content is empty.")
+    return str(content).strip()
+
+
+def _parse_payload(raw_text: str) -> dict[str, Any]:
+    text = raw_text.strip()
+
+    fenced_match = re.search(r"```(?:json)?\\s*(.*?)\\s*```", text, flags=re.DOTALL | re.IGNORECASE)
+    if fenced_match:
+        text = fenced_match.group(1).strip()
+
+    try:
+        parsed = json.loads(text)
+        if isinstance(parsed, dict):
+            return parsed
+    except json.JSONDecodeError:
+        pass
+
+    json_object_match = re.search(r"\{.*\}", text, flags=re.DOTALL)
+    if not json_object_match:
+        raise ValueError(f"Judge response is not valid JSON: {raw_text}")
+
+    parsed = json.loads(json_object_match.group(0))
+    if not isinstance(parsed, dict):
+        raise ValueError("Judge response JSON must be an object.")
+    return parsed

mwin-0.2.4/mwin/eval/metric.py

@@ -0,0 +1,187 @@
+"""Simple evaluation metrics for mwin."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Iterable, Mapping, Sequence
+
+MetricFunc = Callable[[Sequence[Any], Sequence[Any]], float]
+
+
+def _to_list(values: Sequence[Any] | Iterable[Any], *, field_name: str) -> list[Any]:
+    if isinstance(values, (str, bytes, bytearray)):
+        raise TypeError(f"{field_name} must be a sequence of labels, not string-like data.")
+
+    try:
+        return list(values)
+    except TypeError as exc:
+        raise TypeError(f"{field_name} must be iterable.") from exc
+
+
+def _validate_pair_inputs(
+    y_true: Sequence[Any] | Iterable[Any],
+    y_pred: Sequence[Any] | Iterable[Any],
+) -> tuple[list[Any], list[Any]]:
+    true_list = _to_list(y_true, field_name="y_true")
+    pred_list = _to_list(y_pred, field_name="y_pred")
+
+    if len(true_list) != len(pred_list):
+        raise ValueError(
+            f"y_true and y_pred must have the same length. "
+            f"Got {len(true_list)} and {len(pred_list)}."
+        )
+    if len(true_list) == 0:
+        raise ValueError("y_true and y_pred must not be empty.")
+
+    return true_list, pred_list
+
+
+def accuracy(
+    y_true: Sequence[Any] | Iterable[Any],
+    y_pred: Sequence[Any] | Iterable[Any],
+) -> float:
+    """Accuracy = correct / total."""
+    true_list, pred_list = _validate_pair_inputs(y_true, y_pred)
+    correct_count = sum(1 for target, prediction in zip(true_list, pred_list) if target == prediction)
+    return correct_count / len(true_list)
+
+
+def recall(
+    y_true: Sequence[Any] | Iterable[Any],
+    y_pred: Sequence[Any] | Iterable[Any],
+    *,
+    positive_label: Any = 1,
+) -> float:
+    """Binary recall for a positive label."""
+    true_list, pred_list = _validate_pair_inputs(y_true, y_pred)
+
+    true_positive = 0
+    false_negative = 0
+    for target, prediction in zip(true_list, pred_list):
+        if target == positive_label and prediction == positive_label:
+            true_positive += 1
+        elif target == positive_label and prediction != positive_label:
+            false_negative += 1
+
+    denominator = true_positive + false_negative
+    return (true_positive / denominator) if denominator > 0 else 0.0
+
+
+def f1(
+    y_true: Sequence[Any] | Iterable[Any],
+    y_pred: Sequence[Any] | Iterable[Any],
+    *,
+    positive_label: Any = 1,
+) -> float:
+    """Binary F1 for a positive label."""
+    true_list, pred_list = _validate_pair_inputs(y_true, y_pred)
+
+    true_positive = 0
+    false_positive = 0
+    false_negative = 0
+
+    for target, prediction in zip(true_list, pred_list):
+        if target == positive_label and prediction == positive_label:
+            true_positive += 1
+        elif target != positive_label and prediction == positive_label:
+            false_positive += 1
+        elif target == positive_label and prediction != positive_label:
+            false_negative += 1
+
+    precision_denominator = true_positive + false_positive
+    recall_denominator = true_positive + false_negative
+    precision_value = (true_positive / precision_denominator) if precision_denominator > 0 else 0.0
+    recall_value = (true_positive / recall_denominator) if recall_denominator > 0 else 0.0
+
+    denominator = precision_value + recall_value
+    return (2 * precision_value * recall_value / denominator) if denominator > 0 else 0.0
+
+
+_BUILTIN_METRICS: dict[str, MetricFunc] = {
+    "accuracy": accuracy,
+    "f1": f1,
+    "recall": recall,
+}
+
+_CUSTOM_METRICS: dict[str, MetricFunc] = {}
+
+
+def register_metric(name: str, metric_fn: MetricFunc, *, overwrite: bool = False) -> None:
+    """Register a custom metric globally."""
+    if not isinstance(name, str) or len(name.strip()) == 0:
+        raise ValueError("Metric name must be a non-empty string.")
+    if not callable(metric_fn):
+        raise TypeError("metric_fn must be callable.")
+
+    normalized_name = name.strip().lower()
+    if not overwrite and normalized_name in _CUSTOM_METRICS:
+        raise ValueError(
+            f"Metric '{normalized_name}' already exists. "
+            "Set overwrite=True to replace it."
+        )
+    _CUSTOM_METRICS[normalized_name] = metric_fn
+
+
+def unregister_metric(name: str) -> None:
+    """Remove a custom metric from global registry."""
+    _CUSTOM_METRICS.pop(name.strip().lower(), None)
+
+
+def get_metric(name: str) -> MetricFunc:
+    """Get metric by name (custom has higher priority than built-in)."""
+    normalized_name = name.strip().lower()
+
+    if normalized_name in _CUSTOM_METRICS:
+        return _CUSTOM_METRICS[normalized_name]
+    if normalized_name in _BUILTIN_METRICS:
+        return _BUILTIN_METRICS[normalized_name]
+
+    raise KeyError(f"Metric '{normalized_name}' does not exist.")
+
+
+def list_metrics() -> list[str]:
+    """List all metric names."""
+    names = set(_BUILTIN_METRICS.keys())
+    names.update(_CUSTOM_METRICS.keys())
+    return sorted(names)
+
+
+def evaluate(
+    y_true: Sequence[Any] | Iterable[Any],
+    y_pred: Sequence[Any] | Iterable[Any],
+    metrics: Sequence[str | MetricFunc] | None = None,
+    *,
+    custom_metrics: Mapping[str, MetricFunc] | None = None,
+) -> dict[str, float]:
+    """Evaluate one or more metrics.
+
+    Defaults to ["accuracy", "f1", "recall"].
+    """
+    true_list, pred_list = _validate_pair_inputs(y_true, y_pred)
+    targets = list(metrics) if metrics is not None else ["accuracy", "f1", "recall"]
+
+    merged_custom: dict[str, MetricFunc] = dict(_CUSTOM_METRICS)
+    if custom_metrics is not None:
+        for name, metric_fn in custom_metrics.items():
+            if not callable(metric_fn):
+                raise TypeError(f"custom_metrics['{name}'] must be callable.")
+            merged_custom[name.strip().lower()] = metric_fn
+
+    results: dict[str, float] = {}
+    for index, target in enumerate(targets):
+        if isinstance(target, str):
+            normalized_name = target.strip().lower()
+            metric_fn = merged_custom.get(normalized_name) or _BUILTIN_METRICS.get(normalized_name)
+            if metric_fn is None:
+                raise KeyError(f"Metric '{normalized_name}' does not exist.")
+            metric_name = normalized_name
+        elif callable(target):
+            metric_fn = target
+            metric_name = getattr(target, "__name__", f"custom_metric_{index}")
+        else:
+            raise TypeError(
+                "Each metric must be either a metric name string or a callable metric function."
+            )
+
+        results[metric_name] = float(metric_fn(true_list, pred_list))
+
+    return results

{mwin-0.2.2 → mwin-0.2.4}/mwin/helper/args_helper.py

@@ -1,10 +1,10 @@
 from uuid import UUID
-from typing import Any, List, Dict
+from typing import Any, List, Dict, Literal
 
 from pydantic import BaseModel
 from openai.types.completion_usage import CompletionUsage
 from . import id_helper
-from ..models import Step, Trace, StepType
+from ..models import Step, Trace
 from .. import context
 
 class StartArguments(BaseModel):
@@ -27,7 +27,7 @@ def create_new_step(
     input: Any | None = None,
     output: Dict[str, Any] | None = None,
     name: str | None = None,
-    type: StepType = StepType.GENERAL,
+    type: Literal["general", "llm", "retrieve", "tool"] = "general",
     tags: List[str] | None = None,
     model: str | None = None,
     usage: int | None = None,
@@ -45,7 +45,7 @@ def create_new_step(
         output(Dict[str, Any] | None): output of module. Default to `None`. None means it's logging input.
         name(str | None): the step name. Caller can set the name to define what the step role is. Default to ``None`. If it's None,
                                         AITrace will set step name based on step type.
-        type(StepType): step type. Default to `StepType.GENERAL`.
+        type(Literal["general", "llm", "retrieve", "tool"]): step type. Default to `general`.
         tags(List[str] | None): step tags. Default to `None`. If it's None, it will be set an empty list.
         model(str | None): model name. Probably using a llm model in the step. Default to `None`.
         usage(int | None): llm token usage. Default to `None`.

mwin-0.2.4/mwin/helper/prompt_helper.py

@@ -0,0 +1,26 @@
+from typing import Tuple
+
+
+def parse_system_prompt_identifier(system_prompt: str | None) -> Tuple[str | None, str | None, str | None]:
+    """Parse system prompt identifier in `pipeline/name@version` format.
+
+    Returns `(pipeline, prompt_name, prompt_version)`.
+    Returns `(None, None, None)` when `system_prompt` is None.
+    """
+
+    if system_prompt is None:
+        return None, None, None
+
+    identifier = system_prompt.strip()
+    if not identifier or identifier.count("@") != 1:
+        raise ValueError("Invalid system_prompt. Expected format: pipeline/name@version")
+
+    path, prompt_version = identifier.rsplit("@", 1)
+    if not prompt_version or "/" not in path:
+        raise ValueError("Invalid system_prompt. Expected format: pipeline/name@version")
+
+    pipeline, prompt_name = path.split("/", 1)
+    if not pipeline or not prompt_name:
+        raise ValueError("Invalid system_prompt. Expected format: pipeline/name@version")
+
+    return pipeline, prompt_name, prompt_version