langwatch 0.2.8__py3-none-any.whl → 0.2.9__py3-none-any.whl

langwatch/__version__.py

@@ -1,3 +1,3 @@
 """Version information for LangWatch."""
 
-__version__ = "0.2.8"
+__version__ = "0.2.9"

langwatch/dspy/__init__.py

@@ -28,6 +28,7 @@ from pydantic.fields import FieldInfo
 from coolname import generate_slug
 from retry import retry
 from dspy.evaluate.evaluate import Evaluate
+from dspy.utils.callback import with_callbacks
 
 
 class SerializableAndPydanticEncoder(json.JSONEncoder):
@@ -72,7 +73,16 @@ class SerializableAndPydanticEncoder(json.JSONEncoder):
         if isinstance(o, Completions):
             return {"__class__": classname} | o.__dict__
         if isinstance(o, BaseModel):
-            return o.model_dump(exclude_unset=True)
+            try:
+                return o.model_dump(exclude_unset=True)
+            except Exception as e:
+                if 'MockValSer' in str(e):
+                    return {
+                        key: getattr(o, key)
+                        for key in o.model_fields.keys()
+                        if hasattr(o, key) and getattr(o, key) is not None
+                    }
+                raise
         try:
             return super().default(o)
         except:
@@ -152,11 +162,12 @@ class LangWatchDSPy:
     def init(
         self,
         experiment: str,
-        optimizer: Optional[Teleprompter],
+        optimizer: Optional[Teleprompter] = None,
         run_id: Optional[str] = None,
         slug: Optional[str] = None,
         workflow_id: Optional[str] = None,
         workflow_version_id: Optional[str] = None,
+        evaluator: Optional[Evaluate] = None,
     ):
         if langwatch.get_api_key() is None:
             print("API key was not detected, calling langwatch.login()...")
@@ -184,6 +195,9 @@ class LangWatchDSPy:
             )
         response.raise_for_status()
 
+        if optimizer and evaluator:
+            raise ValueError("You can only provide an optimizer or an evaluator, not both.")
+
         self.experiment_slug = slug or experiment
         random.seed()  # MIPRO meses up the global seed, so we need to reset it to random to get a new run_id
         self.run_id = run_id or generate_slug(3)
@@ -193,9 +207,11 @@ class LangWatchDSPy:
         self.patch_llms()
         if optimizer is not None:
             self.patch_optimizer(optimizer)
+        elif evaluator is not None:
+            self.patch_evaluator(evaluator)
         else:
             print(
-                "No optimizer provided, assuming custom optimizer tracking, make sure to call `track_metric` and `log_step` manually: https://docs.langwatch.ai/dspy-visualization/custom-optimizer"
+                "No optimizer or evaluator provided, assuming custom optimizer tracking, make sure to call `track_metric` and `log_step` manually: https://docs.langwatch.ai/dspy-visualization/custom-optimizer"
             )
 
         result = response.json()
@@ -205,6 +221,10 @@ class LangWatchDSPy:
             f"[LangWatch] Open {langwatch.get_endpoint()}{self.experiment_path}?runIds={self.run_id} to track your DSPy training session live\n"
         )
 
+    def patch_evaluator(self, evaluator: Evaluate):
+        evaluator.__class__ = LangWatchTrackedEvaluate
+        print(f"\n[LangWatch] `dspy.evaluate.Evaluate` object detected and patched for live tracking.")
+
     def patch_optimizer(self, optimizer: Teleprompter):
         METRIC_TRACKING_CLASSMAP = {
             BootstrapFewShot: LangWatchTrackedBootstrapFewShot,
@@ -379,14 +399,16 @@ langwatch_dspy = LangWatchDSPy()
 
 def init(
     experiment: str,
-    optimizer: Optional[Teleprompter],
+    optimizer: Optional[Teleprompter] = None,
     run_id: Optional[str] = None,
     slug: Optional[str] = None,
     workflow_id: Optional[str] = None,
     workflow_version_id: Optional[str] = None,
+    evaluator: Optional[Evaluate] = None,
 ):
     langwatch_dspy.init(
-        experiment, optimizer, run_id, slug, workflow_id, workflow_version_id
+        experiment, optimizer, run_id, slug, workflow_id, workflow_version_id,
+        evaluator=evaluator,
     )
 
 
@@ -673,6 +695,40 @@ class LangWatchTrackedMIPROv2(MIPROv2):
             Evaluate.__call__ = original_evaluate_call
 
 
+class LangWatchTrackedEvaluate(Evaluate):
+    @with_callbacks
+    def __call__(self, program: dspy.Module, metric: Optional[Callable] = None, **kwargs):
+        lw_dspy = langwatch_dspy
+
+        metric_to_use = metric if metric is not None else self.metric
+        if not metric_to_use:
+            raise ValueError("Evaluation metric must be provided either during Evaluate initialization or during call.")
+
+        wrapped_metric = lw_dspy.track_metric(metric_to_use)
+
+        # Call the original evaluation logic with the wrapped metric.
+        # We need to make sure we pass the wrapped metric to the super call.
+        kwargs_for_super = kwargs.copy()
+        kwargs_for_super["metric"] = wrapped_metric
+        result = super().__call__(program, **kwargs_for_super)
+
+        score = result[0] if isinstance(result, tuple) else result
+
+        if lw_dspy.run_id:
+            lw_dspy.log_step(
+                optimizer=DSPyOptimizer(name="Evaluate", parameters={}),
+                index="0",
+                score=float(score),
+                label="score",
+                predictors=[
+                    DSPyPredictor(name=name, predictor=predictor)
+                    for name, predictor in program.named_predictors()
+                ],
+            )
+
+        return result
+
+
 # === Tracer ===#
 
 

langwatch/telemetry/tracing.py

@@ -267,9 +267,10 @@ class LangWatchTrace:
         self,
     ):
         ensure_setup()
-        import langwatch.dspy
+        # Making sure it doesn't get mixed up with langwatch.dspy from `import langwatch`
+        from langwatch.dspy.__init__ import DSPyTracer
 
-        langwatch.dspy.DSPyTracer(trace=self)
+        DSPyTracer(trace=self)
 
     def share(self) -> str:
         """Share this trace and get a shareable URL."""

{langwatch-0.2.8.dist-info → langwatch-0.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch
-Version: 0.2.8
+Version: 0.2.9
 Summary: LangWatch Python SDK, for monitoring your LLMs
 Author-email: Langwatch Engineers <engineering@langwatch.ai>
 License: MIT

{langwatch-0.2.8.dist-info → langwatch-0.2.9.dist-info}/RECORD

@@ -1,5 +1,5 @@
 langwatch/__init__.py,sha256=nepKSqGG6MQnLgn3QrzcAQ24wlXc-wBAcS0Ks7oVUKk,3681
-langwatch/__version__.py,sha256=fVjSxPE9TfkSdSWhP2pKjFKPW2e6jMivB_7HF8_G-3Y,64
+langwatch/__version__.py,sha256=_Ef6eRJnXlLxv6Pt9FrqxJZdOHx4onlAQg2rPnAnV1E,64
 langwatch/attributes.py,sha256=VsNstyDPaMIjsM4U7yAIDE7ouVxgwhOdbQlrhce9XU8,3699
 langwatch/batch_evaluation.py,sha256=cZ_OxDEpZhwVQap8fub3vcdWVYw9OdUCg7WTuy5FgoY,15748
 langwatch/client.py,sha256=w-WkYRwgFMlfljAwcR6GBT11kBQOc3Ifb3hGY1htOd4,13156
@@ -15,7 +15,7 @@ langwatch/tracer.py,sha256=t5FOdP1es9H_pPGqGUBLXCyEln0tTi4m4M9b6WxCrPU,975
 langwatch/types.py,sha256=DkZThS7ptUY0_Ao7PNO6ZGBE9Qj0YmJf5auRkq2Cphw,2570
 langwatch/dataset/__init__.py,sha256=1vUqnyFymnzcIOTahtsPvKTyAet_q-HTKByRUkuED80,2290
 langwatch/domain/__init__.py,sha256=luuin3-6mmMqDaOJE_1wb7M1ccjhhZL80UN0TBA_TXM,6040
-langwatch/dspy/__init__.py,sha256=VQoriSl0Um4A0K3dbYxj6ZLUqNLgKKyODd3vkVp0wD4,32027
+langwatch/dspy/__init__.py,sha256=E9rqyhOyIO3E-GxJSZlHtsvjapFjKQGF85QRcpKSvKE,34280
 langwatch/evaluation/__init__.py,sha256=Jy7PW5VQbMoDGdOLRlQmDEvo_9TDkBLmrLrfocxddLM,281
 langwatch/evaluation/evaluation.py,sha256=AF3VXCcTGB3F8ChsjwBxqjUXkLzvTbkWbiWYxRzVWik,16037
 langwatch/exporters/filterable_batch_span_exporter.py,sha256=MlhZjui56XD6p2sa8kEGyr-Hb3wqudknngmemnB4Twg,2142
@@ -207,7 +207,7 @@ langwatch/prompt/prompt.py,sha256=ujcyL_7d0fCLXT-UbFcdnH90D6tkSD5Ro5JGFncU9fQ,19
 langwatch/telemetry/context.py,sha256=ixKvTBZADFuWglFCK95r8bF4-f-iq_WixzC4ydirCE8,3888
 langwatch/telemetry/sampling.py,sha256=XDf6ZoXiwpHaHDYd_dDszSqH8_9-CHFNsGAZWOW1VYk,1327
 langwatch/telemetry/span.py,sha256=HKYouwWWHCsc1PtnrB3UFFB-X1LT7DA5cSy-iA5Qoug,32779
-langwatch/telemetry/tracing.py,sha256=pAYwpNzhqDrv2meWCtKgaNi4em2dKIAaGbsdvsZlvGg,26550
+langwatch/telemetry/tracing.py,sha256=7zPQ1W3KZKi35vzs1veTLviWkRe6bEA4x5_baNsjP2c,26650
 langwatch/telemetry/types.py,sha256=Q9H7nT3GMK1aluRB7CCX8BR7VFKrQY_vdFdyF4Yc98U,501
 langwatch/utils/__init__.py,sha256=3rqQTgzEtmICJW_KSPuLa5q8p5udxt5SRi28Z2vZB10,138
 langwatch/utils/capture.py,sha256=uVKPqHCm-o8CpabsUfhqbNFr5sgUHzcKnBadvL2oIwI,1172
@@ -216,6 +216,6 @@ langwatch/utils/initialization.py,sha256=LOmGCiox7JRiHprnB04AdO8BhUggZtRbWgBPF9m
 langwatch/utils/module.py,sha256=KLBNOK3mA9gCSifCcQX_lOtU48BJQDWvFKtF6NMvwVA,688
 langwatch/utils/transformation.py,sha256=5XUnW7Oz8Ck9EMsKeKeoDOrIw3EXpLGMk_fMSeA0Zng,7216
 langwatch/utils/utils.py,sha256=ZCOSie4o9LdJ7odshNfCNjmgwgQ27ojc5ENqt1rXuSs,596
-langwatch-0.2.8.dist-info/METADATA,sha256=1M8Yyf8TEbm7UWcUkNPGMiFcvGFTb1y_eRn3whPdfKQ,12886
-langwatch-0.2.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langwatch-0.2.8.dist-info/RECORD,,
+langwatch-0.2.9.dist-info/METADATA,sha256=FbRafO8xi9pzIgZXDnJkchcqI6izMMaejU_EznN2KZU,12886
+langwatch-0.2.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langwatch-0.2.9.dist-info/RECORD,,