dreadnode 1.11.0__tar.gz → 1.12.0__tar.gz

{dreadnode-1.11.0 → dreadnode-1.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dreadnode
-Version: 1.11.0
+Version: 1.12.0
 Summary: Dreadnode SDK
 Author: Nick Landers
 Author-email: monoxgas@gmail.com
@@ -22,6 +22,7 @@ Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: pillow (>=11.2.1,<12.0.0) ; extra == "multimodal"
 Requires-Dist: pydantic (>=2.9.2,<3.0.0)
 Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
+Requires-Dist: rigging (>=3.1.1,<4.0.0)
 Requires-Dist: soundfile (>=0.13.1,<0.14.0) ; extra == "multimodal"
 Requires-Dist: transformers (>=4.41.0,<5.0.0) ; extra == "training"
 Project-URL: Repository, https://github.com/dreadnode/sdk

{dreadnode-1.11.0 → dreadnode-1.12.0}/dreadnode/__init__.py

@@ -1,9 +1,9 @@
-from dreadnode import convert, data_types
-from dreadnode.data_types import Audio, Image, Object3D, Table, Video
+from dreadnode import convert, data_types, scorers
+from dreadnode.data_types import Audio, Code, Image, Markdown, Object3D, Table, Text, Video
 from dreadnode.main import DEFAULT_INSTANCE, Dreadnode
 from dreadnode.metric import Metric, MetricDict, Scorer
 from dreadnode.object import Object
-from dreadnode.task import Task
+from dreadnode.task import Task, TaskInput
 from dreadnode.tracing.span import RunSpan, Span, TaskSpan
 from dreadnode.version import VERSION
 
@@ -36,8 +36,10 @@ __version__ = VERSION
 __all__ = [
     "DEFAULT_INSTANCE",
     "Audio",
+    "Code",
     "Dreadnode",
     "Image",
+    "Markdown",
     "Metric",
     "MetricDict",
     "Object",
@@ -48,7 +50,9 @@ __all__ = [
     "Span",
     "Table",
     "Task",
+    "TaskInput",
     "TaskSpan",
+    "Text",
     "Video",
     "__version__",
     "api",
@@ -68,6 +72,7 @@ __all__ = [
     "push_update",
     "run",
     "scorer",
+    "scorers",
     "shutdown",
     "span",
     "tag",

{dreadnode-1.11.0 → dreadnode-1.12.0}/dreadnode/data_types/image.py

@@ -8,9 +8,9 @@ import numpy as np
 from dreadnode.data_types.base import DataType
 
 try:
-    from PIL import Image as PILImage
+    from PIL import Image as PILImage  # type: ignore[import-not-found,unused-ignore]
 except ImportError:
-    PILImage = None  # type: ignore[assignment]
+    PILImage = None  # type: ignore[assignment,unused-ignore]
 
 ImageDataType = t.Any | np.ndarray[t.Any, t.Any]
 ImageDataOrPathType = str | Path | bytes | ImageDataType

{dreadnode-1.11.0 → dreadnode-1.12.0}/dreadnode/integrations/transformers.py

@@ -12,8 +12,14 @@ if importlib.util.find_spec("transformers") is None:
 
 import typing as t
 
-from transformers.trainer_callback import TrainerCallback, TrainerControl, TrainerState
-from transformers.training_args import TrainingArguments
+from transformers.trainer_callback import (  # type: ignore[import-not-found,unused-ignore]
+    TrainerCallback,
+    TrainerControl,
+    TrainerState,
+)
+from transformers.training_args import (  # type: ignore[import-not-found,unused-ignore]
+    TrainingArguments,
+)
 
 import dreadnode as dn
 
@@ -40,7 +46,7 @@ def _clean_keys(data: dict[str, t.Any]) -> dict[str, t.Any]:
     return cleaned
 
 
-class DreadnodeCallback(TrainerCallback):
+class DreadnodeCallback(TrainerCallback):  # type: ignore[misc,unused-ignore]
     """
     An implementation of the `TrainerCallback` interface for Dreadnode.
 

{dreadnode-1.11.0 → dreadnode-1.12.0}/dreadnode/main.py

@@ -634,9 +634,7 @@ class Dreadnode:
                 attributes=_attributes,
                 func=t.cast("t.Callable[P, R]", func),
                 scorers=[
-                    scorer
-                    if isinstance(scorer, Scorer)
-                    else Scorer.from_callable(self._get_tracer(), scorer)
+                    scorer if isinstance(scorer, Scorer) else Scorer.from_callable(scorer)
                     for scorer in scorers or []
                 ],
                 tags=list(tags or []),
@@ -726,7 +724,6 @@ class Dreadnode:
 
         def make_scorer(func: ScorerCallable[T]) -> Scorer[T]:
             return Scorer.from_callable(
-                self._get_tracer(),
                 func,
                 name=name,
                 tags=tags,
@@ -860,7 +857,11 @@ class Dreadnode:
 
         target = (task or run) if to == "task-or-run" else run
         if target is None:
-            raise RuntimeError("Tagging must be done within a run")
+            warn_at_user_stacklevel(
+                "tag() was called outside of a task or run.",
+                category=DreadnodeUsageWarning,
+            )
+            return
 
         target.add_tags(tag)
 
@@ -883,7 +884,11 @@ class Dreadnode:
                 # do more work
         """
         if (run := current_run_span.get()) is None:
-            raise RuntimeError("Run updates must be pushed within a run")
+            warn_at_user_stacklevel(
+                "push_update() was called outside of a run.",
+                category=DreadnodeUsageWarning,
+            )
+            return
 
         run.push_update(force=True)
 
@@ -934,7 +939,12 @@ class Dreadnode:
             **params: The parameters to log. Each parameter is a key-value pair.
         """
         if (run := current_run_span.get()) is None:
-            raise RuntimeError("Parameters must be logged within a run")
+            warn_at_user_stacklevel(
+                "log_params() was called outside of a run.",
+                category=DreadnodeUsageWarning,
+            )
+            return
+
         run.log_params(**params)
 
     @t.overload
@@ -1085,13 +1095,6 @@ class Dreadnode:
         Returns:
             The logged metric object.
         """
-        task = current_task_span.get()
-        run = current_run_span.get()
-
-        target = (task or run) if to == "task-or-run" else run
-        if target is None:
-            raise RuntimeError("log_metric() must be called within a run")
-
         metric = (
             value
             if isinstance(value, Metric)
@@ -1102,6 +1105,18 @@ class Dreadnode:
                 attributes or {},
             )
         )
+
+        task = current_task_span.get()
+        run = current_run_span.get()
+
+        target = (task or run) if to == "task-or-run" else run
+        if target is None:
+            warn_at_user_stacklevel(
+                "log_metric() was called outside of a task or run.",
+                category=DreadnodeUsageWarning,
+            )
+            return metric
+
         return target.log_metric(name, metric, origin=origin, mode=mode)
 
     @t.overload
@@ -1240,7 +1255,11 @@ class Dreadnode:
 
         target = (task or run) if to == "task-or-run" else run
         if target is None:
-            raise RuntimeError("log_metrics() must be called within a run")
+            warn_at_user_stacklevel(
+                "log_metrics() was called outside of a task or run.",
+                category=DreadnodeUsageWarning,
+            )
+            return []
 
         logged_metrics: list[Metric] = []
 
@@ -1312,7 +1331,11 @@ class Dreadnode:
             local_uri: The local path to the file to upload.
         """
         if (run := current_run_span.get()) is None:
-            raise RuntimeError("log_artifact() must be called within a run")
+            warn_at_user_stacklevel(
+                "log_artifact() was called outside of a run.",
+                category=DreadnodeUsageWarning,
+            )
+            return
 
         run.log_artifact(local_uri=local_uri)
 
@@ -1350,7 +1373,11 @@ class Dreadnode:
 
         target = (task or run) if to == "task-or-run" else run
         if target is None:
-            raise RuntimeError("log_inputs() must be called within a run")
+            warn_at_user_stacklevel(
+                "log_input() was called outside of a task or run.",
+                category=DreadnodeUsageWarning,
+            )
+            return
 
         target.log_input(name, value, label=label, attributes=attributes)
 
@@ -1412,9 +1439,11 @@ class Dreadnode:
 
         target = (task or run) if to == "task-or-run" else run
         if target is None:
-            raise RuntimeError(
-                "log_output() must be called within a run or a task",
+            warn_at_user_stacklevel(
+                "log_output() was called outside of a task or run.",
+                category=DreadnodeUsageWarning,
             )
+            return
 
         target.log_output(name, value, label=label, attributes=attributes)
 
@@ -1461,7 +1490,11 @@ class Dreadnode:
             attributes: Additional attributes to attach to the link.
         """
         if (run := current_run_span.get()) is None:
-            raise RuntimeError("link() must be called within a run")
+            warn_at_user_stacklevel(
+                "link_objects() was called outside of a run.",
+                category=DreadnodeUsageWarning,
+            )
+            return
 
         origin_hash = run.log_object(origin)
         link_hash = run.log_object(link)

{dreadnode-1.11.0 → dreadnode-1.12.0}/dreadnode/metric.py

@@ -6,7 +6,6 @@ from datetime import datetime, timezone
 import typing_extensions as te
 from logfire._internal.stack_info import warn_at_user_stacklevel
 from logfire._internal.utils import safe_repr
-from opentelemetry.trace import Tracer
 
 from dreadnode.types import JsonDict, JsonValue
 
@@ -73,7 +72,11 @@ class Metric:
         total = sum(value * weight for _, value, weight in values)
         weight = sum(weight for _, _, weight in values)
         score_attributes = {name: value for name, value, _ in values}
-        return cls(value=total / weight, step=step, attributes={**attributes, **score_attributes})
+        return cls(
+            value=total / weight,
+            step=step,
+            attributes={**attributes, **score_attributes},
+        )
 
     def apply_mode(self, mode: MetricAggMode, others: "list[Metric]") -> "Metric":
         """
@@ -124,8 +127,6 @@ ScorerCallable = t.Callable[[T], t.Awaitable[ScorerResult]] | t.Callable[[T], Sc
 
 @dataclass
 class Scorer(t.Generic[T]):
-    tracer: Tracer
-
     name: str
     "The name of the scorer, used for reporting metrics."
     tags: t.Sequence[str]
@@ -138,25 +139,27 @@ class Scorer(t.Generic[T]):
     "The step value to attach to metrics produced by this Scorer."
     auto_increment_step: bool = False
     "Whether to automatically increment the step for each time this scorer is called."
+    catch: bool = False
+    "Whether to catch exceptions in the scorer function and return a 0 Metric with error information."
 
     @classmethod
     def from_callable(
         cls,
-        tracer: Tracer,
         func: "ScorerCallable[T] | Scorer[T]",
         *,
         name: str | None = None,
         tags: t.Sequence[str] | None = None,
+        catch: bool = False,
         **attributes: t.Any,
     ) -> "Scorer[T]":
         """
         Create a scorer from a callable function.
 
         Args:
-            tracer: The tracer to use for reporting metrics.
             func: The function to call to get the metric.
             name: The name of the scorer, used for reporting metrics.
             tags: A list of tags to attach to the metric.
+            catch: Whether to catch exceptions in the scorer function and return a 0 Metric with error information.
             **attributes: A dictionary of attributes to attach to the metric.
 
         Returns:
@@ -177,11 +180,11 @@ class Scorer(t.Generic[T]):
         )
         name = name or func_name
         return cls(
-            tracer=tracer,
             name=name,
             tags=tags or [],
             attributes=attributes or {},
             func=func,
+            catch=catch,
         )
 
     def __post_init__(self) -> None:
@@ -196,13 +199,13 @@ class Scorer(t.Generic[T]):
             A new Scorer.
         """
         return Scorer(
-            tracer=self.tracer,
             name=self.name,
             tags=self.tags,
             attributes=self.attributes,
             func=self.func,
             step=self.step,
             auto_increment_step=self.auto_increment_step,
+            catch=self.catch,
         )
 
     async def __call__(self, object: T) -> Metric:
@@ -217,17 +220,19 @@ class Scorer(t.Generic[T]):
         Returns:
             A Metric object.
         """
-        from dreadnode.tracing.span import Span
-
-        with Span(
-            name=self.name,
-            tags=self.tags,
-            attributes=self.attributes,
-            tracer=self.tracer,
-        ):
+        try:
             metric = self.func(object)
             if inspect.isawaitable(metric):
                 metric = await metric
+        except Exception as exc:
+            if not self.catch:
+                raise
+
+            warn_at_user_stacklevel(
+                f"Error executing scorer {self.name!r} for object {object!r}: {exc}",
+                MetricWarning,
+            )
+            metric = Metric(value=0.0, step=self.step, attributes={"error": str(exc)})
 
         if not isinstance(metric, Metric):
             metric = Metric(

dreadnode-1.12.0/dreadnode/object.py

@@ -0,0 +1,48 @@
+import typing as t
+from dataclasses import dataclass
+
+from pydantic import BaseModel, Field
+
+from dreadnode.types import AnyDict
+
+
+@dataclass
+class ObjectRef:
+    name: str
+    label: str
+    hash: str
+    attributes: AnyDict | None
+
+
+class ObjectUri(BaseModel):
+    hash: str
+    schema_hash: str
+    uri: str
+    size: int
+    type: t.Literal["uri"] = "uri"
+
+    # During execution, we might want to dynamically pull a value
+    # in it's unserialized form, so we store it here.
+    runtime_value: t.Any | None = Field(None, init=False, repr=False, exclude=True)
+
+    @property
+    def value(self) -> t.Any:
+        return self.runtime_value or self.uri
+
+
+class ObjectVal(BaseModel):
+    hash: str
+    schema_hash: str
+    value_: t.Any = Field(alias="value")
+    type: t.Literal["val"] = "val"
+
+    # During execution, we might want to dynamically pull a value
+    # in it's unserialized form, so we store it here.
+    runtime_value: t.Any | None = Field(None, init=False, repr=False, exclude=True)
+
+    @property
+    def value(self) -> t.Any:
+        return self.runtime_value or self.value_
+
+
+Object = ObjectUri | ObjectVal

dreadnode-1.12.0/dreadnode/scorers/__init__.py

@@ -0,0 +1,35 @@
+from dreadnode.scorers.consistency import character_consistency
+from dreadnode.scorers.contains import (
+    contains,
+    detect_ansi_escapes,
+    detect_refusal,
+    detect_sensitive_keywords,
+    detect_unsafe_shell_content,
+)
+from dreadnode.scorers.length import length_in_range, length_ratio, length_target
+from dreadnode.scorers.pii import detect_pii, detect_pii_with_presidio
+from dreadnode.scorers.readability import readability
+from dreadnode.scorers.rigging import wrap_chat
+from dreadnode.scorers.sentiment import sentiment, sentiment_with_perspective
+from dreadnode.scorers.similarity import bleu, semantic_similarity, similarity
+
+__all__ = [
+    "bleu",
+    "character_consistency",
+    "contains",
+    "detect_ansi_escapes",
+    "detect_pii",
+    "detect_pii_with_presidio",
+    "detect_refusal",
+    "detect_sensitive_keywords",
+    "detect_unsafe_shell_content",
+    "length_in_range",
+    "length_ratio",
+    "length_target",
+    "readability",
+    "semantic_similarity",
+    "sentiment",
+    "sentiment_with_perspective",
+    "similarity",
+    "wrap_chat",
+]

dreadnode-1.12.0/dreadnode/scorers/consistency.py

@@ -0,0 +1,66 @@
+import re
+import typing as t
+
+from dreadnode.metric import Metric, Scorer
+from dreadnode.task import TaskInput
+from dreadnode.util import clean_str
+
+if t.TYPE_CHECKING:
+    from dreadnode.types import JsonDict
+
+
+def character_consistency(
+    reference: str | TaskInput,
+    *,
+    max_ratio_diff: float = 2.0,
+    name: str | None = None,
+) -> "Scorer[t.Any]":
+    """
+    Scores character type consistency between the data and a reference text.
+
+    It compares the ratio of letters, numbers, and symbols in both texts.
+    A score of 1.0 indicates identical distributions.
+
+    Args:
+        reference: The reference text (e.g., the prompt) or a TaskInput.
+        max_ratio_diff: The denominator for normalizing ratio differences.
+        name: Name of the scorer.
+    """
+
+    def _analyze_text(text: str) -> dict[str, int]:
+        return {
+            "letters": len(re.findall(r"[a-zA-Z]", text)),
+            "numbers": len(re.findall(r"\d", text)),
+            "symbols": len(re.findall(r"[^\w\s]", text)),
+        }
+
+    def evaluate(data: t.Any) -> Metric:
+        candidate_text = str(data)
+        reference_text = str(reference.resolve()) if isinstance(reference, TaskInput) else reference
+
+        candidate_chars = _analyze_text(candidate_text)
+        reference_chars = _analyze_text(reference_text)
+
+        candidate_total = sum(candidate_chars.values())
+        reference_total = sum(reference_chars.values())
+
+        if reference_total == 0 or candidate_total == 0:
+            return Metric(value=0.0, attributes={"error": "Reference or candidate text is empty."})
+
+        scores: dict[str, float] = {}
+        metadata: JsonDict = {}
+        for char_type in ["letters", "numbers", "symbols"]:
+            ref_ratio = reference_chars[char_type] / reference_total
+            cand_ratio = candidate_chars[char_type] / candidate_total
+            diff = abs(ref_ratio - cand_ratio)
+            score = max(0.0, 1.0 - (diff / max_ratio_diff))
+            scores[char_type] = score
+            metadata[f"{char_type}_ratio_diff"] = round(diff, 4)
+
+        return Metric.from_many([(name, score, 1.0) for name, score in scores.items()])
+
+    if name is None:
+        ref_name = reference.name if isinstance(reference, TaskInput) else "static_text"
+        name = f"char_consistency_{clean_str(ref_name)}"
+
+    return Scorer.from_callable(evaluate, name=name)