lmnr 0.4.27__py3-none-any.whl → 0.4.29b0__py3-none-any.whl

lmnr/__init__.py

@@ -1,7 +1,13 @@
 from .sdk.datasets import EvaluationDataset, LaminarDataset
 from .sdk.evaluations import evaluate
 from .sdk.laminar import Laminar
-from .sdk.types import ChatMessage, PipelineRunError, PipelineRunResponse, NodeInput
+from .sdk.types import (
+    ChatMessage,
+    HumanEvaluator,
+    NodeInput,
+    PipelineRunError,
+    PipelineRunResponse,
+)
 from .sdk.decorators import observe
 from .traceloop_sdk import Instruments
 from .traceloop_sdk.tracing.attributes import Attributes

lmnr/sdk/datasets.py

@@ -34,7 +34,7 @@ class LaminarDataset(EvaluationDataset):
         self._fetched_items = []
         self._offset = 0
         self._fetch_size = fetch_size
-        self._logger = get_default_logger(self.__class__.__name__, level=logging.DEBUG)
+        self._logger = get_default_logger(self.__class__.__name__)
 
     def _fetch_batch(self):
         self._logger.debug(

lmnr/sdk/evaluations.py

@@ -18,6 +18,7 @@ from .types import (
     EvaluationResultDatapoint,
     EvaluatorFunction,
     ExecutorFunction,
+    HumanEvaluator,
     Numeric,
     NumericTypes,
     SpanType,
@@ -99,6 +100,7 @@ class Evaluation:
         data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
         executor: Any,
         evaluators: dict[str, EvaluatorFunction],
+        human_evaluators: dict[str, HumanEvaluator] = {},
         name: Optional[str] = None,
         group_id: Optional[str] = None,
         batch_size: int = DEFAULT_BATCH_SIZE,
@@ -126,6 +128,11 @@ class Evaluation:
                 If the score is a single number, it will be named after the\
                 evaluator function. Evaluator function names must contain only\
                 letters, digits, hyphens, underscores, or spaces.
+            human_evaluators (dict[str, HumanEvaluator], optional):\
+                [Beta] Dictionary from human evaluator names to instances of\
+                HumanEvaluator. For now, human evaluator only holds the queue\
+                name.
+                Defaults to an empty dictionary.
             name (Optional[str], optional): Optional name of the evaluation.\
                 Used to identify the evaluation in the group.\
                 If not provided, a random name will be generated.
@@ -159,14 +166,27 @@ class Evaluation:
         if not evaluators:
             raise ValueError("No evaluators provided")
 
-        # TODO: Compile regex once and then reuse it
+        evaluator_name_regex = re.compile(r"^[\w\s-]+$")
         for evaluator_name in evaluators:
-            if not re.match(r"^[\w\s-]+$", evaluator_name):
+            if not evaluator_name_regex.match(evaluator_name):
                 raise ValueError(
                     f'Invalid evaluator key: "{evaluator_name}". '
                     "Keys must only contain letters, digits, hyphens,"
                     "underscores, or spaces."
                 )
+        for evaluator_name in human_evaluators or {}:
+            if not evaluator_name_regex.match(evaluator_name):
+                raise ValueError(
+                    f'Invalid human evaluator key: "{evaluator_name}". '
+                    "Keys must only contain letters, digits, hyphens,"
+                    "underscores, or spaces."
+                )
+
+        if intersection := set(evaluators.keys()) & set(human_evaluators.keys()):
+            raise ValueError(
+                "Evaluator and human evaluator names must not overlap. "
+                f"Repeated keys: {intersection}"
+            )
 
         self.is_finished = False
         self.reporter = EvaluationReporter()
@@ -183,6 +203,7 @@ class Evaluation:
         self.name = name
         self.batch_size = batch_size
         self._logger = get_default_logger(self.__class__.__name__)
+        self.human_evaluators = human_evaluators
         L.initialize(
             project_api_key=project_api_key,
             base_url=base_url,
@@ -202,9 +223,7 @@ class Evaluation:
             return loop.run_until_complete(self._run())
 
     async def _run(self) -> None:
-        self.reporter.start(
-            len(self.data),
-        )
+        self.reporter.start(len(self.data))
 
         try:
             result_datapoints = await self._evaluate_in_batches()
@@ -212,13 +231,19 @@ class Evaluation:
             self.reporter.stopWithError(e)
             self.is_finished = True
             return
-        else:
-            evaluation = L.create_evaluation(
-                data=result_datapoints, group_id=self.group_id, name=self.name
-            )
-            average_scores = get_average_scores(result_datapoints)
-            self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
-            self.is_finished = True
+
+        # For now add all human evaluators to all result datapoints
+        # In the future, we will add ways to specify which human evaluators
+        # to add to which result datapoints, e.g. sample some randomly
+        for result_datapoint in result_datapoints:
+            result_datapoint.human_evaluators = self.human_evaluators or {}
+
+        evaluation = L.create_evaluation(
+            data=result_datapoints, group_id=self.group_id, name=self.name
+        )
+        average_scores = get_average_scores(result_datapoints)
+        self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
+        self.is_finished = True
 
     async def _evaluate_in_batches(self) -> list[EvaluationResultDatapoint]:
         result_datapoints = []
@@ -292,6 +317,7 @@ def evaluate(
     data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
     executor: ExecutorFunction,
     evaluators: dict[str, EvaluatorFunction],
+    human_evaluators: dict[str, HumanEvaluator] = {},
     name: Optional[str] = None,
     group_id: Optional[str] = None,
     batch_size: int = DEFAULT_BATCH_SIZE,
@@ -326,6 +352,10 @@ def evaluate(
                 If the score is a single number, it will be named after the\
                 evaluator function. Evaluator function names must contain only\
                 letters, digits, hyphens, underscores, or spaces.
+        human_evaluators (dict[str, HumanEvaluator], optional):\
+            [Beta] Dictionary from human evaluator names to instances of\
+            HumanEvaluator. For now, human evaluator only holds the queue name.
+            Defaults to an empty dictionary.
         name (Optional[str], optional): Optional name of the evaluation.\
                         Used to identify the evaluation in the group.\
                         If not provided, a random name will be generated.
@@ -359,6 +389,7 @@ def evaluate(
         executor=executor,
         evaluators=evaluators,
         group_id=group_id,
+        human_evaluators=human_evaluators,
         name=name,
         batch_size=batch_size,
         project_api_key=project_api_key,

lmnr/sdk/types.py

@@ -110,6 +110,13 @@ EvaluatorFunction = Callable[
 ]
 
 
+class HumanEvaluator(pydantic.BaseModel):
+    queueName: str
+
+    def __init__(self, queue_name: str):
+        super().__init__(queueName=queue_name)
+
+
 class CreateEvaluationResponse(pydantic.BaseModel):
     id: uuid.UUID
     createdAt: datetime.datetime
@@ -123,6 +130,7 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
     target: EvaluationDatapointTarget
     executor_output: ExecutorFunctionReturnType
     scores: dict[str, Numeric]
+    human_evaluators: dict[str, HumanEvaluator] = pydantic.Field(default_factory=dict)
     trace_id: uuid.UUID
 
     # uuid is not serializable by default, so we need to convert it to a string
@@ -139,6 +147,10 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
             "executorOutput": serialize(self.executor_output),
             "scores": self.scores,
             "traceId": str(self.trace_id),
+            "humanEvaluators": {
+                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
+                for k, v in self.human_evaluators.items()
+            },
         }
 
 

lmnr/traceloop_sdk/tracing/attributes.py

@@ -5,6 +5,7 @@ SPAN_INPUT = "lmnr.span.input"
 SPAN_OUTPUT = "lmnr.span.output"
 SPAN_TYPE = "lmnr.span.type"
 SPAN_PATH = "lmnr.span.path"
+SPAN_INSTRUMENTATION_SOURCE = "lmnr.span.instrumentation_source"
 
 ASSOCIATION_PROPERTIES = "lmnr.association.properties"
 SESSION_ID = "session_id"

lmnr/traceloop_sdk/tracing/tracing.py

@@ -25,7 +25,11 @@ from opentelemetry.instrumentation.threading import ThreadingInstrumentor
 
 # from lmnr.traceloop_sdk import Telemetry
 from lmnr.traceloop_sdk.instruments import Instruments
-from lmnr.traceloop_sdk.tracing.attributes import ASSOCIATION_PROPERTIES, SPAN_PATH
+from lmnr.traceloop_sdk.tracing.attributes import (
+    ASSOCIATION_PROPERTIES,
+    SPAN_INSTRUMENTATION_SOURCE,
+    SPAN_PATH,
+)
 from lmnr.traceloop_sdk.tracing.content_allow_list import ContentAllowList
 from lmnr.traceloop_sdk.utils import is_notebook
 from lmnr.traceloop_sdk.utils.package_check import is_package_installed
@@ -235,6 +239,8 @@ class TracerWrapper(object):
             # the final part of the name to the span on the backend.
             span.set_attribute(SPAN_PATH, span_path)
 
+        span.set_attribute(SPAN_INSTRUMENTATION_SOURCE, "python")
+
         association_properties = get_value("association_properties")
         if association_properties is not None:
             _set_association_properties_attributes(span, association_properties)
@@ -266,10 +272,7 @@ class TracerWrapper(object):
         if hasattr(cls, "instance"):
             return True
 
-        if (os.getenv("TRACELOOP_SUPPRESS_WARNINGS") or "false").lower() == "true":
-            return False
-
-        print("Warning: Laminar not initialized, make sure to initialize")
+        logging.warning("Warning: Laminar not initialized, make sure to initialize")
         return False
 
     def flush(self):
@@ -557,7 +560,9 @@ def init_langchain_instrumentor():
                 instrumentor.instrument()
         return True
     except Exception as e:
-        logging.error(f"Error initializing LangChain instrumentor: {e}")
+        # FIXME: silence this error temporarily, it appears to not be critical
+        if str(e) != "No module named 'langchain_community'":
+            logging.error(f"Error initializing LangChain instrumentor: {e}")
         # Telemetry().log_exception(e)
         return False
 

{lmnr-0.4.27.dist-info → lmnr-0.4.29b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.27
+Version: 0.4.29b0
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai

{lmnr-0.4.27.dist-info → lmnr-0.4.29b0.dist-info}/RECORD

@@ -1,12 +1,12 @@
-lmnr/__init__.py,sha256=87ZHKNA2xmZvqtXrWjhXUNR7YUrdP8-77fYqe-PEtrE,348
+lmnr/__init__.py,sha256=qwI8S02jRm7QvXsyljuEurp-kUt8HOCAN_m9RKVQVtU,389
 lmnr/cli.py,sha256=Ptvm5dsNLKUY5lwnN8XkT5GtCYjzpRNi2WvefknB3OQ,1079
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lmnr/sdk/datasets.py,sha256=V6q0Zcv7znx5cB1YZd0pBgTEcAKXbb-pEwguScEhBw0,1612
+lmnr/sdk/datasets.py,sha256=w8U9E6fvetAo65Cb2CbYzlfhY8CfXAR-VysrakG6-4I,1591
 lmnr/sdk/decorators.py,sha256=ZSDaEZyjo-RUzRCltsNbe6x0t9SKl2xRQ2q4uaKvXtk,2250
-lmnr/sdk/evaluations.py,sha256=vG788rSDppAGEvIpyglKvm8Ac_D5cw07a6btMeMr8AI,15124
+lmnr/sdk/evaluations.py,sha256=BUdsxuh3Rjk-8oj-481geW7fqTmgLcq0CuMOYbgMFx8,16807
 lmnr/sdk/laminar.py,sha256=H87fXSWb9shcPW4AeoYwvTXJ-jSTjzm2sI1A1U1Vkg8,18780
 lmnr/sdk/log.py,sha256=cZBeUoSK39LMEV-X4-eEhTWOciULRfHaKfRK8YqIM8I,1532
-lmnr/sdk/types.py,sha256=6_C2LhcbI9PwlntnSuREE0FRsBUxc3WS_yC_Y_trPBI,5052
+lmnr/sdk/types.py,sha256=kj-xIe3uK2WPu47RjP6xIMWnasv_SQybptkx1OVovq8,5486
 lmnr/sdk/utils.py,sha256=s81p6uJehgJSaLWy3sR5fTpEDH7vzn3i_UujUHChl6M,3346
 lmnr/traceloop_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
 lmnr/traceloop_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
@@ -36,17 +36,17 @@ lmnr/traceloop_sdk/tests/test_sdk_initialization.py,sha256=fRaf6lrxFzJIN94P1Tav_
 lmnr/traceloop_sdk/tests/test_tasks.py,sha256=xlEx8BKp4yG83SCjK5WkPGfyC33JSrx4h8VyjVwGbgw,906
 lmnr/traceloop_sdk/tests/test_workflows.py,sha256=RVcfY3WAFIDZC15-aSua21aoQyYeWE7KypDyUsm-2EM,9372
 lmnr/traceloop_sdk/tracing/__init__.py,sha256=Ckq7zCM26VdJVB5tIZv0GTPyMZKyfso_KWD5yPHaqdo,66
-lmnr/traceloop_sdk/tracing/attributes.py,sha256=QzfFVhbuYnT-ym4lHocQF_e5zm5Aw8r36FTxV5Qrjlk,1136
+lmnr/traceloop_sdk/tracing/attributes.py,sha256=QeqItpCCwUipkwgXG7J7swJCD0yk9uuI28aepPhemtE,1201
 lmnr/traceloop_sdk/tracing/content_allow_list.py,sha256=3feztm6PBWNelc8pAZUcQyEGyeSpNiVKjOaDk65l2ps,846
 lmnr/traceloop_sdk/tracing/context_manager.py,sha256=csVlB6kDmbgSPsROHwnddvGGblx55v6lJMRj0wsSMQM,304
-lmnr/traceloop_sdk/tracing/tracing.py,sha256=_HDLuyy4XgobC1ig4qz5jYbB4tWAZSfD6gbgUqwmYJU,35522
+lmnr/traceloop_sdk/tracing/tracing.py,sha256=2Vyc0hUdTqXYlBzoW7YO6DlLaUKWouLv7eEvrVMZYOo,35680
 lmnr/traceloop_sdk/utils/__init__.py,sha256=pNhf0G3vTd5ccoc03i1MXDbricSaiqCbi1DLWhSekK8,604
 lmnr/traceloop_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQvzJk_fZH0OOsRAM1iZQXsR8,2112
 lmnr/traceloop_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
 lmnr/traceloop_sdk/utils/package_check.py,sha256=TZSngzJOpFhfUZLXIs38cpMxQiZSmp0D-sCrIyhz7BA,251
 lmnr/traceloop_sdk/version.py,sha256=OlatFEFA4ttqSSIiV8jdE-sq3KG5zu2hnC4B4mzWF3s,23
-lmnr-0.4.27.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.27.dist-info/METADATA,sha256=QI26rjCwIxbUPQYfq6HquwkjmEwq0YPP8d5VNfRHJ_M,10622
-lmnr-0.4.27.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-lmnr-0.4.27.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.4.27.dist-info/RECORD,,
+lmnr-0.4.29b0.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.29b0.dist-info/METADATA,sha256=ATL1IcRQIpALFB7apCO0TX0N2ve93ZxwytS61_vt_jA,10624
+lmnr-0.4.29b0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+lmnr-0.4.29b0.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.4.29b0.dist-info/RECORD,,