lmnr 0.4.53.dev0__py3-none-any.whl → 0.7.26__py3-none-any.whl

lmnr/cli/evals.py

@@ -0,0 +1,111 @@
+from argparse import Namespace
+
+import glob
+import importlib.util
+import json
+import os
+import re
+import sys
+
+from lmnr.sdk.evaluations import Evaluation
+from lmnr.sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCES
+from lmnr.sdk.log import get_default_logger
+
+LOG = get_default_logger(__name__)
+EVAL_DIR = "evals"
+DEFAULT_DATASET_PULL_BATCH_SIZE = 100
+DEFAULT_DATASET_PUSH_BATCH_SIZE = 100
+
+
+def log_evaluation_instance_not_found() -> None:
+    LOG.warning(
+        "Evaluation instance not found. "
+        "`evaluate` must be called at the top level of the file, "
+        "not inside a function when running evaluations from the CLI."
+    )
+
+
+async def run_evaluation(args: Namespace) -> None:
+    sys.path.append(os.getcwd())
+
+    if len(args.file) == 0:
+        files = [
+            os.path.join(EVAL_DIR, f)
+            for f in os.listdir(EVAL_DIR)
+            if re.match(r".*_eval\.py$", f) or re.match(r"eval_.*\.py$", f)
+        ]
+        if len(files) == 0:
+            LOG.error("No evaluation files found in `evals` directory")
+            LOG.info(
+                "Eval files must be located in the `evals` directory and must be named *_eval.py or eval_*.py"
+            )
+            return
+        files.sort()
+        LOG.info(f"Located {len(files)} evaluation files in {EVAL_DIR}")
+
+    else:
+        files = []
+        for pattern in args.file:
+            matches = glob.glob(pattern)
+            if matches:
+                files.extend(matches)
+            else:
+                # If no matches found, treat as literal filename
+                files.append(pattern)
+
+    prep_token = PREPARE_ONLY.set(True)
+    scores = []
+    try:
+        for file in files:
+            # Reset EVALUATION_INSTANCES before loading each file
+            EVALUATION_INSTANCES.set([])
+
+            LOG.info(f"Running evaluation from {file}")
+            file = os.path.abspath(file)
+            name = "user_module" + file
+
+            spec = importlib.util.spec_from_file_location(name, file)
+            if spec is None or spec.loader is None:
+                LOG.error(f"Could not load module specification from {file}")
+                if args.continue_on_error:
+                    continue
+                return
+            mod = importlib.util.module_from_spec(spec)
+            sys.modules[name] = mod
+
+            spec.loader.exec_module(mod)
+            evaluations = []
+            try:
+                evaluations: list[Evaluation] | None = EVALUATION_INSTANCES.get()
+                if evaluations is None:
+                    raise LookupError()
+            # may be raised by `get()` or manually by us above
+            except LookupError:
+                log_evaluation_instance_not_found()
+                if args.continue_on_error:
+                    continue
+                return
+
+            LOG.info(f"Loaded {len(evaluations)} evaluations from {file}")
+
+            for evaluation in evaluations:
+                try:
+                    eval_result = await evaluation.run()
+                    scores.append(
+                        {
+                            "file": file,
+                            "scores": eval_result["average_scores"],
+                            "evaluation_id": str(eval_result["evaluation_id"]),
+                            "url": eval_result["url"],
+                        }
+                    )
+                except Exception as e:
+                    LOG.error(f"Error running evaluation: {e}")
+                    if not args.continue_on_error:
+                        raise
+
+        if args.output_file:
+            with open(args.output_file, "w") as f:
+                json.dump(scores, f, indent=2)
+    finally:
+        PREPARE_ONLY.reset(prep_token)

lmnr/cli/rules.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+import urllib.request
+import urllib.error
+import sys
+
+
+from lmnr.sdk.log import get_default_logger
+
+LOG = get_default_logger(__name__)
+
+
+def add_cursor_rules() -> None:
+    """Download laminar.mdc file from a hardcoded public URL and save it to .cursor/rules/laminar.mdc"""
+    # Hardcoded URL for the laminar.mdc file
+    url = "https://raw.githubusercontent.com/lmnr-ai/lmnr/dev/rules/laminar.mdc"
+
+    # Create .cursor/rules directory if it doesn't exist
+    rules_dir = Path(".cursor/rules")
+    rules_dir.mkdir(parents=True, exist_ok=True)
+
+    # Define the target file path
+    target_file = rules_dir / "laminar.mdc"
+
+    try:
+        LOG.info(f"Downloading laminar.mdc from {url}")
+
+        # Download the file
+        with urllib.request.urlopen(url) as response:
+            content = response.read()
+
+        # Write the content to the target file (this will overwrite if it exists)
+        with open(target_file, "wb") as f:
+            f.write(content)
+
+        LOG.info(f"Successfully downloaded laminar.mdc to {target_file}")
+
+    except urllib.error.URLError as e:
+        LOG.error(f"Failed to download file from {url}: {e}")
+        sys.exit(1)
+    except Exception as e:
+        LOG.error(f"Unexpected error: {e}")
+        sys.exit(1)

lmnr/opentelemetry_lib/__init__.py

@@ -0,0 +1,70 @@
+import logging
+import sys
+
+from opentelemetry.sdk.trace.export import SpanExporter
+from opentelemetry.sdk.resources import SERVICE_NAME
+
+from lmnr.opentelemetry_lib.tracing.instruments import Instruments
+from lmnr.opentelemetry_lib.tracing import TracerWrapper
+from lmnr.sdk.types import SessionRecordingOptions
+
+
+class TracerManager:
+    __tracer_wrapper: TracerWrapper
+
+    @staticmethod
+    def init(
+        app_name: str | None = sys.argv[0],
+        disable_batch=False,
+        exporter: SpanExporter | None = None,
+        resource_attributes: dict = {},
+        instruments: set[Instruments] | None = None,
+        block_instruments: set[Instruments] | None = None,
+        base_url: str = "https://api.lmnr.ai",
+        port: int = 8443,
+        http_port: int = 443,
+        project_api_key: str | None = None,
+        max_export_batch_size: int | None = None,
+        force_http: bool = False,
+        timeout_seconds: int = 30,
+        set_global_tracer_provider: bool = True,
+        otel_logger_level: int = logging.ERROR,
+        session_recording_options: SessionRecordingOptions | None = None,
+    ) -> None:
+        enable_content_tracing = True
+
+        # Tracer init
+        resource_attributes.update({SERVICE_NAME: app_name})
+        TracerWrapper.set_static_params(resource_attributes, enable_content_tracing)
+        TracerManager.__tracer_wrapper = TracerWrapper(
+            disable_batch=disable_batch,
+            exporter=exporter,
+            instruments=instruments,
+            block_instruments=block_instruments,
+            base_url=base_url,
+            port=port,
+            http_port=http_port,
+            project_api_key=project_api_key,
+            max_export_batch_size=max_export_batch_size,
+            force_http=force_http,
+            timeout_seconds=timeout_seconds,
+            set_global_tracer_provider=set_global_tracer_provider,
+            otel_logger_level=otel_logger_level,
+            session_recording_options=session_recording_options,
+        )
+
+    @staticmethod
+    def flush() -> bool:
+        if not hasattr(TracerManager, "_TracerManager__tracer_wrapper"):
+            return False
+        return TracerManager.__tracer_wrapper.flush()
+
+    @staticmethod
+    def shutdown():
+        TracerManager.__tracer_wrapper.shutdown()
+
+    @staticmethod
+    def force_reinit_processor():
+        if not hasattr(TracerManager, "_TracerManager__tracer_wrapper"):
+            return False
+        return TracerManager.__tracer_wrapper.force_reinit_processor()

lmnr/opentelemetry_lib/decorators/__init__.py

@@ -0,0 +1,337 @@
+from functools import wraps
+import types
+from typing import Any, AsyncGenerator, Callable, Generator, Literal, TypeVar
+
+from opentelemetry import context as context_api
+from opentelemetry.trace import Span, Status, StatusCode
+
+from lmnr.opentelemetry_lib.tracing.context import (
+    CONTEXT_METADATA_KEY,
+    attach_context,
+    detach_context,
+    get_event_attributes_from_context,
+)
+from lmnr.opentelemetry_lib.tracing.span import LaminarSpan
+from lmnr.opentelemetry_lib.tracing.utils import set_association_props_in_context
+from lmnr.sdk.utils import get_input_from_func_args, is_method
+from lmnr.opentelemetry_lib.tracing.tracer import get_tracer_with_context
+from lmnr.opentelemetry_lib.tracing.attributes import (
+    ASSOCIATION_PROPERTIES,
+    METADATA,
+    SPAN_TYPE,
+)
+from lmnr.opentelemetry_lib.tracing import TracerWrapper
+from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.utils import is_otel_attribute_value_type, json_dumps
+
+logger = get_default_logger(__name__)
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def _setup_span(
+    span_name: str,
+    span_type: str,
+    association_properties: dict[str, Any] | None,
+    preserve_global_context: bool = False,
+    metadata: dict[str, Any] | None = None,
+):
+    """Set up a span with the given name, type, and association properties."""
+    with get_tracer_with_context() as (tracer, isolated_context):
+        # Create span in isolated context
+        span = tracer.start_span(
+            span_name,
+            context=isolated_context if not preserve_global_context else None,
+            attributes={SPAN_TYPE: span_type},
+        )
+
+        ctx_metadata = context_api.get_value(CONTEXT_METADATA_KEY, isolated_context)
+        merged_metadata = {
+            **(ctx_metadata or {}),
+            **(metadata or {}),
+        }
+        for key, value in merged_metadata.items():
+            span.set_attribute(
+                f"{ASSOCIATION_PROPERTIES}.{METADATA}.{key}",
+                (value if is_otel_attribute_value_type(value) else json_dumps(value)),
+            )
+
+        if association_properties is not None:
+            for key, value in association_properties.items():
+                span.set_attribute(f"{ASSOCIATION_PROPERTIES}.{key}", value)
+
+        return span
+
+
+def _process_input(
+    span: Span,
+    fn: Callable,
+    args: tuple,
+    kwargs: dict,
+    ignore_input: bool,
+    ignore_inputs: list[str] | None,
+    input_formatter: Callable[..., str] | None,
+):
+    """Process and set input attributes on the span."""
+    if ignore_input:
+        return
+
+    try:
+        if input_formatter is not None:
+            inp = input_formatter(*args, **kwargs)
+        else:
+            inp = get_input_from_func_args(
+                fn,
+                is_method=is_method(fn),
+                func_args=args,
+                func_kwargs=kwargs,
+                ignore_inputs=ignore_inputs,
+            )
+
+        if not isinstance(span, LaminarSpan):
+            span = LaminarSpan(span)
+        span.set_input(inp)
+    except Exception:
+        msg = "Failed to process input, ignoring"
+        if input_formatter is not None:
+            # Only warn the user if they provided an input formatter
+            # because it's their responsibility to make sure it works.
+            logger.warning(msg, exc_info=True)
+        else:
+            logger.debug(msg, exc_info=True)
+        pass
+
+
+def _process_output(
+    span: Span,
+    result: Any,
+    ignore_output: bool,
+    output_formatter: Callable[..., str] | None,
+):
+    """Process and set output attributes on the span."""
+    if ignore_output:
+        return
+
+    try:
+        if output_formatter is not None:
+            output = output_formatter(result)
+        else:
+            output = result
+
+        if not isinstance(span, LaminarSpan):
+            span = LaminarSpan(span)
+        span.set_output(output)
+    except Exception:
+        msg = "Failed to process output, ignoring"
+        if output_formatter is not None:
+            # Only warn the user if they provided an output formatter
+            # because it's their responsibility to make sure it works.
+            logger.warning(msg, exc_info=True)
+        else:
+            logger.debug(msg, exc_info=True)
+        pass
+
+
+def _cleanup_span(span: Span, wrapper: TracerWrapper):
+    """Clean up span and context."""
+    span.end()
+    wrapper.pop_span_context()
+
+
+def observe_base(
+    *,
+    name: str | None = None,
+    ignore_input: bool = False,
+    ignore_inputs: list[str] | None = None,
+    ignore_output: bool = False,
+    span_type: Literal["DEFAULT", "LLM", "TOOL"] = "DEFAULT",
+    metadata: dict[str, Any] | None = None,
+    association_properties: dict[str, Any] | None = None,
+    input_formatter: Callable[..., str] | None = None,
+    output_formatter: Callable[..., str] | None = None,
+    preserve_global_context: bool = False,
+) -> Callable[[F], F]:
+    def decorate(fn: F) -> F:
+        @wraps(fn)
+        def wrap(*args, **kwargs):
+            if not TracerWrapper.verify_initialized():
+                return fn(*args, **kwargs)
+
+            span_name = name or fn.__name__
+            wrapper = TracerWrapper()
+
+            span = _setup_span(
+                span_name,
+                span_type,
+                association_properties,
+                preserve_global_context,
+                metadata,
+            )
+
+            # Set association props in context before push_span_context
+            # so child spans inherit them
+            assoc_props_token = set_association_props_in_context(span)
+            if assoc_props_token and isinstance(span, LaminarSpan):
+                span._lmnr_assoc_props_token = assoc_props_token
+
+            new_context = wrapper.push_span_context(span)
+            # Some auto-instrumentations are not under our control, so they
+            # don't have access to our isolated context. We attach the context
+            # to the OTEL global context, so that spans know their parent
+            # span and trace_id.
+            ctx_token = context_api.attach(new_context)
+            # update our isolated context too
+            isolated_ctx_token = attach_context(new_context)
+
+            _process_input(
+                span, fn, args, kwargs, ignore_input, ignore_inputs, input_formatter
+            )
+
+            try:
+                res = fn(*args, **kwargs)
+            except Exception as e:
+                _process_exception(span, e)
+                _cleanup_span(span, wrapper)
+                raise
+            finally:
+                # Always restore global context
+                context_api.detach(ctx_token)
+                detach_context(isolated_ctx_token)
+            # span will be ended in the generator
+            if isinstance(res, types.GeneratorType):
+                return _handle_generator(span, wrapper, res)
+            if isinstance(res, types.AsyncGeneratorType):
+                # async def foo() -> AsyncGenerator[int, None]:
+                # is not considered async in a classical sense in Python,
+                # so we handle this inside the sync wrapper.
+                # In particular, CO_COROUTINE is different from CO_ASYNC_GENERATOR.
+                # Flags are listed from LSB here:
+                # https://docs.python.org/3/library/inspect.html#inspect-module-co-flags
+                # See also: https://groups.google.com/g/python-tulip/c/6rWweGXLutU?pli=1
+                return _ahandle_generator(span, wrapper, res)
+
+            _process_output(span, res, ignore_output, output_formatter)
+            _cleanup_span(span, wrapper)
+            return res
+
+        return wrap
+
+    return decorate
+
+
+# Async Decorators
+def async_observe_base(
+    *,
+    name: str | None = None,
+    ignore_input: bool = False,
+    ignore_inputs: list[str] | None = None,
+    ignore_output: bool = False,
+    span_type: Literal["DEFAULT", "LLM", "TOOL"] = "DEFAULT",
+    metadata: dict[str, Any] | None = None,
+    association_properties: dict[str, Any] | None = None,
+    input_formatter: Callable[..., str] | None = None,
+    output_formatter: Callable[..., str] | None = None,
+    preserve_global_context: bool = False,
+) -> Callable[[F], F]:
+    def decorate(fn: F) -> F:
+        @wraps(fn)
+        async def wrap(*args, **kwargs):
+            if not TracerWrapper.verify_initialized():
+                return await fn(*args, **kwargs)
+
+            span_name = name or fn.__name__
+            wrapper = TracerWrapper()
+
+            span = _setup_span(
+                span_name,
+                span_type,
+                association_properties,
+                preserve_global_context,
+                metadata,
+            )
+
+            # Set association props in context before push_span_context
+            # so child spans inherit them
+            assoc_props_token = set_association_props_in_context(span)
+            if assoc_props_token and isinstance(span, LaminarSpan):
+                span._lmnr_assoc_props_token = assoc_props_token
+
+            new_context = wrapper.push_span_context(span)
+            # Some auto-instrumentations are not under our control, so they
+            # don't have access to our isolated context. We attach the context
+            # to the OTEL global context, so that spans know their parent
+            # span and trace_id.
+            ctx_token = context_api.attach(new_context)
+            # update our isolated context too
+            isolated_ctx_token = attach_context(new_context)
+
+            _process_input(
+                span, fn, args, kwargs, ignore_input, ignore_inputs, input_formatter
+            )
+
+            try:
+                res = await fn(*args, **kwargs)
+            except Exception as e:
+                _process_exception(span, e)
+                _cleanup_span(span, wrapper)
+                raise e
+            finally:
+                # Always restore global context
+                context_api.detach(ctx_token)
+                detach_context(isolated_ctx_token)
+
+            # span will be ended in the generator
+            if isinstance(res, types.AsyncGeneratorType):
+                # probably unreachable, read the comment in the similar
+                # part of the sync wrapper.
+                return await _ahandle_generator(span, wrapper, res)
+
+            _process_output(span, res, ignore_output, output_formatter)
+            _cleanup_span(span, wrapper)
+            return res
+
+        return wrap
+
+    return decorate
+
+
+def _handle_generator(
+    span: Span,
+    wrapper: TracerWrapper,
+    res: Generator,
+    ignore_output: bool = False,
+    output_formatter: Callable[..., str] | None = None,
+):
+    results = []
+    try:
+        for part in res:
+            results.append(part)
+            yield part
+    finally:
+        _process_output(span, results, ignore_output, output_formatter)
+        _cleanup_span(span, wrapper)
+
+
+async def _ahandle_generator(
+    span: Span,
+    wrapper: TracerWrapper,
+    res: AsyncGenerator,
+    ignore_output: bool = False,
+    output_formatter: Callable[..., str] | None = None,
+):
+    results = []
+    try:
+        async for part in res:
+            results.append(part)
+            yield part
+    finally:
+        _process_output(span, results, ignore_output, output_formatter)
+        _cleanup_span(span, wrapper)
+
+
+def _process_exception(span: Span, e: Exception):
+    # Note that this `escaped` is sent as a StringValue("True"), not a boolean.
+    span.record_exception(
+        e, attributes=get_event_attributes_from_context(), escaped=True
+    )
+    span.set_status(Status(StatusCode.ERROR, str(e)))