arize-phoenix 0.0.32rc1__py3-none-any.whl → 0.0.33__py3-none-any.whl

phoenix/experimental/callbacks/langchain_tracer.py

@@ -0,0 +1,228 @@
+import json
+import logging
+from copy import deepcopy
+from datetime import datetime
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+from langchain.callbacks.tracers.base import BaseTracer
+from langchain.callbacks.tracers.schemas import Run
+
+from phoenix.trace.exporter import HttpExporter
+from phoenix.trace.schemas import (
+    Span,
+    SpanEvent,
+    SpanException,
+    SpanKind,
+    SpanStatusCode,
+)
+from phoenix.trace.semantic_conventions import (
+    DOCUMENT_CONTENT,
+    DOCUMENT_METADATA,
+    INPUT_MIME_TYPE,
+    INPUT_VALUE,
+    LLM_FUNCTION_CALL,
+    LLM_INVOCATION_PARAMETERS,
+    LLM_MODEL_NAME,
+    LLM_PROMPT_TEMPLATE,
+    LLM_PROMPT_TEMPLATE_VARIABLES,
+    LLM_PROMPT_TEMPLATE_VERSION,
+    LLM_TOKEN_COUNT_COMPLETION,
+    LLM_TOKEN_COUNT_PROMPT,
+    LLM_TOKEN_COUNT_TOTAL,
+    OUTPUT_MIME_TYPE,
+    OUTPUT_VALUE,
+    RETRIEVAL_DOCUMENTS,
+    TOOL_DESCRIPTION,
+    TOOL_NAME,
+    MimeType,
+)
+from phoenix.trace.tracer import Tracer
+
+logger = logging.getLogger(__name__)
+
+
+def _langchain_run_type_to_span_kind(run_type: str) -> SpanKind:
+    # TODO: LangChain is moving away from enums and to arbitrary strings
+    # for the run_type variable, so we may need to do the same
+    try:
+        return SpanKind(run_type.upper())
+    except ValueError:
+        return SpanKind.UNKNOWN
+
+
+def _serialize_json(obj: Any) -> str:
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    return str(obj)
+
+
+def _convert_io(obj: Optional[Dict[str, Any]]) -> Iterator[Any]:
+    if not obj:
+        return
+    if not isinstance(obj, dict):
+        raise ValueError(f"obj should be dict, but obj={obj}")
+    if len(obj) == 1 and isinstance(value := next(iter(obj.values())), str):
+        yield value
+    else:
+        yield json.dumps(obj, default=_serialize_json)
+        yield MimeType.JSON
+
+
+def _prompt_template(run_serialized: Dict[str, Any]) -> Iterator[Tuple[str, Any]]:
+    """
+    A best-effort attempt to locate the PromptTemplate object among the
+    keyword arguments of a serialized object, e.g. an LLMChain object.
+    """
+    for obj in run_serialized.get("kwargs", {}).values():
+        if not isinstance(obj, dict) or "id" not in obj:
+            continue
+        # The `id` field of the object is a list indicating the path to the
+        # object's class in the LangChain package, e.g. `PromptTemplate` in
+        # the `langchain.prompts.prompt` module is represented as
+        # ["langchain", "prompts", "prompt", "PromptTemplate"]
+        if obj["id"][-1].endswith("PromptTemplate"):
+            kwargs = obj.get("kwargs", {})
+            if not (template := kwargs.get("template", "")):
+                continue
+            yield LLM_PROMPT_TEMPLATE, template
+            yield LLM_PROMPT_TEMPLATE_VARIABLES, kwargs.get("input_variables", [])
+            yield LLM_PROMPT_TEMPLATE_VERSION, "unknown"
+            break
+
+
+def _invocation_parameters(run: Dict[str, Any]) -> Iterator[Tuple[str, str]]:
+    """Yields invocation parameters if present."""
+    if run["run_type"] != "llm":
+        return
+    run_extra = run["extra"]
+    yield LLM_INVOCATION_PARAMETERS, json.dumps(run_extra.get("invocation_params", {}))
+
+
+def _model_name(run_extra: Dict[str, Any]) -> Iterator[Tuple[str, str]]:
+    """Yields model name if present."""
+    if not (invocation_params := run_extra.get("invocation_params")):
+        return
+    for key in ["model_name", "model"]:
+        if name := invocation_params.get(key):
+            yield LLM_MODEL_NAME, name
+            return
+
+
+def _token_counts(run_outputs: Dict[str, Any]) -> Iterator[Tuple[str, int]]:
+    """Yields token count information if present."""
+    try:
+        token_usage = run_outputs["llm_output"]["token_usage"]
+    except Exception:
+        return
+    for attribute_name, key in [
+        (LLM_TOKEN_COUNT_PROMPT, "prompt_tokens"),
+        (LLM_TOKEN_COUNT_COMPLETION, "completion_tokens"),
+        (LLM_TOKEN_COUNT_TOTAL, "total_tokens"),
+    ]:
+        if (token_count := token_usage.get(key)) is not None:
+            yield attribute_name, token_count
+
+
+def _function_calls(run_outputs: Dict[str, Any]) -> Iterator[Tuple[str, str]]:
+    """Yields function call information if present."""
+    try:
+        function_call_data = deepcopy(
+            run_outputs["generations"][0][0]["message"]["kwargs"]["additional_kwargs"][
+                "function_call"
+            ]
+        )
+        function_call_data["arguments"] = json.loads(function_call_data["arguments"])
+        yield LLM_FUNCTION_CALL, json.dumps(function_call_data)
+    except Exception:
+        pass
+
+
+def _tools(run: Dict[str, Any]) -> Iterator[Tuple[str, str]]:
+    """Yields tool attributes if present."""
+    if run["run_type"] != "tool":
+        return
+    run_serialized = run["serialized"]
+    if "name" in run_serialized:
+        yield TOOL_NAME, run_serialized["name"]
+    if "description" in run_serialized:
+        yield TOOL_DESCRIPTION, run_serialized["description"]
+    # TODO: tool parameters https://github.com/Arize-ai/phoenix/issues/1330
+
+
+def _retrieval_documents(
+    run: Dict[str, Any],
+) -> Iterator[Tuple[str, List[Any]]]:
+    if run["run_type"] != "retriever":
+        return
+    yield RETRIEVAL_DOCUMENTS, [
+        {
+            DOCUMENT_CONTENT: document.get("page_content"),
+            DOCUMENT_METADATA: document.get("metadata") or {},
+        }
+        for document in (run.get("outputs") or {}).get("documents") or []
+    ]
+
+
+class OpenInferenceTracer(Tracer, BaseTracer):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._exporter = self._exporter or HttpExporter()
+
+    def _convert_run_to_spans(
+        self,
+        run: Dict[str, Any],
+        parent: Optional[Span] = None,
+    ) -> None:
+        attributes: Dict[str, Any] = {}
+        for io_key, io_attributes in {
+            "inputs": (INPUT_VALUE, INPUT_MIME_TYPE),
+            "outputs": (OUTPUT_VALUE, OUTPUT_MIME_TYPE),
+        }.items():
+            attributes.update(zip(io_attributes, _convert_io(run.get(io_key))))
+        attributes.update(_prompt_template(run["serialized"]))
+        attributes.update(_invocation_parameters(run))
+        attributes.update(_model_name(run["extra"]))
+        attributes.update(_token_counts(run["outputs"]))
+        attributes.update(_function_calls(run["outputs"]))
+        attributes.update(_tools(run))
+        attributes.update(_retrieval_documents(run))
+        events: List[SpanEvent] = []
+        if (error := run["error"]) is None:
+            status_code = SpanStatusCode.OK
+        else:
+            status_code = SpanStatusCode.ERROR
+            # Since there is only one error message, keep just the
+            # first error event.
+            error_event = next(
+                filter(
+                    lambda event: event["name"] == "error",
+                    run["events"],
+                )
+            )
+            events.append(
+                SpanException(
+                    message=error,
+                    timestamp=error_event["time"],
+                )
+            )
+        span = self.create_span(
+            name=run["name"],
+            span_kind=_langchain_run_type_to_span_kind(run["run_type"]),
+            parent_id=None if parent is None else parent.context.span_id,
+            trace_id=None if parent is None else parent.context.trace_id,
+            start_time=run["start_time"],
+            end_time=run["end_time"],
+            status_code=status_code,
+            attributes=attributes,
+            events=events,
+        )
+        for child_run in run["child_runs"]:
+            self._convert_run_to_spans(child_run, span)
+
+    def _persist_run(self, run: Run) -> None:
+        # Note that this relies on `.dict()` from pydantic for the
+        # serialization of objects like `langchain.schema.Document`.
+        try:
+            self._convert_run_to_spans(run.dict())
+        except Exception:
+            logger.exception("Failed to convert run to spans")

phoenix/experimental/callbacks/llama_index_trace_callback_handler.py

@@ -0,0 +1,364 @@
+"""
+Callback handler for emitting trace data in OpenInference tracing format.
+OpenInference tracing is an open standard for capturing and storing
+LLM Application execution logs.
+
+It enables production LLMapp servers to seamlessly integrate with LLM
+observability solutions such as Arize and Phoenix.
+
+For more information on the specification, see
+https://github.com/Arize-ai/open-inference-spec
+"""
+import json
+import logging
+from collections import defaultdict
+from datetime import datetime
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TypedDict, cast
+from uuid import uuid4
+
+from llama_index.callbacks.base_handler import BaseCallbackHandler
+from llama_index.callbacks.schema import (
+    TIMESTAMP_FORMAT,
+    CBEvent,
+    CBEventType,
+    EventPayload,
+)
+from llama_index.llms.base import ChatMessage, ChatResponse
+from llama_index.tools import ToolMetadata
+from openai.openai_object import OpenAIObject
+
+from phoenix.trace.exporter import HttpExporter
+from phoenix.trace.schemas import Span, SpanID, SpanKind, SpanStatusCode
+from phoenix.trace.semantic_conventions import (
+    DOCUMENT_CONTENT,
+    DOCUMENT_ID,
+    DOCUMENT_METADATA,
+    DOCUMENT_SCORE,
+    EMBEDDING_EMBEDDINGS,
+    EMBEDDING_MODEL_NAME,
+    EMBEDDING_TEXT,
+    EMBEDDING_VECTOR,
+    INPUT_MIME_TYPE,
+    INPUT_VALUE,
+    LLM_INVOCATION_PARAMETERS,
+    LLM_MESSAGES,
+    LLM_MODEL_NAME,
+    LLM_PROMPT,
+    LLM_TOKEN_COUNT_COMPLETION,
+    LLM_TOKEN_COUNT_PROMPT,
+    LLM_TOKEN_COUNT_TOTAL,
+    MESSAGE_CONTENT,
+    MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON,
+    MESSAGE_FUNCTION_CALL_NAME,
+    MESSAGE_NAME,
+    MESSAGE_ROLE,
+    OUTPUT_MIME_TYPE,
+    OUTPUT_VALUE,
+    RETRIEVAL_DOCUMENTS,
+    TOOL_DESCRIPTION,
+    TOOL_NAME,
+    TOOL_PARAMETERS,
+    MimeType,
+)
+from phoenix.trace.tracer import SpanExporter, Tracer
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.NullHandler())
+
+CBEventID = str
+_LOCAL_TZINFO = datetime.now().astimezone().tzinfo
+
+
+class CBEventData(TypedDict, total=False):
+    name: str
+    event_type: CBEventType
+    start_event: CBEvent
+    end_event: CBEvent
+    attributes: Dict[str, Any]
+
+
+def payload_to_semantic_attributes(
+    event_type: CBEventType,
+    payload: Dict[str, Any],
+) -> Dict[str, Any]:
+    """
+    Converts a LLMapp payload to a dictionary of semantic conventions compliant attributes.
+    """
+    attributes: Dict[str, Any] = {}
+    if event_type in (CBEventType.NODE_PARSING, CBEventType.CHUNKING):
+        # TODO(maybe): handle these events
+        return attributes
+    if EventPayload.CHUNKS in payload and EventPayload.EMBEDDINGS in payload:
+        attributes[EMBEDDING_EMBEDDINGS] = [
+            {EMBEDDING_TEXT: text, EMBEDDING_VECTOR: vector}
+            for text, vector in zip(payload[EventPayload.CHUNKS], payload[EventPayload.EMBEDDINGS])
+        ]
+    if EventPayload.QUERY_STR in payload:
+        attributes[INPUT_VALUE] = payload[EventPayload.QUERY_STR]
+        attributes[INPUT_MIME_TYPE] = MimeType.TEXT
+    if EventPayload.NODES in payload:
+        attributes[RETRIEVAL_DOCUMENTS] = [
+            {
+                DOCUMENT_ID: node_with_score.node.node_id,
+                DOCUMENT_SCORE: node_with_score.score,
+                DOCUMENT_CONTENT: node_with_score.node.text,
+                DOCUMENT_METADATA: node_with_score.node.metadata,
+            }
+            for node_with_score in payload[EventPayload.NODES]
+        ]
+    if EventPayload.PROMPT in payload:
+        attributes[LLM_PROMPT] = payload[EventPayload.PROMPT]
+    if EventPayload.MESSAGES in payload:
+        messages = payload[EventPayload.MESSAGES]
+        # Messages is only relevant to the LLM invocation
+        if event_type is CBEventType.LLM:
+            attributes[LLM_MESSAGES] = [
+                _message_payload_to_attributes(message_data) for message_data in messages
+            ]
+        elif event_type is CBEventType.AGENT_STEP and len(messages):
+            # the agent step contains a message that is actually the input
+            # akin to the query_str
+            attributes[INPUT_VALUE] = _message_payload_to_str(messages[0])
+    if response := (payload.get(EventPayload.RESPONSE) or payload.get(EventPayload.COMPLETION)):
+        attributes[OUTPUT_VALUE] = _get_response_content(response)
+        attributes[OUTPUT_MIME_TYPE] = MimeType.TEXT
+        if raw := getattr(response, "raw", None):
+            if isinstance(raw, OpenAIObject):
+                usage = raw.usage
+                attributes[LLM_TOKEN_COUNT_PROMPT] = usage.prompt_tokens
+                attributes[LLM_TOKEN_COUNT_COMPLETION] = usage.completion_tokens
+                attributes[LLM_TOKEN_COUNT_TOTAL] = usage.total_tokens
+    if EventPayload.TEMPLATE in payload:
+        ...
+    if event_type is CBEventType.RERANKING:
+        ...  # TODO
+        # if EventPayload.TOP_K in payload:
+        #     attributes[RERANKING_TOP_K] = payload[EventPayload.TOP_K]
+        # if EventPayload.MODEL_NAME in payload:
+        #     attributes[RERANKING_MODEL_NAME] = payload[EventPayload.MODEL_NAME]
+    if EventPayload.TOOL in payload:
+        tool_metadata = cast(ToolMetadata, payload.get(EventPayload.TOOL))
+        attributes[TOOL_NAME] = tool_metadata.name
+        attributes[TOOL_DESCRIPTION] = tool_metadata.description
+        attributes[TOOL_PARAMETERS] = tool_metadata.to_openai_function()["parameters"]
+    if EventPayload.SERIALIZED in payload:
+        serialized = payload[EventPayload.SERIALIZED]
+        if event_type is CBEventType.EMBEDDING:
+            if model_name := serialized.get("model_name"):
+                attributes[EMBEDDING_MODEL_NAME] = model_name
+        if event_type is CBEventType.LLM:
+            if model_name := serialized.get("model"):
+                attributes[LLM_MODEL_NAME] = model_name
+                attributes[LLM_INVOCATION_PARAMETERS] = json.dumps(
+                    {
+                        "model": model_name,
+                        "temperature": serialized["temperature"],
+                        "max_tokens": serialized["max_tokens"],
+                        **serialized["additional_kwargs"],
+                    }
+                )
+    return attributes
+
+
+class OpenInferenceTraceCallbackHandler(BaseCallbackHandler):
+    """Callback handler for storing LLM application trace data in OpenInference format.
+    OpenInference is an open standard for capturing and storing AI model
+    inferences. It enables production LLMapp servers to seamlessly integrate
+    with LLM observability solutions such as Arize and Phoenix.
+
+    For more information on the specification, see
+    https://github.com/Arize-ai/open-inference-spec
+    """
+
+    def __init__(
+        self,
+        callback: Optional[Callable[[List[Span]], None]] = None,
+        exporter: Optional[SpanExporter] = HttpExporter(),
+    ) -> None:
+        super().__init__(event_starts_to_ignore=[], event_ends_to_ignore=[])
+        self._tracer = Tracer(on_append=callback, exporter=exporter)
+        self._event_id_to_event_data: Dict[CBEventID, CBEventData] = defaultdict(
+            lambda: CBEventData()
+        )
+
+    def on_event_start(
+        self,
+        event_type: CBEventType,
+        payload: Optional[Dict[str, Any]] = None,
+        event_id: CBEventID = "",
+        **kwargs: Any,
+    ) -> CBEventID:
+        event_id = event_id or str(uuid4())
+        event_data = self._event_id_to_event_data[event_id]
+        event_data["name"] = event_type.value
+        event_data["event_type"] = event_type
+        event_data["start_event"] = CBEvent(
+            event_type=event_type,
+            payload=payload,
+            id_=event_id,
+        )
+        event_data["attributes"] = {}
+        # Parse the payload to extract the parameters
+        if payload is not None:
+            event_data["attributes"].update(
+                payload_to_semantic_attributes(event_type, payload),
+            )
+
+        return event_id
+
+    def on_event_end(
+        self,
+        event_type: CBEventType,
+        payload: Optional[Dict[str, Any]] = None,
+        event_id: CBEventID = "",
+        **kwargs: Any,
+    ) -> None:
+        event_data = self._event_id_to_event_data[event_id]
+        event_data.setdefault("name", event_type.value)
+        event_data.setdefault("event_type", event_type)
+        event_data["end_event"] = CBEvent(
+            event_type=event_type,
+            payload=payload,
+            id_=event_id,
+        )
+
+        # Parse the payload to extract the parameters
+        if payload is not None:
+            event_data["attributes"].update(
+                payload_to_semantic_attributes(event_type, payload),
+            )
+
+    def start_trace(self, trace_id: Optional[str] = None) -> None:
+        self._event_id_to_event_data = defaultdict(lambda: CBEventData())
+
+    def end_trace(
+        self,
+        trace_id: Optional[str] = None,
+        trace_map: Optional[Dict[CBEventID, List[CBEventID]]] = None,
+    ) -> None:
+        if not trace_map:
+            return  # TODO: investigate when empty or None trace_map is passed
+        try:
+            _add_to_tracer(
+                event_id_to_event_data=self._event_id_to_event_data,
+                trace_map=trace_map,
+                tracer=self._tracer,
+            )
+        except Exception:
+            logger.exception("OpenInferenceCallbackHandler trace processing failed")
+        self._event_id_to_event_data = defaultdict(lambda: CBEventData())
+
+    def get_spans(self) -> Iterator[Span]:
+        """
+        Returns the spans stored in the tracer. This is useful if you are running
+        LlamaIndex in a notebook environment and you want to inspect the spans.
+        """
+        return self._tracer.get_spans()
+
+
+def _add_to_tracer(
+    event_id_to_event_data: Dict[CBEventID, CBEventData],
+    trace_map: Dict[CBEventID, List[CBEventID]],
+    tracer: Tracer,
+) -> None:
+    """Adds event data to the tracer, where it is converted to a span and stored in a buffer.
+
+    Args:
+        event_id_to_event_data (Dict[CBEventID, CBEventData]): A map of event IDs to event data.
+
+        trace_map (Dict[CBEventID, List[CBEventID]]): A map of parent event IDs to child event IDs.
+        The root event IDs are stored under the key "root".
+
+        tracer (Tracer): The tracer that stores spans.
+    """
+
+    trace_id = uuid4()
+    parent_child_id_stack: List[Tuple[Optional[SpanID], CBEventID]] = [
+        (None, root_event_id) for root_event_id in trace_map["root"]
+    ]
+    while parent_child_id_stack:
+        parent_span_id, event_id = parent_child_id_stack.pop()
+        event_data = event_id_to_event_data[event_id]
+        start_event = event_data["start_event"]
+        start_time_tz_naive = datetime.strptime(start_event.time, TIMESTAMP_FORMAT)
+        start_time_tz_aware = start_time_tz_naive.replace(tzinfo=_LOCAL_TZINFO)
+        end_event = event_data["end_event"]
+        end_time_tz_naive = datetime.strptime(end_event.time, TIMESTAMP_FORMAT)
+        end_time_tz_aware = end_time_tz_naive.replace(tzinfo=_LOCAL_TZINFO)
+        name = event_data["name"]
+        event_type = event_data["event_type"]
+        span_kind = _get_span_kind(event_type)
+        span = tracer.create_span(
+            name=name,
+            span_kind=span_kind,
+            trace_id=trace_id,
+            start_time=start_time_tz_aware,
+            end_time=end_time_tz_aware,
+            status_code=SpanStatusCode.OK,
+            status_message="",
+            parent_id=parent_span_id,
+            attributes=event_data["attributes"],
+            events=None,
+            conversation=None,
+        )
+        new_parent_span_id = span.context.span_id
+        for new_child_event_id in trace_map.get(event_id, []):
+            parent_child_id_stack.append((new_parent_span_id, new_child_event_id))
+
+
+def _get_span_kind(event_type: CBEventType) -> SpanKind:
+    """Maps a CBEventType to a SpanKind.
+
+    Args:
+        event_type (CBEventType): LlamaIndex callback event type.
+
+    Returns:
+        SpanKind: The corresponding span kind.
+    """
+    return {
+        CBEventType.EMBEDDING: SpanKind.EMBEDDING,
+        CBEventType.LLM: SpanKind.LLM,
+        CBEventType.RETRIEVE: SpanKind.RETRIEVER,
+        CBEventType.FUNCTION_CALL: SpanKind.TOOL,
+        CBEventType.AGENT_STEP: SpanKind.AGENT,
+    }.get(event_type, SpanKind.CHAIN)
+
+
+def _message_payload_to_attributes(message: Any) -> Dict[str, Optional[str]]:
+    if isinstance(message, ChatMessage):
+        message_attributes = {
+            MESSAGE_ROLE: message.role.value,
+            MESSAGE_CONTENT: message.content,
+        }
+        # Parse the kwargs to extract the function name and parameters for function calling
+        # NB: these additional kwargs exist both for 'agent' and 'function' roles
+        if "name" in message.additional_kwargs:
+            message_attributes[MESSAGE_NAME] = message.additional_kwargs["name"]
+        if "function_call" in message.additional_kwargs:
+            function_call = message.additional_kwargs["function_call"]
+            message_attributes[MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON] = function_call.arguments
+            message_attributes[MESSAGE_FUNCTION_CALL_NAME] = function_call.name
+        return message_attributes
+
+    return {
+        MESSAGE_ROLE: "user",  # assume user if not ChatMessage
+        MESSAGE_CONTENT: str(message),
+    }
+
+
+def _message_payload_to_str(message: Any) -> Optional[str]:
+    """Converts a message payload to a string, if possible"""
+    if isinstance(message, ChatMessage):
+        return message.content
+
+    return str(message)
+
+
+def _get_response_content(response: Any) -> str:
+    """
+    Gets content from response objects. This is needed since the string representation of some
+    response objects includes extra information in addition to the content itself.
+    """
+    if isinstance(response, ChatResponse):
+        return response.message.content or ""
+    return str(response)

phoenix/experimental/evals/__init__.py

@@ -0,0 +1,33 @@
+from .functions import llm_eval_binary, llm_generate, run_relevance_eval
+from .models import OpenAiModel
+from .retrievals import compute_precisions_at_k
+from .templates import (
+    CODE_READABILITY_PROMPT_RAILS_MAP,
+    CODE_READABILITY_PROMPT_TEMPLATE_STR,
+    HALLUCINATION_PROMPT_RAILS_MAP,
+    HALLUCINATION_PROMPT_TEMPLATE_STR,
+    RAG_RELEVANCY_PROMPT_RAILS_MAP,
+    RAG_RELEVANCY_PROMPT_TEMPLATE_STR,
+    TOXICITY_PROMPT_RAILS_MAP,
+    TOXICITY_PROMPT_TEMPLATE_STR,
+    PromptTemplate,
+)
+from .utils.downloads import download_benchmark_dataset
+
+__all__ = [
+    "compute_precisions_at_k",
+    "download_benchmark_dataset",
+    "llm_eval_binary",
+    "llm_generate",
+    "OpenAiModel",
+    "PromptTemplate",
+    "CODE_READABILITY_PROMPT_RAILS_MAP",
+    "CODE_READABILITY_PROMPT_TEMPLATE_STR",
+    "HALLUCINATION_PROMPT_RAILS_MAP",
+    "HALLUCINATION_PROMPT_TEMPLATE_STR",
+    "RAG_RELEVANCY_PROMPT_RAILS_MAP",
+    "RAG_RELEVANCY_PROMPT_TEMPLATE_STR",
+    "TOXICITY_PROMPT_TEMPLATE_STR",
+    "TOXICITY_PROMPT_RAILS_MAP",
+    "run_relevance_eval",
+]

phoenix/experimental/evals/functions/__init__.py

@@ -0,0 +1,4 @@
+from .binary import llm_eval_binary, run_relevance_eval
+from .generate import llm_generate
+
+__all__ = ["llm_eval_binary", "run_relevance_eval", "llm_generate"]