arize-phoenix 4.20.1__tar.gz → 4.20.2__tar.gz

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 4.20.1
+Version: 4.20.2
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/examples/manually-instrumented-chatbot/chat-service/chat/app.py

@@ -1,9 +1,11 @@
 import json
 import os
+import warnings
 from typing import Any, Dict, Iterator, List, Tuple
+from uuid import uuid4
 
 from fastapi import FastAPI, HTTPException
-from httpx import AsyncClient
+from httpx import AsyncClient, ConnectError
 from openinference.semconv.trace import (
     MessageAttributes,
     OpenInferenceMimeTypeValues,
@@ -17,12 +19,14 @@ from opentelemetry.sdk.trace.export import (
     SimpleSpanProcessor,
 )
 
-from chat.types import Message, MessagesPayload, MessagesResponse
+from chat.types import FeedbackRequest, Message, MessagesPayload, MessagesResponse
 
 COLLECTOR_HOST = os.getenv("COLLECTOR_HOST", "localhost")
-endpoint = f"http://{COLLECTOR_HOST}:6006/v1/traces"
+endpoint = f"http://{COLLECTOR_HOST}:6006/v1"
 tracer_provider = trace_sdk.TracerProvider()
-tracer_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint)))
+tracer_provider.add_span_processor(
+    SimpleSpanProcessor(OTLPSpanExporter(f"{endpoint}/traces"))
+)
 trace_api.set_tracer_provider(tracer_provider)
 tracer = trace_api.get_tracer(__name__)
 
@@ -78,9 +82,11 @@ async def messages(messages_payload: MessagesPayload) -> MessagesResponse:
         span.set_status(trace_api.StatusCode.OK)
         response_data = response.json()
         assistant_message_content = response_data["choices"][0]["message"]["content"]
+        message_uuid = str(uuid4())
         assistant_message = Message(
             role="assistant",
             content=assistant_message_content,
+            uuid=message_uuid,
         )
         for (
             attribute_key,
@@ -91,9 +97,40 @@ async def messages(messages_payload: MessagesPayload) -> MessagesResponse:
             *_llm_token_usage_attributes(response_data),
         ):
             span.set_attribute(attribute_key, attribute_value)
+        span_id = span.get_span_context().span_id.to_bytes(8, "big").hex()
+        assistant_message.span_id = span_id
+
     return MessagesResponse(message=assistant_message)
 
 
+@app.post("/feedback/")
+async def post_feedback(feedback_request: FeedbackRequest) -> None:
+    if feedback_request.feedback == 1:
+        label = "thumbs_up"
+    elif feedback_request.feedback == 0:
+        label = "thumbs_down"
+
+    request_body = {
+        "data": [
+            {
+                "span_id": feedback_request.span_id,
+                "name": "correctness",
+                "annotator_kind": "HUMAN",
+                "result": {"label": label, "score": feedback_request.feedback},
+                "metadata": {},
+            }
+        ]
+    }
+
+    try:
+        await http_client.post(
+            f"{endpoint}/span_annotations",
+            json=request_body,
+        )
+    except ConnectError:
+        warnings.warn("Could not connect to Phoenix server.")
+
+
 def _llm_span_kind_attributes() -> Iterator[Tuple[str, str]]:
     """
     Yields the OpenInference span kind attribute for LLMs.

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/examples/manually-instrumented-chatbot/chat-service/chat/types.py

@@ -1,4 +1,4 @@
-from typing import List, Literal
+from typing import List, Literal, Optional
 
 from pydantic import BaseModel
 
@@ -8,6 +8,8 @@ Role = Literal["system", "assistant", "user"]
 class Message(BaseModel):
     role: Role
     content: str
+    uuid: str
+    span_id: Optional[str] = None
 
 
 class MessagesPayload(BaseModel):
@@ -16,3 +18,8 @@ class MessagesPayload(BaseModel):
 
 class MessagesResponse(BaseModel):
     message: Message
+
+
+class FeedbackRequest(BaseModel):
+    feedback: int
+    span_id: str

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/examples/manually-instrumented-chatbot/frontend/Dockerfile

@@ -10,6 +10,6 @@ FROM gcr.io/distroless/python3-debian12
 WORKDIR /app
 COPY --from=builder /app/.venv/lib/python3.11/site-packages/ /app/site-packages/
 ENV PYTHONPATH="/app/site-packages:$PYTHONPATH"
-COPY frontend/ /app/frontend/
+COPY src/ /app/src/
 EXPOSE 8501
-CMD ["-m", "streamlit", "run", "frontend/app.py", "--server.port=8501", "--server.address=0.0.0.0"]
+CMD ["-m", "streamlit", "run", "src/app.py", "--server.port=8501", "--server.address=0.0.0.0"]

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/examples/manually-instrumented-chatbot/frontend/Makefile

@@ -1,5 +1,5 @@
-server:
-	streamlit run app.py
+frontend:
+	streamlit run ./src/app.py
 
 format:
 	ruff check --fix . && ruff format .

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/src/phoenix/db/insertion/span.py

@@ -71,6 +71,13 @@ async def insert_span(
     cumulative_llm_token_count_completion = cast(
         int, get_attribute_value(span.attributes, SpanAttributes.LLM_TOKEN_COUNT_COMPLETION) or 0
     )
+    llm_token_count_prompt = cast(
+        Optional[int], get_attribute_value(span.attributes, SpanAttributes.LLM_TOKEN_COUNT_PROMPT)
+    )
+    llm_token_count_completion = cast(
+        Optional[int],
+        get_attribute_value(span.attributes, SpanAttributes.LLM_TOKEN_COUNT_COMPLETION),
+    )
     if accumulation := (
         await session.execute(
             select(
@@ -100,6 +107,8 @@ async def insert_span(
                 cumulative_error_count=cumulative_error_count,
                 cumulative_llm_token_count_prompt=cumulative_llm_token_count_prompt,
                 cumulative_llm_token_count_completion=cumulative_llm_token_count_completion,
+                llm_token_count_prompt=llm_token_count_prompt,
+                llm_token_count_completion=llm_token_count_completion,
             ),
             dialect=dialect,
             table=models.Span,

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/src/phoenix/db/migrations/versions/10460e46d750_datasets.py

@@ -6,11 +6,37 @@ Create Date: 2024-05-10 11:24:23.985834
 
 """
 
-from typing import Sequence, Union
+from typing import Any, Sequence, Union
 
 import sqlalchemy as sa
 from alembic import op
-from phoenix.db.migrations.types import JSON_
+from sqlalchemy import JSON
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.ext.compiler import compiles
+
+
+class JSONB(JSON):
+    # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
+    __visit_name__ = "JSONB"
+
+
+@compiles(JSONB, "sqlite")  # type: ignore
+def _(*args: Any, **kwargs: Any) -> str:
+    # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
+    return "JSONB"
+
+
+JSON_ = (
+    JSON()
+    .with_variant(
+        postgresql.JSONB(),  # type: ignore
+        "postgresql",
+    )
+    .with_variant(
+        JSONB(),
+        "sqlite",
+    )
+)
 
 # revision identifiers, used by Alembic.
 revision: str = "10460e46d750"

arize_phoenix-4.20.2/src/phoenix/db/migrations/versions/3be8647b87d8_add_token_columns_to_spans_table.py

@@ -0,0 +1,134 @@
+"""add token columns to spans table
+
+Revision ID: 3be8647b87d8
+Revises: 10460e46d750
+Create Date: 2024-08-03 22:11:28.733133
+
+"""
+
+from typing import Any, Dict, List, Optional, Sequence, TypedDict, Union
+
+import sqlalchemy as sa
+from alembic import op
+from openinference.semconv.trace import SpanAttributes
+from sqlalchemy import (
+    JSON,
+    Dialect,
+    MetaData,
+    TypeDecorator,
+    update,
+)
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.ext.asyncio.engine import AsyncConnection
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.orm import (
+    DeclarativeBase,
+    Mapped,
+    mapped_column,
+)
+
+
+class JSONB(JSON):
+    # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
+    __visit_name__ = "JSONB"
+
+
+@compiles(JSONB, "sqlite")  # type: ignore
+def _(*args: Any, **kwargs: Any) -> str:
+    # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
+    return "JSONB"
+
+
+JSON_ = (
+    JSON()
+    .with_variant(
+        postgresql.JSONB(),  # type: ignore
+        "postgresql",
+    )
+    .with_variant(
+        JSONB(),
+        "sqlite",
+    )
+)
+
+
+class JsonDict(TypeDecorator[Dict[str, Any]]):
+    # See # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
+    cache_ok = True
+    impl = JSON_
+
+    def process_bind_param(self, value: Optional[Dict[str, Any]], _: Dialect) -> Dict[str, Any]:
+        return value if isinstance(value, dict) else {}
+
+
+class JsonList(TypeDecorator[List[Any]]):
+    # See # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
+    cache_ok = True
+    impl = JSON_
+
+    def process_bind_param(self, value: Optional[List[Any]], _: Dialect) -> List[Any]:
+        return value if isinstance(value, list) else []
+
+
+class ExperimentRunOutput(TypedDict, total=False):
+    task_output: Any
+
+
+class Base(DeclarativeBase):
+    # Enforce best practices for naming constraints
+    # https://alembic.sqlalchemy.org/en/latest/naming.html#integration-of-naming-conventions-into-operations-autogenerate
+    metadata = MetaData(
+        naming_convention={
+            "ix": "ix_%(table_name)s_%(column_0_N_name)s",
+            "uq": "uq_%(table_name)s_%(column_0_N_name)s",
+            "ck": "ck_%(table_name)s_`%(constraint_name)s`",
+            "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+            "pk": "pk_%(table_name)s",
+        }
+    )
+    type_annotation_map = {
+        Dict[str, Any]: JsonDict,
+        List[Dict[str, Any]]: JsonList,
+        ExperimentRunOutput: JsonDict,
+    }
+
+
+class Span(Base):
+    __tablename__ = "spans"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    attributes: Mapped[Dict[str, Any]]
+    llm_token_count_prompt: Mapped[Optional[int]]
+    llm_token_count_completion: Mapped[Optional[int]]
+
+
+# revision identifiers, used by Alembic.
+revision: str = "3be8647b87d8"
+down_revision: Union[str, None] = "10460e46d750"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+LLM_TOKEN_COUNT_PROMPT = SpanAttributes.LLM_TOKEN_COUNT_PROMPT.split(".")
+LLM_TOKEN_COUNT_COMPLETION = SpanAttributes.LLM_TOKEN_COUNT_COMPLETION.split(".")
+
+
+async def get_token_counts_from_attributes(connection: AsyncConnection) -> None:
+    """
+    Gets token counts from attributes if present.
+    """
+    await connection.execute(
+        update(Span).values(
+            llm_token_count_prompt=Span.attributes[LLM_TOKEN_COUNT_PROMPT].as_float(),
+            llm_token_count_completion=Span.attributes[LLM_TOKEN_COUNT_COMPLETION].as_float(),
+        )
+    )
+
+
+def upgrade() -> None:
+    op.add_column("spans", sa.Column("llm_token_count_prompt", sa.Integer, nullable=True))
+    op.add_column("spans", sa.Column("llm_token_count_completion", sa.Integer, nullable=True))
+    op.run_async(get_token_counts_from_attributes)
+
+
+def downgrade() -> None:
+    op.drop_column("spans", "llm_token_count_completion")
+    op.drop_column("spans", "llm_token_count_prompt")

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/src/phoenix/db/migrations/versions/cf03bd6bae1d_init.py

@@ -6,11 +6,37 @@ Create Date: 2024-04-03 19:41:48.871555
 
 """
 
-from typing import Sequence, Union
+from typing import Any, Sequence, Union
 
 import sqlalchemy as sa
 from alembic import op
-from phoenix.db.migrations.types import JSON_
+from sqlalchemy import JSON
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.ext.compiler import compiles
+
+
+class JSONB(JSON):
+    # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
+    __visit_name__ = "JSONB"
+
+
+@compiles(JSONB, "sqlite")  # type: ignore
+def _(*args: Any, **kwargs: Any) -> str:
+    # See https://docs.sqlalchemy.org/en/20/core/custom_types.html
+    return "JSONB"
+
+
+JSON_ = (
+    JSON()
+    .with_variant(
+        postgresql.JSONB(),  # type: ignore
+        "postgresql",
+    )
+    .with_variant(
+        JSONB(),
+        "sqlite",
+    )
+)
 
 # revision identifiers, used by Alembic.
 revision: str = "cf03bd6bae1d"

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/src/phoenix/db/models.py

@@ -197,7 +197,7 @@ class Span(Base):
         ForeignKey("traces.id", ondelete="CASCADE"),
         index=True,
     )
-    span_id: Mapped[str]
+    span_id: Mapped[str] = mapped_column(index=True)
     parent_id: Mapped[Optional[str]] = mapped_column(index=True)
     name: Mapped[str]
     span_kind: Mapped[str]
@@ -214,6 +214,8 @@ class Span(Base):
     cumulative_error_count: Mapped[int]
     cumulative_llm_token_count_prompt: Mapped[int]
     cumulative_llm_token_count_completion: Mapped[int]
+    llm_token_count_prompt: Mapped[Optional[int]]
+    llm_token_count_completion: Mapped[Optional[int]]
 
     @hybrid_property
     def latency_ms(self) -> float:
@@ -230,6 +232,12 @@ class Span(Base):
     def cumulative_llm_token_count_total(self) -> int:
         return self.cumulative_llm_token_count_prompt + self.cumulative_llm_token_count_completion
 
+    @hybrid_property
+    def llm_token_count_total(self) -> Optional[int]:
+        if self.llm_token_count_prompt is None and self.llm_token_count_completion is None:
+            return None
+        return (self.llm_token_count_prompt or 0) + (self.llm_token_count_completion or 0)
+
     trace: Mapped["Trace"] = relationship("Trace", back_populates="spans")
     document_annotations: Mapped[List["DocumentAnnotation"]] = relationship(back_populates="span")
     dataset_examples: Mapped[List["DatasetExample"]] = relationship(back_populates="span")

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/src/phoenix/server/api/dataloaders/token_counts.py

@@ -10,7 +10,6 @@ from typing import (
 )
 
 from cachetools import LFUCache, TTLCache
-from openinference.semconv.trace import SpanAttributes
 from sqlalchemy import Select, func, select
 from sqlalchemy.sql.functions import coalesce
 from strawberry.dataloader import AbstractCache, DataLoader
@@ -107,8 +106,8 @@ def _get_stmt(
     *params: Param,
 ) -> Select[Any]:
     (start_time, end_time), filter_condition = segment
-    prompt = func.sum(models.Span.attributes[_LLM_TOKEN_COUNT_PROMPT].as_float())
-    completion = func.sum(models.Span.attributes[_LLM_TOKEN_COUNT_COMPLETION].as_float())
+    prompt = func.sum(models.Span.llm_token_count_prompt)
+    completion = func.sum(models.Span.llm_token_count_completion)
     total = coalesce(prompt, 0) + coalesce(completion, 0)
     pid = models.Trace.project_rowid
     stmt: Select[Any] = (
@@ -130,7 +129,3 @@ def _get_stmt(
         stmt = sf(stmt)
     stmt = stmt.where(pid.in_([rowid for rowid, _ in params]))
     return stmt
-
-
-_LLM_TOKEN_COUNT_PROMPT = SpanAttributes.LLM_TOKEN_COUNT_PROMPT.split(".")
-_LLM_TOKEN_COUNT_COMPLETION = SpanAttributes.LLM_TOKEN_COUNT_COMPLETION.split(".")

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/src/phoenix/server/api/input_types/SpanSort.py

@@ -3,7 +3,6 @@ from enum import Enum, auto
 from typing import Any, Optional, Protocol
 
 import strawberry
-from openinference.semconv.trace import SpanAttributes
 from sqlalchemy import and_, desc, nulls_last
 from sqlalchemy.orm import InstrumentedAttribute
 from sqlalchemy.sql.expression import Select
@@ -16,10 +15,6 @@ from phoenix.server.api.types.pagination import CursorSortColumnDataType
 from phoenix.server.api.types.SortDir import SortDir
 from phoenix.trace.schemas import SpanID
 
-LLM_TOKEN_COUNT_PROMPT = SpanAttributes.LLM_TOKEN_COUNT_PROMPT.split(".")
-LLM_TOKEN_COUNT_COMPLETION = SpanAttributes.LLM_TOKEN_COUNT_COMPLETION.split(".")
-LLM_TOKEN_COUNT_TOTAL = SpanAttributes.LLM_TOKEN_COUNT_TOTAL.split(".")
-
 
 @strawberry.enum
 class SpanColumn(Enum):
@@ -47,11 +42,11 @@ class SpanColumn(Enum):
         elif self is SpanColumn.latencyMs:
             expr = models.Span.latency_ms
         elif self is SpanColumn.tokenCountTotal:
-            expr = models.Span.attributes[LLM_TOKEN_COUNT_TOTAL].as_float()
+            expr = models.Span.llm_token_count_total
         elif self is SpanColumn.tokenCountPrompt:
-            expr = models.Span.attributes[LLM_TOKEN_COUNT_PROMPT].as_float()
+            expr = models.Span.llm_token_count_prompt
         elif self is SpanColumn.tokenCountCompletion:
-            expr = models.Span.attributes[LLM_TOKEN_COUNT_COMPLETION].as_float()
+            expr = models.Span.llm_token_count_completion
         elif self is SpanColumn.cumulativeTokenCountTotal:
             expr = (
                 models.Span.cumulative_llm_token_count_prompt

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/src/phoenix/server/api/types/Span.py

@@ -40,9 +40,6 @@ EMBEDDING_EMBEDDINGS = SpanAttributes.EMBEDDING_EMBEDDINGS
 EMBEDDING_VECTOR = EmbeddingAttributes.EMBEDDING_VECTOR
 INPUT_MIME_TYPE = SpanAttributes.INPUT_MIME_TYPE
 INPUT_VALUE = SpanAttributes.INPUT_VALUE
-LLM_TOKEN_COUNT_COMPLETION = SpanAttributes.LLM_TOKEN_COUNT_COMPLETION
-LLM_TOKEN_COUNT_PROMPT = SpanAttributes.LLM_TOKEN_COUNT_PROMPT
-LLM_TOKEN_COUNT_TOTAL = SpanAttributes.LLM_TOKEN_COUNT_TOTAL
 LLM_PROMPT_TEMPLATE_VARIABLES = SpanAttributes.LLM_PROMPT_TEMPLATE_VARIABLES
 LLM_INPUT_MESSAGES = SpanAttributes.LLM_INPUT_MESSAGES
 LLM_OUTPUT_MESSAGES = SpanAttributes.LLM_OUTPUT_MESSAGES
@@ -308,18 +305,9 @@ def to_gql_span(span: models.Span) -> Span:
         attributes=json.dumps(_hide_embedding_vectors(span.attributes), cls=_JSONEncoder),
         metadata=_convert_metadata_to_string(get_attribute_value(span.attributes, METADATA)),
         num_documents=num_documents,
-        token_count_total=cast(
-            Optional[int],
-            get_attribute_value(span.attributes, LLM_TOKEN_COUNT_TOTAL),
-        ),
-        token_count_prompt=cast(
-            Optional[int],
-            get_attribute_value(span.attributes, LLM_TOKEN_COUNT_PROMPT),
-        ),
-        token_count_completion=cast(
-            Optional[int],
-            get_attribute_value(span.attributes, LLM_TOKEN_COUNT_COMPLETION),
-        ),
+        token_count_total=span.llm_token_count_total,
+        token_count_prompt=span.llm_token_count_prompt,
+        token_count_completion=span.llm_token_count_completion,
         cumulative_token_count_total=span.cumulative_llm_token_count_prompt
         + span.cumulative_llm_token_count_completion,
         cumulative_token_count_prompt=span.cumulative_llm_token_count_prompt,

{arize_phoenix-4.20.1 → arize_phoenix-4.20.2}/src/phoenix/server/static/.vite/manifest.json

@@ -1,22 +1,22 @@
 {
-  "_components-CAummAJx.js": {
-    "file": "assets/components-CAummAJx.js",
+  "_components-BSw2e1Zr.js": {
+    "file": "assets/components-BSw2e1Zr.js",
     "name": "components",
     "imports": [
       "_vendor-BMWfu6zp.js",
-      "_vendor-arizeai-CkyzG9Wl.js",
-      "_pages-BU__X1UX.js",
+      "_vendor-arizeai-CIETbKDq.js",
+      "_pages-p_fuED5k.js",
       "_vendor-three-DwGkEfCM.js",
       "_vendor-codemirror-DO3VqEcD.js"
     ]
   },
-  "_pages-BU__X1UX.js": {
-    "file": "assets/pages-BU__X1UX.js",
+  "_pages-p_fuED5k.js": {
+    "file": "assets/pages-p_fuED5k.js",
     "name": "pages",
     "imports": [
       "_vendor-BMWfu6zp.js",
-      "_components-CAummAJx.js",
-      "_vendor-arizeai-CkyzG9Wl.js",
+      "_components-BSw2e1Zr.js",
+      "_vendor-arizeai-CIETbKDq.js",
       "_vendor-recharts-BGN0SxgJ.js",
       "_vendor-codemirror-DO3VqEcD.js"
     ]
@@ -35,8 +35,8 @@
       "assets/vendor-DxkFTwjz.css"
     ]
   },
-  "_vendor-arizeai-CkyzG9Wl.js": {
-    "file": "assets/vendor-arizeai-CkyzG9Wl.js",
+  "_vendor-arizeai-CIETbKDq.js": {
+    "file": "assets/vendor-arizeai-CIETbKDq.js",
     "name": "vendor-arizeai",
     "imports": [
       "_vendor-BMWfu6zp.js"
@@ -61,15 +61,15 @@
     "name": "vendor-three"
   },
   "index.tsx": {
-    "file": "assets/index-Cg5hdf3g.js",
+    "file": "assets/index-BYUFcdtx.js",
     "name": "index",
     "src": "index.tsx",
     "isEntry": true,
     "imports": [
       "_vendor-BMWfu6zp.js",
-      "_vendor-arizeai-CkyzG9Wl.js",
-      "_components-CAummAJx.js",
-      "_pages-BU__X1UX.js",
+      "_vendor-arizeai-CIETbKDq.js",
+      "_components-BSw2e1Zr.js",
+      "_pages-p_fuED5k.js",
       "_vendor-three-DwGkEfCM.js",
       "_vendor-codemirror-DO3VqEcD.js",
       "_vendor-recharts-BGN0SxgJ.js"