langtrace-python-sdk 1.3.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

tests/anthropic/cassettes/test_async_anthropic_streaming.yaml

@@ -0,0 +1,328 @@
+interactions:
+- request:
+    body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "How are
+      you today?"}], "model": "claude-3-opus-20240229", "stream": true}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      anthropic-version:
+      - '2023-06-01'
+      connection:
+      - keep-alive
+      content-length:
+      - '136'
+      content-type:
+      - application/json
+      host:
+      - api.anthropic.com
+      user-agent:
+      - AsyncAnthropic/Python 0.25.6
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - async:asyncio
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 0.25.6
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.11.5
+    method: POST
+    uri: https://api.anthropic.com/v1/messages
+  response:
+    body:
+      string: 'event: message_start
+
+        data: {"type":"message_start","message":{"id":"msg_01WefXuaU9ctjjQqxDgV2v4r","type":"message","role":"assistant","model":"claude-3-opus-20240229","stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":1},"content":[],"stop_reason":null}}
+
+
+        event: content_block_start
+
+        data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}        }
+
+
+        event: ping
+
+        data: {"type": "ping"}
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"As"}  }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        an"}          }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        AI"}  }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        language"}}
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        model"}        }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}             }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        I"}    }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        don"}              }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"''t"}      }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        have"}}
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        feelings"}             }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}             }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        but"}    }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        I"}       }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"''m"}        }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        functioning"}}
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        well"}              }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        an"}        }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d
+        ready"}     }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        to"}              }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        assist"}       }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        you"}           }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        with"}            }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        any"}       }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        questions"}    }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        or"}}
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        tasks"}          }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        you"}}
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        may"}             }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        have"}         }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"."}  }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        How"}       }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        can"}   }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        I"}       }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        help"} }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        you"}       }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"
+        today"}      }
+
+
+        event: content_block_delta
+
+        data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"?"}       }
+
+
+        event: content_block_stop
+
+        data: {"type":"content_block_stop","index":0             }
+
+
+        event: message_delta
+
+        data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":41}               }
+
+
+        event: message_stop
+
+        data: {"type":"message_stop"        }
+
+
+        '
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 87963ba7cab90d92-MRS
+      Cache-Control:
+      - no-cache
+      Connection:
+      - keep-alive
+      Content-Type:
+      - text/event-stream; charset=utf-8
+      Date:
+      - Wed, 24 Apr 2024 12:52:54 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      anthropic-ratelimit-requests-limit:
+      - '50'
+      anthropic-ratelimit-requests-remaining:
+      - '49'
+      anthropic-ratelimit-requests-reset:
+      - '2024-04-24T12:53:29Z'
+      anthropic-ratelimit-tokens-limit:
+      - '20000'
+      anthropic-ratelimit-tokens-remaining:
+      - '19000'
+      anthropic-ratelimit-tokens-reset:
+      - '2024-04-24T12:53:29Z'
+      request-id:
+      - req_0165QboZpxrrNfD1oVun4cPB
+      via:
+      - 1.1 google
+    status:
+      code: 200
+      message: OK
+version: 1

tests/anthropic/conftest.py

@@ -0,0 +1,38 @@
+"""Unit tests configuration module."""
+
+import pytest
+import os
+from anthropic import Anthropic, AsyncAnthropic
+
+from langtrace_python_sdk.instrumentation.anthropic.instrumentation import (
+    AnthropicInstrumentation,
+)
+
+
+@pytest.fixture(autouse=True)
+def environment():
+    if not os.getenv("ANTHROPIC_API_KEY"):
+        os.environ["ANTHROPIC_API_KEY"] = "test_api_key"
+
+
+@pytest.fixture
+def anthropic_client():
+    return Anthropic()
+
+
+@pytest.fixture
+def async_anthropic_client():
+    return AsyncAnthropic()
+
+
+@pytest.fixture(scope="module")
+def vcr_config():
+    return {"filter_headers": ["authorization", "x-api-key"]}
+
+
+@pytest.fixture(scope="session", autouse=True)
+def instrument():
+    AnthropicInstrumentation().instrument()
+
+    yield
+    # exporter.shutdown()

tests/anthropic/test_anthropic.py

@@ -1,73 +1,109 @@
-import unittest
-from unittest.mock import MagicMock, call
-from langtrace_python_sdk.instrumentation.anthropic.patch import messages_create
-from opentelemetry.trace import SpanKind
-import importlib.metadata
-from langtrace_python_sdk.constants.instrumentation.anthropic import APIS
-from opentelemetry.trace.status import Status, StatusCode
+import pytest
 import json
-from langtrace.trace_attributes import Event, LLMSpanAttributes
-
-from tests.utils import common_setup
-
-class TestAnthropic(unittest.TestCase):
-
-    data = {
-        "content" : [MagicMock(text="Some text", type="text")],
-        "system_fingerprint" : "None",
-        "usage" : MagicMock(input_tokens=23, output_tokens=44),
-        "chunks" : [MagicMock(delta="Some text", message="text")]}
-
-
-    def setUp(self):
-        
-        # Mock the original method
-        self.anthropic_mock, self.tracer, self.span = common_setup(self.data, None)
-
-    def tearDown(self):
-        pass
-
-    def test_anthropic(self):
-        # Arrange
-        version = importlib.metadata.version('anthropic')        
-        kwargs = {
-            "model": "claude-3-opus-20240229",
-            "messages" : [{"role": "user", "content": "How are you today?"}],
-            "stream": False
-        }   
-
-        # Act
-        wrapped_function = messages_create("anthropic.messages.create", version, self.tracer)
-        result = wrapped_function(self.anthropic_mock, MagicMock(), (), kwargs)
-        
-
-        # Assert
-        self.assertTrue(self.tracer.start_as_current_span.called_once_with("anthropic.messages.create", kind=SpanKind.CLIENT))
-        self.assertTrue(self.span.set_status.has_calls([call(Status(StatusCode.OK))]))
-        
-        expected_attributes = {
-            "langtrace.sdk.name": "langtrace-python-sdk",
-            "langtrace.service.name": "Anthropic",
-            "langtrace.service.type": "llm",
-            "langtrace.service.version": version,
-            "langtrace.version": "1.0.0",
-            "url.full": "/v1/messages",
-            "llm.api": APIS["MESSAGES_CREATE"]["ENDPOINT"],
-            "llm.model": kwargs.get("model"),
-            "llm.prompts": json.dumps(kwargs.get("messages", [])),
-            "llm.stream": kwargs.get("stream"),
-        }
-
-        self.assertTrue(
-            self.span.set_attribute.has_calls(
-                [call(key, value) for key, value in expected_attributes.items()], any_order=True
-            )
-        )
-       
-        expected_result_data = {"system_fingerprint": "None"  }   
-
-        self.assertEqual(result.system_fingerprint, expected_result_data["system_fingerprint"])
-
-
-if __name__ == '__main__':
-    unittest.main()
+import importlib
+from langtrace_python_sdk.constants.instrumentation.anthropic import APIS
+
+
+@pytest.mark.vcr()
+def test_anthropic(anthropic_client, exporter):
+    llm_model_value = "claude-3-opus-20240229"
+    messages_value = [{"role": "user", "content": "How are you today?"}]
+
+    kwargs = {
+        "model": llm_model_value,
+        "messages": messages_value,
+        # "system": "Respond only in Yoda-speak.",
+        "stream": False,
+        "max_tokens": 1024,
+    }
+    response = anthropic_client.messages.create(**kwargs)
+    spans = exporter.get_finished_spans()
+    completion_span = spans[-1]
+
+    assert completion_span.name == "anthropic.messages.create"
+    attributes = completion_span.attributes
+
+    assert attributes.get("langtrace.sdk.name") == "langtrace-python-sdk"
+    assert attributes.get("langtrace.service.name") == "Anthropic"
+    assert attributes.get("langtrace.service.type") == "llm"
+    assert attributes.get("langtrace.service.version") == importlib.metadata.version(
+        "anthropic"
+    )
+    assert attributes.get("langtrace.version") == "1.0.0"
+    assert attributes.get("url.full") == "https://api.anthropic.com"
+    assert attributes.get("llm.api") == APIS["MESSAGES_CREATE"]["ENDPOINT"]
+    assert attributes.get("llm.model") == llm_model_value
+    assert attributes.get("llm.prompts") == json.dumps(messages_value)
+    assert attributes.get("llm.stream") is False
+
+    tokens = json.loads(attributes.get("llm.token.counts"))
+    output_tokens = tokens.get("output_tokens")
+    prompt_tokens = tokens.get("input_tokens")
+    total_tokens = tokens.get("total_tokens")
+
+    assert output_tokens and prompt_tokens and total_tokens
+    assert output_tokens + prompt_tokens == total_tokens
+
+    langtrace_responses = json.loads(attributes.get("llm.responses"))
+    assert isinstance(langtrace_responses, list)
+    for langtrace_response in langtrace_responses:
+        assert isinstance(langtrace_response, dict)
+        assert "role" in langtrace_response
+        assert "content" in langtrace_response
+
+
+@pytest.mark.vcr()
+def test_anthropic_streaming(anthropic_client, exporter):
+    llm_model_value = "claude-3-opus-20240229"
+    messages_value = [{"role": "user", "content": "How are you today?"}]
+
+    kwargs = {
+        "model": llm_model_value,
+        "messages": messages_value,
+        # "system": "Respond only in Yoda-speak.",
+        "stream": True,
+        "max_tokens": 1024,
+    }
+    response = anthropic_client.messages.create(**kwargs)
+    chunk_count = 0
+
+    for chunk in response:
+        if chunk:
+            chunk_count += 1
+
+    spans = exporter.get_finished_spans()
+    streaming_span = spans[-1]
+
+    assert streaming_span.name == "anthropic.messages.create"
+    attributes = streaming_span.attributes
+
+    assert attributes.get("langtrace.sdk.name") == "langtrace-python-sdk"
+    assert attributes.get("langtrace.service.name") == "Anthropic"
+    assert attributes.get("langtrace.service.type") == "llm"
+    assert attributes.get("langtrace.service.version") == importlib.metadata.version(
+        "anthropic"
+    )
+    assert attributes.get("langtrace.version") == "1.0.0"
+    assert attributes.get("url.full") == "https://api.anthropic.com"
+    assert attributes.get("llm.api") == APIS["MESSAGES_CREATE"]["ENDPOINT"]
+    assert attributes.get("llm.model") == llm_model_value
+    assert attributes.get("llm.prompts") == json.dumps(messages_value)
+    assert attributes.get("llm.stream") is True
+    events = streaming_span.events
+
+    assert len(events) - 2 == chunk_count  # -2 for start and end events
+
+    tokens = json.loads(attributes.get("llm.token.counts"))
+    output_tokens = tokens.get("output_tokens")
+    prompt_tokens = tokens.get("input_tokens")
+    total_tokens = tokens.get("total_tokens")
+
+    assert output_tokens and prompt_tokens and total_tokens
+    assert output_tokens + prompt_tokens == total_tokens
+
+    langtrace_responses = json.loads(attributes.get("llm.responses"))
+    assert isinstance(langtrace_responses, list)
+    for langtrace_response in langtrace_responses:
+        assert isinstance(langtrace_response, dict)
+        assert "role" in langtrace_response
+        assert "content" in langtrace_response

tests/conftest.py

@@ -0,0 +1,17 @@
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry import trace
+
+
+@pytest.fixture(scope="session")
+def exporter():
+    exporter = InMemorySpanExporter()
+    processor = SimpleSpanProcessor(exporter)
+
+    provider = TracerProvider()
+    provider.add_span_processor(processor)
+    trace.set_tracer_provider(provider)
+
+    return exporter

tests/openai/conftest.py

@@ -3,10 +3,6 @@
 import pytest
 import os
 from openai import OpenAI, AsyncOpenAI
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
-from opentelemetry.sdk.trace.export import SimpleSpanProcessor
-from opentelemetry import trace
 from langtrace_python_sdk.instrumentation.openai.instrumentation import (
     OpenAIInstrumentation,
 )
@@ -33,13 +29,9 @@ def vcr_config():
     return {"filter_headers": ["authorization", "api-key"]}
 
 
-@pytest.fixture(scope="session")
-def exporter():
-    exporter = InMemorySpanExporter()
-    processor = SimpleSpanProcessor(exporter)
-
-    provider = TracerProvider()
-    provider.add_span_processor(processor)
-    trace.set_tracer_provider(provider)
+@pytest.fixture(scope="session", autouse=True)
+def instrument():
     OpenAIInstrumentation().instrument()
-    return exporter
+
+    yield
+    # exporter.shutdown()

tests/openai/test_chat_completion.py

@@ -42,6 +42,13 @@ def test_chat_completion(exporter, openai_client):
     assert output_tokens and prompt_tokens and total_tokens
     assert output_tokens + prompt_tokens == total_tokens
 
+    langtrace_responses = json.loads(attributes.get("llm.responses"))
+    assert isinstance(langtrace_responses, list)
+    for langtrace_response in langtrace_responses:
+        assert isinstance(langtrace_response, dict)
+        assert "role" in langtrace_response
+        assert "content" in langtrace_response
+
 
 @pytest.mark.vcr()
 def test_chat_completion_streaming(exporter, openai_client):
@@ -91,6 +98,13 @@ def test_chat_completion_streaming(exporter, openai_client):
     assert output_tokens and prompt_tokens and total_tokens
     assert output_tokens + prompt_tokens == total_tokens
 
+    langtrace_responses = json.loads(attributes.get("llm.responses"))
+    assert isinstance(langtrace_responses, list)
+    for langtrace_response in langtrace_responses:
+        assert isinstance(langtrace_response, dict)
+        assert "role" in langtrace_response
+        assert "content" in langtrace_response
+
 
 @pytest.mark.vcr()
 @pytest.mark.asyncio()
@@ -140,3 +154,10 @@ async def test_async_chat_completion_streaming(exporter, async_openai_client):
 
     assert output_tokens and prompt_tokens and total_tokens
     assert output_tokens + prompt_tokens == total_tokens
+
+    langtrace_responses = json.loads(attributes.get("llm.responses"))
+    assert isinstance(langtrace_responses, list)
+    for langtrace_response in langtrace_responses:
+        assert isinstance(langtrace_response, dict)
+        assert "role" in langtrace_response
+        assert "content" in langtrace_response