ddtrace 3.11.0rc1__cp311-cp311-macosx_12_0_arm64.whl → 3.11.0rc3__cp311-cp311-macosx_12_0_arm64.whl

ddtrace/llmobs/_experiment.py

@@ -21,6 +21,7 @@ from ddtrace.constants import ERROR_STACK
 from ddtrace.constants import ERROR_TYPE
 from ddtrace.internal.logger import get_logger
 from ddtrace.llmobs._constants import EXPERIMENT_EXPECTED_OUTPUT
+from ddtrace.llmobs._utils import convert_tags_dict_to_list
 
 
 if TYPE_CHECKING:
@@ -153,6 +154,12 @@ class Dataset:
         self._deleted_record_ids.append(record_id)
         del self._records[index]
 
+    @property
+    def url(self) -> str:
+        # FIXME: need to use the user's site
+        # also will not work for subdomain orgs
+        return f"https://app.datadoghq.com/llm/datasets/{self._id}"
+
     @overload
     def __getitem__(self, index: int) -> DatasetRecord:
         ...
@@ -170,6 +177,50 @@ class Dataset:
     def __iter__(self) -> Iterator[DatasetRecord]:
         return iter(self._records)
 
+    def as_dataframe(self) -> None:
+        try:
+            import pandas as pd
+        except ImportError as e:
+            raise ImportError(
+                "pandas is required to convert dataset to DataFrame. Please install via `pip install pandas`"
+            ) from e
+
+        column_tuples = set()
+        data_rows = []
+        for record in self._records:
+            flat_record = {}  # type: Dict[Union[str, Tuple[str, str]], Any]
+
+            input_data = record.get("input_data", {})
+            if isinstance(input_data, dict):
+                for input_data_col, input_data_val in input_data.items():
+                    flat_record[("input_data", input_data_col)] = input_data_val
+                    column_tuples.add(("input_data", input_data_col))
+            else:
+                flat_record[("input_data", "")] = input_data
+                column_tuples.add(("input_data", ""))
+
+            expected_output = record.get("expected_output", {})
+            if isinstance(expected_output, dict):
+                for expected_output_col, expected_output_val in expected_output.items():
+                    flat_record[("expected_output", expected_output_col)] = expected_output_val
+                    column_tuples.add(("expected_output", expected_output_col))
+            else:
+                flat_record[("expected_output", "")] = expected_output
+                column_tuples.add(("expected_output", ""))
+
+            for metadata_col, metadata_val in record.get("metadata", {}).items():
+                flat_record[("metadata", metadata_col)] = metadata_val
+                column_tuples.add(("metadata", metadata_col))
+
+            data_rows.append(flat_record)
+
+        records_list = []
+        for flat_record in data_rows:
+            row = [flat_record.get(col, None) for col in column_tuples]
+            records_list.append(row)
+
+        return pd.DataFrame(data=records_list, columns=pd.MultiIndex.from_tuples(column_tuples))
+
 
 class Experiment:
     def __init__(
@@ -180,7 +231,7 @@ class Experiment:
         evaluators: List[Callable[[DatasetRecordInputType, JSONType, JSONType], JSONType]],
         project_name: str,
         description: str = "",
-        tags: Optional[List[str]] = None,
+        tags: Optional[Dict[str, str]] = None,
         config: Optional[ExperimentConfigType] = None,
         _llmobs_instance: Optional["LLMObs"] = None,
     ) -> None:
@@ -189,8 +240,8 @@ class Experiment:
         self._dataset = dataset
         self._evaluators = evaluators
         self._description = description
-        self._tags: List[str] = [f"ddtrace.version:{ddtrace.__version__}"]
-        self._tags.extend(tags or [])
+        self._tags: Dict[str, str] = tags or {}
+        self._tags["ddtrace.version"] = str(ddtrace.__version__)
         self._config: Dict[str, JSONType] = config or {}
         self._llmobs_instance = _llmobs_instance
 
@@ -217,7 +268,8 @@ class Experiment:
         if not self._llmobs_instance.enabled:
             logger.warning(
                 "Skipping experiment as LLMObs is not enabled. "
-                "Ensure LLM Observability is enabled via `LLMObs.enable(...)` or set `DD_LLMOBS_ENABLED=1`."
+                "Ensure LLM Observability is enabled via `LLMObs.enable(...)` "
+                "or set `DD_LLMOBS_ENABLED=1` and use `ddtrace-run` to run your application."
             )
             return []
 
@@ -230,19 +282,27 @@ class Experiment:
             self._project_id,
             self._dataset._version,
             self._config,
-            self._tags,
+            convert_tags_dict_to_list(self._tags),
             self._description,
         )
         self._id = experiment_id
-        self._tags.append(f"experiment_id:{experiment_id}")
+        self._tags["experiment_id"] = str(experiment_id)
         self._run_name = experiment_run_name
         task_results = self._run_task(jobs, raise_errors, sample_size)
         evaluations = self._run_evaluators(task_results, raise_errors=raise_errors)
         experiment_results = self._merge_results(task_results, evaluations)
         experiment_evals = self._generate_metrics_from_exp_results(experiment_results)
-        self._llmobs_instance._dne_client.experiment_eval_post(self._id, experiment_evals, self._tags)
+        self._llmobs_instance._dne_client.experiment_eval_post(
+            self._id, experiment_evals, convert_tags_dict_to_list(self._tags)
+        )
         return experiment_results
 
+    @property
+    def url(self) -> str:
+        # FIXME: need to use the user's site
+        # also will not work for subdomain orgs
+        return f"https://app.datadoghq.com/llm/experiments/{self._id}"
+
     def _process_record(self, idx_record: Tuple[int, DatasetRecord]) -> Optional[TaskResult]:
         if not self._llmobs_instance or not self._llmobs_instance.enabled:
             return None
@@ -256,7 +316,12 @@ class Experiment:
                 span_id, trace_id = "", ""
             input_data = record["input_data"]
             record_id = record.get("record_id", "")
-            tags = {"dataset_id": self._dataset._id, "dataset_record_id": record_id, "experiment_id": self._id}
+            tags = {
+                **self._tags,
+                "dataset_id": str(self._dataset._id),
+                "dataset_record_id": str(record_id),
+                "experiment_id": str(self._id),
+            }
             output_data = None
             try:
                 output_data = self._task(input_data, self._config)
@@ -342,7 +407,7 @@ class Experiment:
         experiment_results = []
         for idx, task_result in enumerate(task_results):
             output_data = task_result["output"]
-            metadata: Dict[str, JSONType] = {"tags": cast(List[JSONType], self._tags)}
+            metadata: Dict[str, JSONType] = {"tags": cast(List[JSONType], convert_tags_dict_to_list(self._tags))}
             metadata.update(task_result.get("metadata") or {})
             record: DatasetRecord = self._dataset[idx]
             evals = evaluations[idx]["evaluations"]
@@ -383,7 +448,7 @@ class Experiment:
             "label": eval_name,
             f"{metric_type}_value": eval_value,  # type: ignore
             "error": err,
-            "tags": self._tags,
+            "tags": convert_tags_dict_to_list(self._tags),
             "experiment_id": self._id,
         }
 

ddtrace/llmobs/_integrations/bedrock.py

@@ -13,6 +13,7 @@ from ddtrace.llmobs._constants import CACHE_READ_INPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import CACHE_WRITE_INPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import INPUT_MESSAGES
 from ddtrace.llmobs._constants import INPUT_VALUE
+from ddtrace.llmobs._constants import INTEGRATION
 from ddtrace.llmobs._constants import METADATA
 from ddtrace.llmobs._constants import METRICS
 from ddtrace.llmobs._constants import MODEL_NAME
@@ -30,6 +31,7 @@ from ddtrace.llmobs._integrations.bedrock_utils import normalize_input_tokens
 from ddtrace.llmobs._integrations.utils import get_final_message_converse_stream_message
 from ddtrace.llmobs._integrations.utils import get_messages_from_converse_content
 from ddtrace.llmobs._integrations.utils import update_proxy_workflow_input_output_value
+from ddtrace.llmobs._telemetry import record_bedrock_agent_span_event_created
 from ddtrace.llmobs._writer import LLMObsSpanEvent
 from ddtrace.trace import Span
 
@@ -151,6 +153,7 @@ class BedrockIntegration(BaseLLMIntegration):
                 INPUT_VALUE: str(input_value),
                 TAGS: {"session_id": session_id},
                 METADATA: {"agent_id": agent_id, "agent_alias_id": agent_alias_id},
+                INTEGRATION: "bedrock_agents",
             }
         )
         if not response:
@@ -176,6 +179,7 @@ class BedrockIntegration(BaseLLMIntegration):
             )
         for _, span_event in self._spans.items():
             LLMObs._instance._llmobs_span_writer.enqueue(span_event)
+            record_bedrock_agent_span_event_created(span_event)
         self._spans.clear()
         self._active_span_by_step_id.clear()
 

ddtrace/llmobs/_integrations/bedrock_agents.py

@@ -15,6 +15,7 @@ from ddtrace.internal.utils.formats import format_trace_id
 from ddtrace.llmobs._constants import LLMOBS_TRACE_ID
 from ddtrace.llmobs._integrations.bedrock_utils import parse_model_id
 from ddtrace.llmobs._utils import _get_ml_app
+from ddtrace.llmobs._utils import _get_session_id
 from ddtrace.llmobs._utils import safe_json
 
 
@@ -57,12 +58,15 @@ def _build_span_event(
     llmobs_trace_id = root_span._get_ctx_item(LLMOBS_TRACE_ID)
     if llmobs_trace_id is None:
         llmobs_trace_id = root_span.trace_id
+    session_id = _get_session_id(root_span)
+    ml_app = _get_ml_app(root_span)
+    tags = [f"ml_app:{ml_app}", f"session_id:{session_id}", "integration:bedrock_agents"]
     span_event = {
         "name": span_name,
         "span_id": str(span_id),
         "trace_id": format_trace_id(llmobs_trace_id),
         "parent_id": str(parent_id or root_span.span_id),
-        "tags": ["ml_app:{}".format(_get_ml_app(root_span))],
+        "tags": tags,
         "start_ns": int(start_ns or root_span.start_ns),
         "duration": int(duration_ns or DEFAULT_SPAN_DURATION),
         "status": "error" if error else "ok",

ddtrace/llmobs/_integrations/crewai.py

@@ -8,6 +8,7 @@ from ddtrace.internal import core
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.utils import get_argument_value
 from ddtrace.internal.utils.formats import format_trace_id
+from ddtrace.llmobs._constants import AGENT_MANIFEST
 from ddtrace.llmobs._constants import INPUT_VALUE
 from ddtrace.llmobs._constants import METADATA
 from ddtrace.llmobs._constants import NAME
@@ -151,9 +152,8 @@ class CrewAIIntegration(BaseLLMIntegration):
         Agent spans are 1:1 with its parent (task/tool) span, so we link them directly here, even on the parent itself.
         """
         agent_instance = kwargs.get("instance")
+        self._tag_agent_manifest(span, agent_instance)
         agent_role = getattr(agent_instance, "role", "")
-        agent_goal = getattr(agent_instance, "goal", "")
-        agent_backstory = getattr(agent_instance, "backstory", "")
         task_description = getattr(kwargs.get("task"), "description", "")
         context = get_argument_value(args, kwargs, 1, "context", optional=True) or ""
 
@@ -174,7 +174,6 @@ class CrewAIIntegration(BaseLLMIntegration):
         span._set_ctx_items(
             {
                 NAME: agent_role if agent_role else "CrewAI Agent",
-                METADATA: {"description": agent_goal, "backstory": agent_backstory},
                 INPUT_VALUE: {"context": context, "input": task_description},
                 SPAN_LINKS: curr_span_links + [span_link],
             }
@@ -198,6 +197,56 @@ class CrewAIIntegration(BaseLLMIntegration):
             return
         span._set_ctx_item(OUTPUT_VALUE, response)
 
+    def _tag_agent_manifest(self, span, agent):
+        if not agent:
+            return
+
+        manifest = {}
+        manifest["framework"] = "CrewAI"
+        manifest["name"] = agent.role if hasattr(agent, "role") and agent.role else "CrewAI Agent"
+        if hasattr(agent, "goal"):
+            manifest["goal"] = agent.goal
+        if hasattr(agent, "backstory"):
+            manifest["backstory"] = agent.backstory
+        if hasattr(agent, "llm"):
+            if hasattr(agent.llm, "model"):
+                manifest["model"] = agent.llm.model
+            model_settings = {}
+            if hasattr(agent.llm, "max_tokens"):
+                model_settings["max_tokens"] = agent.llm.max_tokens
+            if hasattr(agent.llm, "temperature"):
+                model_settings["temperature"] = agent.llm.temperature
+            if model_settings:
+                manifest["model_settings"] = model_settings
+        if hasattr(agent, "allow_delegation"):
+            manifest["handoffs"] = {"allow_delegation": agent.allow_delegation}
+        code_execution_permissions = {}
+        if hasattr(agent, "allow_code_execution"):
+            manifest["code_execution_permissions"] = {"allow_code_execution": agent.allow_code_execution}
+        if hasattr(agent, "code_execution_mode"):
+            manifest["code_execution_permissions"] = {"code_execution_mode": agent.code_execution_mode}
+        if code_execution_permissions:
+            manifest["code_execution_permissions"] = code_execution_permissions
+        if hasattr(agent, "max_iter"):
+            manifest["max_iterations"] = agent.max_iter
+        if hasattr(agent, "tools"):
+            manifest["tools"] = self._get_agent_tools(agent.tools)
+
+        span._set_ctx_item(AGENT_MANIFEST, manifest)
+
+    def _get_agent_tools(self, tools):
+        if not tools or not isinstance(tools, list):
+            return []
+        formatted_tools = []
+        for tool in tools:
+            tool_dict = {}
+            if hasattr(tool, "name"):
+                tool_dict["name"] = tool.name
+            if hasattr(tool, "description"):
+                tool_dict["description"] = tool.description
+            formatted_tools.append(tool_dict)
+        return formatted_tools
+
     def _llmobs_set_span_link_on_task(self, span, args, kwargs):
         """Set span links for the next queued task in a CrewAI workflow.
         This happens between task executions, (the current span is the crew span and the task span hasn't started yet)

ddtrace/llmobs/_integrations/gemini.py

@@ -16,9 +16,9 @@ from ddtrace.llmobs._constants import OUTPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import SPAN_KIND
 from ddtrace.llmobs._constants import TOTAL_TOKENS_METRIC_KEY
 from ddtrace.llmobs._integrations.base import BaseLLMIntegration
-from ddtrace.llmobs._integrations.utils import extract_message_from_part_google
-from ddtrace.llmobs._integrations.utils import get_system_instructions_from_google_model
-from ddtrace.llmobs._integrations.utils import llmobs_get_metadata_google
+from ddtrace.llmobs._integrations.google_utils import extract_message_from_part_gemini_vertexai
+from ddtrace.llmobs._integrations.google_utils import get_system_instructions_gemini_vertexai
+from ddtrace.llmobs._integrations.google_utils import llmobs_get_metadata_gemini_vertexai
 from ddtrace.llmobs._utils import _get_attr
 from ddtrace.trace import Span
 
@@ -43,9 +43,9 @@ class GeminiIntegration(BaseLLMIntegration):
         operation: str = "",
     ) -> None:
         instance = kwargs.get("instance", None)
-        metadata = llmobs_get_metadata_google(kwargs, instance)
+        metadata = llmobs_get_metadata_gemini_vertexai(kwargs, instance)
 
-        system_instruction = get_system_instructions_from_google_model(instance)
+        system_instruction = get_system_instructions_gemini_vertexai(instance)
         input_contents = get_argument_value(args, kwargs, 0, "contents")
         input_messages = self._extract_input_message(input_contents, system_instruction)
 
@@ -95,7 +95,7 @@ class GeminiIntegration(BaseLLMIntegration):
                 messages.append(message)
                 continue
             for part in parts:
-                message = extract_message_from_part_google(part, role)
+                message = extract_message_from_part_gemini_vertexai(part, role)
                 messages.append(message)
         return messages
 
@@ -107,7 +107,7 @@ class GeminiIntegration(BaseLLMIntegration):
             role = content.get("role", "model")
             parts = content.get("parts", [])
             for part in parts:
-                message = extract_message_from_part_google(part, role)
+                message = extract_message_from_part_gemini_vertexai(part, role)
                 output_messages.append(message)
         return output_messages
 

ddtrace/llmobs/_integrations/google_genai.py

@@ -14,12 +14,12 @@ from ddtrace.llmobs._constants import OUTPUT_MESSAGES
 from ddtrace.llmobs._constants import OUTPUT_VALUE
 from ddtrace.llmobs._constants import SPAN_KIND
 from ddtrace.llmobs._integrations.base import BaseLLMIntegration
-from ddtrace.llmobs._integrations.google_genai_utils import DEFAULT_MODEL_ROLE
-from ddtrace.llmobs._integrations.google_genai_utils import extract_embedding_metrics_google_genai
-from ddtrace.llmobs._integrations.google_genai_utils import extract_generation_metrics_google_genai
-from ddtrace.llmobs._integrations.google_genai_utils import extract_message_from_part_google_genai
-from ddtrace.llmobs._integrations.google_genai_utils import extract_provider_and_model_name
-from ddtrace.llmobs._integrations.google_genai_utils import normalize_contents
+from ddtrace.llmobs._integrations.google_utils import GOOGLE_GENAI_DEFAULT_MODEL_ROLE
+from ddtrace.llmobs._integrations.google_utils import extract_embedding_metrics_google_genai
+from ddtrace.llmobs._integrations.google_utils import extract_generation_metrics_google_genai
+from ddtrace.llmobs._integrations.google_utils import extract_message_from_part_google_genai
+from ddtrace.llmobs._integrations.google_utils import extract_provider_and_model_name
+from ddtrace.llmobs._integrations.google_utils import normalize_contents_google_genai
 from ddtrace.llmobs._utils import _get_attr
 from ddtrace.llmobs.utils import Document
 
@@ -71,7 +71,7 @@ class GoogleGenAIIntegration(BaseLLMIntegration):
         response: Optional[Any] = None,
         operation: str = "",
     ) -> None:
-        provider_name, model_name = extract_provider_and_model_name(kwargs)
+        provider_name, model_name = extract_provider_and_model_name(kwargs=kwargs)
         span._set_ctx_items(
             {
                 SPAN_KIND: operation,
@@ -120,7 +120,7 @@ class GoogleGenAIIntegration(BaseLLMIntegration):
 
     def _extract_messages_from_contents(self, contents, default_role: str) -> List[Dict[str, Any]]:
         messages = []
-        for content in normalize_contents(contents):
+        for content in normalize_contents_google_genai(contents):
             role = content.get("role") or default_role
             for part in content.get("parts", []):
                 messages.append(extract_message_from_part_google_genai(part, role))
@@ -128,7 +128,7 @@ class GoogleGenAIIntegration(BaseLLMIntegration):
 
     def _extract_output_messages(self, response) -> List[Dict[str, Any]]:
         if not response:
-            return [{"content": "", "role": DEFAULT_MODEL_ROLE}]
+            return [{"content": "", "role": GOOGLE_GENAI_DEFAULT_MODEL_ROLE}]
         messages = []
         candidates = _get_attr(response, "candidates", [])
         for candidate in candidates:
@@ -136,7 +136,7 @@ class GoogleGenAIIntegration(BaseLLMIntegration):
             if not content:
                 continue
             parts = _get_attr(content, "parts", [])
-            role = _get_attr(content, "role", DEFAULT_MODEL_ROLE)
+            role = _get_attr(content, "role", GOOGLE_GENAI_DEFAULT_MODEL_ROLE)
             for part in parts:
                 message = extract_message_from_part_google_genai(part, role)
                 messages.append(message)

ddtrace/llmobs/_integrations/{google_genai_utils.py → google_utils.py}

@@ -1,6 +1,7 @@
 from typing import Any
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Tuple
 
 from ddtrace.llmobs._constants import BILLABLE_CHARACTER_COUNT_METRIC_KEY
@@ -11,9 +12,9 @@ from ddtrace.llmobs._constants import TOTAL_TOKENS_METRIC_KEY
 from ddtrace.llmobs._utils import _get_attr
 
 
-# google genai has roles "model" and "user", but in order to stay consistent with other integrations,
+# Google GenAI has roles "model" and "user", but in order to stay consistent with other integrations,
 # we use "assistant" as the default role for model messages
-DEFAULT_MODEL_ROLE = "assistant"
+GOOGLE_GENAI_DEFAULT_MODEL_ROLE = "assistant"
 
 # https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-partner-models
 # GeminiAPI: only exports google provided models
@@ -40,9 +41,31 @@ KNOWN_MODEL_PREFIX_TO_PROVIDER = {
 }
 
 
-def extract_provider_and_model_name(kwargs: Dict[str, Any]) -> Tuple[str, str]:
-    model_path = kwargs.get("model", "")
-    model_name = model_path.split("/")[-1]
+def extract_provider_and_model_name(
+    kwargs: Optional[Dict[str, Any]] = None, instance: Any = None, model_name_attr: Optional[str] = None
+) -> Tuple[str, str]:
+    """
+    Function to extract provider and model name from either kwargs or instance attributes.
+    Args:
+        kwargs: Dictionary containing model information (used for google_genai)
+        instance: Model instance with attributes (used for vertexai and google_generativeai)
+        model_name_attr: Attribute name to extract from instance (e.g., "_model_name", "model_name", used for vertexai
+                         and google_generativeai)
+
+    Returns:
+        Tuple of (provider_name, model_name)
+    """
+    model_path = ""
+    if kwargs is not None:
+        model_path = kwargs.get("model", "")
+    elif instance is not None and model_name_attr is not None:
+        model_path = _get_attr(instance, model_name_attr, "")
+
+    if not model_path or not isinstance(model_path, str):
+        return "custom", "custom"
+
+    model_name = model_path.split("/")[-1] if "/" in model_path else model_path
+
     for prefix in KNOWN_MODEL_PREFIX_TO_PROVIDER.keys():
         if model_name.lower().startswith(prefix):
             provider_name = KNOWN_MODEL_PREFIX_TO_PROVIDER[prefix]
@@ -50,7 +73,7 @@ def extract_provider_and_model_name(kwargs: Dict[str, Any]) -> Tuple[str, str]:
     return "custom", model_name if model_name else "custom"
 
 
-def normalize_contents(contents) -> List[Dict[str, Any]]:
+def normalize_contents_google_genai(contents) -> List[Dict[str, Any]]:
     """
     contents has a complex union type structure:
     - contents: Union[ContentListUnion, ContentListUnionDict]
@@ -142,7 +165,7 @@ def extract_message_from_part_google_genai(part, role: str) -> Dict[str, Any]:
     returns a dict representing a message with format {"role": role, "content": content}
     """
     if role == "model":
-        role = DEFAULT_MODEL_ROLE
+        role = GOOGLE_GENAI_DEFAULT_MODEL_ROLE
 
     message: Dict[str, Any] = {"role": role}
     if isinstance(part, str):
@@ -187,3 +210,76 @@ def extract_message_from_part_google_genai(part, role: str) -> Dict[str, Any]:
         return message
 
     return {"content": "Unsupported file type: {}".format(type(part)), "role": role}
+
+
+def llmobs_get_metadata_gemini_vertexai(kwargs, instance):
+    metadata = {}
+    model_config = getattr(instance, "_generation_config", {}) or {}
+    model_config = model_config.to_dict() if hasattr(model_config, "to_dict") else model_config
+    request_config = kwargs.get("generation_config", {}) or {}
+    request_config = request_config.to_dict() if hasattr(request_config, "to_dict") else request_config
+
+    parameters = ("temperature", "max_output_tokens", "candidate_count", "top_p", "top_k")
+    for param in parameters:
+        model_config_value = _get_attr(model_config, param, None)
+        request_config_value = _get_attr(request_config, param, None)
+        if model_config_value or request_config_value:
+            metadata[param] = request_config_value or model_config_value
+    return metadata
+
+
+def extract_message_from_part_gemini_vertexai(part, role=None):
+    text = _get_attr(part, "text", "")
+    function_call = _get_attr(part, "function_call", None)
+    function_response = _get_attr(part, "function_response", None)
+    message = {"content": text}
+    if role:
+        message["role"] = role
+    if function_call:
+        function_call_dict = function_call
+        if not isinstance(function_call, dict):
+            function_call_dict = type(function_call).to_dict(function_call)
+        message["tool_calls"] = [
+            {"name": function_call_dict.get("name", ""), "arguments": function_call_dict.get("args", {})}
+        ]
+    if function_response:
+        function_response_dict = function_response
+        if not isinstance(function_response, dict):
+            function_response_dict = type(function_response).to_dict(function_response)
+        message["content"] = "[tool result: {}]".format(function_response_dict.get("response", ""))
+    return message
+
+
+def get_system_instructions_gemini_vertexai(model_instance):
+    """
+    Extract system instructions from model and convert to []str for tagging.
+    """
+    try:
+        from google.ai.generativelanguage_v1beta.types.content import Content
+    except ImportError:
+        Content = None
+    try:
+        from vertexai.generative_models._generative_models import Part
+    except ImportError:
+        Part = None
+
+    raw_system_instructions = getattr(model_instance, "_system_instruction", [])
+    if Content is not None and isinstance(raw_system_instructions, Content):
+        system_instructions = []
+        for part in raw_system_instructions.parts:
+            system_instructions.append(_get_attr(part, "text", ""))
+        return system_instructions
+    elif isinstance(raw_system_instructions, str):
+        return [raw_system_instructions]
+    elif Part is not None and isinstance(raw_system_instructions, Part):
+        return [_get_attr(raw_system_instructions, "text", "")]
+    elif not isinstance(raw_system_instructions, list):
+        return []
+
+    system_instructions = []
+    for elem in raw_system_instructions:
+        if isinstance(elem, str):
+            system_instructions.append(elem)
+        elif Part is not None and isinstance(elem, Part):
+            system_instructions.append(_get_attr(elem, "text", ""))
+    return system_instructions

ddtrace/llmobs/_integrations/langchain.py

@@ -163,7 +163,6 @@ class LangChainIntegration(BaseLLMIntegration):
 
         self._set_links(span)
         model_provider = span.get_tag(PROVIDER)
-        self._llmobs_set_metadata(span, model_provider)
 
         is_workflow = False
 
@@ -365,26 +364,37 @@ class LangChainIntegration(BaseLLMIntegration):
         if hasattr(instance, "_datadog_spans"):
             delattr(instance, "_datadog_spans")
 
-    def _llmobs_set_metadata(self, span: Span, model_provider: Optional[str] = None) -> None:
-        if not model_provider:
+    def _llmobs_set_metadata(self, span: Span, kwargs: Dict[str, Any]) -> None:
+        identifying_params = kwargs.pop("_dd.identifying_params", None)
+        if not identifying_params:
             return
+        metadata = self._llmobs_extract_parameters(identifying_params)
+        for val in identifying_params.values():
+            if metadata:
+                break
+            if not isinstance(val, dict):
+                continue
+            metadata = self._llmobs_extract_parameters(val)
 
-        metadata = {}
-        temperature = span.get_tag(f"langchain.request.{model_provider}.parameters.temperature") or span.get_tag(
-            f"langchain.request.{model_provider}.parameters.model_kwargs.temperature"
-        )  # huggingface
-        max_tokens = (
-            span.get_tag(f"langchain.request.{model_provider}.parameters.max_tokens")
-            or span.get_tag(f"langchain.request.{model_provider}.parameters.maxTokens")  # ai21
-            or span.get_tag(f"langchain.request.{model_provider}.parameters.model_kwargs.max_tokens")  # huggingface
-        )
+        if metadata:
+            span._set_ctx_item(METADATA, metadata)
 
+    def _llmobs_extract_parameters(self, parameters: Dict[str, Any]) -> Dict[str, Any]:
+        metadata: Dict[str, Any] = {}
+        max_tokens = None
+        temperature = None
+        if "temperature" in parameters:
+            temperature = parameters["temperature"]
+        for max_token_key in ["max_tokens", "maxTokens", "max_completion_tokens"]:
+            if max_token_key in parameters:
+                max_tokens = parameters[max_token_key]
+                break
         if temperature is not None and temperature != "None":
             metadata["temperature"] = float(temperature)
         if max_tokens is not None and max_tokens != "None":
             metadata["max_tokens"] = int(max_tokens)
-        if metadata:
-            span._set_ctx_item(METADATA, metadata)
+
+        return metadata
 
     def _llmobs_set_tags_from_llm(
         self, span: Span, args: List[Any], kwargs: Dict[str, Any], completions: Any, is_workflow: bool = False
@@ -411,6 +421,8 @@ class LangChainIntegration(BaseLLMIntegration):
             }
         )
 
+        self._llmobs_set_metadata(span, kwargs)
+
         if span.error:
             span._set_ctx_item(output_tag_key, [{"content": ""}])
             return
@@ -444,6 +456,9 @@ class LangChainIntegration(BaseLLMIntegration):
                 MODEL_PROVIDER: span.get_tag(PROVIDER) or "",
             }
         )
+
+        self._llmobs_set_metadata(span, kwargs)
+
         input_tag_key = INPUT_VALUE if is_workflow else INPUT_MESSAGES
         output_tag_key = OUTPUT_VALUE if is_workflow else OUTPUT_MESSAGES
         stream = span.get_tag("langchain.request.stream")
@@ -700,16 +715,10 @@ class LangChainIntegration(BaseLLMIntegration):
         **kwargs,
     ) -> None:
         """Set base level tags that should be present on all LangChain spans (if they are not None)."""
-        span.set_tag_str(TYPE, interface_type)
         if provider is not None:
             span.set_tag_str(PROVIDER, provider)
         if model is not None:
             span.set_tag_str(MODEL, model)
-        if api_key is not None:
-            if len(api_key) >= 4:
-                span.set_tag_str(API_KEY, "...%s" % str(api_key[-4:]))
-            else:
-                span.set_tag_str(API_KEY, api_key)
 
     def check_token_usage_chat_or_llm_result(self, result):
         """Checks for token usage on the top-level ChatResult or LLMResult object"""