ddtrace 3.11.0rc1__cp311-cp311-macosx_12_0_arm64.whl → 3.11.0rc3__cp311-cp311-macosx_12_0_arm64.whl

ddtrace/internal/ci_visibility/encoder.py

@@ -1,7 +1,14 @@
+from __future__ import annotations
+
 import json
 import os
 import threading
 from typing import TYPE_CHECKING  # noqa:F401
+from typing import Any  # noqa:F401
+from typing import Dict  # noqa:F401
+from typing import List  # noqa:F401
+from typing import Optional  # noqa:F401
+from typing import Tuple  # noqa:F401
 from uuid import uuid4
 
 from ddtrace.ext import SpanTypes
@@ -28,11 +35,6 @@ from ddtrace.internal.writer.writer import NoEncodableSpansError
 log = get_logger(__name__)
 
 if TYPE_CHECKING:  # pragma: no cover
-    from typing import Any  # noqa:F401
-    from typing import Dict  # noqa:F401
-    from typing import List  # noqa:F401
-    from typing import Optional  # noqa:F401
-
     from ddtrace._trace.span import Span  # noqa:F401
 
 
@@ -42,74 +44,153 @@ class CIVisibilityEncoderV01(BufferedEncoder):
     TEST_SUITE_EVENT_VERSION = 1
     TEST_EVENT_VERSION = 2
     ENDPOINT_TYPE = ENDPOINT.TEST_CYCLE
+    _MAX_PAYLOAD_SIZE = 5 * 1024 * 1024  # 5MB
 
     def __init__(self, *args):
         # DEV: args are not used here, but are used by BufferedEncoder's __cinit__() method,
         #      which is called implicitly by Cython.
         super(CIVisibilityEncoderV01, self).__init__()
+        self._metadata: Dict[str, Dict[str, str]] = {}
         self._lock = threading.RLock()
-        self._metadata = {}
+        self._is_xdist_worker = os.getenv("PYTEST_XDIST_WORKER") is not None
         self._init_buffer()
 
     def __len__(self):
         with self._lock:
             return len(self.buffer)
 
-    def set_metadata(self, event_type, metadata):
-        # type: (str, Dict[str, str]) -> None
+    def set_metadata(self, event_type: str, metadata: Dict[str, str]):
         self._metadata.setdefault(event_type, {}).update(metadata)
 
     def _init_buffer(self):
         with self._lock:
             self.buffer = []
 
-    def put(self, spans):
+    def put(self, item):
         with self._lock:
-            self.buffer.append(spans)
+            self.buffer.append(item)
 
     def encode_traces(self, traces):
-        return self._build_payload(traces=traces)
+        """
+        Only used for LogWriter, not called for CI Visibility currently
+        """
+        raise NotImplementedError()
 
-    def encode(self):
+    def encode(self) -> List[Tuple[Optional[bytes], int]]:
         with self._lock:
+            if not self.buffer:
+                return []
+            payloads = []
             with StopWatch() as sw:
-                payload = self._build_payload(self.buffer)
+                payloads = self._build_payload(self.buffer)
             record_endpoint_payload_events_serialization_time(endpoint=self.ENDPOINT_TYPE, seconds=sw.elapsed())
-            buffer_size = len(self.buffer)
             self._init_buffer()
-            return payload, buffer_size
+            return payloads
 
-    def _get_parent_session(self, traces):
+    def _get_parent_session(self, traces: List[List[Span]]) -> int:
         for trace in traces:
             for span in trace:
                 if span.get_tag(EVENT_TYPE) == SESSION_TYPE and span.parent_id is not None and span.parent_id != 0:
                     return span.parent_id
         return 0
 
-    def _build_payload(self, traces):
+    def _build_payload(self, traces: List[List[Span]]) -> List[Tuple[Optional[bytes], int]]:
+        """
+        Build multiple payloads from traces, splitting when necessary to stay under size limits.
+        Uses index-based recursive approach to avoid copying slices.
+
+        Returns a list of (payload_bytes, trace_count) tuples, where each payload contains
+        as many traces as possible without exceeding _MAX_PAYLOAD_SIZE.
+        """
+        if not traces:
+            return []
+
         new_parent_session_span_id = self._get_parent_session(traces)
-        is_not_xdist_worker = os.getenv("PYTEST_XDIST_WORKER") is None
-        normalized_spans = [
-            self._convert_span(span, trace[0].context.dd_origin, new_parent_session_span_id)
-            for trace in traces
-            for span in trace
-            if (is_not_xdist_worker or span.get_tag(EVENT_TYPE) != SESSION_TYPE)
-        ]
-        if not normalized_spans:
-            return None
-        record_endpoint_payload_events_count(endpoint=ENDPOINT.TEST_CYCLE, count=len(normalized_spans))
+        return self._build_payloads_recursive(traces, 0, len(traces), new_parent_session_span_id)
 
-        # TODO: Split the events in several payloads as needed to avoid hitting the intake's maximum payload size.
+    def _build_payloads_recursive(
+        self, traces: List[List[Span]], start_idx: int, end_idx: int, new_parent_session_span_id: int
+    ) -> List[Tuple[Optional[bytes], int]]:
+        """
+        Recursively build payloads using start/end indexes to avoid slice copying.
+
+        Args:
+            traces: Full list of traces
+            start_idx: Start index (inclusive)
+            end_idx: End index (exclusive)
+            new_parent_session_span_id: Parent session span ID
+
+        Returns:
+            List of (payload_bytes, trace_count) tuples
+        """
+        if start_idx >= end_idx:
+            return []
+
+        trace_count = end_idx - start_idx
+
+        # Convert traces to spans with filtering (using indexes)
+        all_spans_with_trace_info = self._convert_traces_to_spans_indexed(
+            traces, start_idx, end_idx, new_parent_session_span_id
+        )
+
+        # Get all spans (flattened)
+        all_spans = [span for _, trace_spans in all_spans_with_trace_info for span in trace_spans]
+
+        if not all_spans:
+            log.debug("No spans to encode after filtering, skipping chunk")
+            return []
+
+        # Try to create payload from all spans
+        payload = self._create_payload_from_spans(all_spans)
+
+        if len(payload) <= self._MAX_PAYLOAD_SIZE or trace_count == 1:
+            # Payload fits or we can't split further (single trace)
+            record_endpoint_payload_events_count(endpoint=self.ENDPOINT_TYPE, count=len(all_spans))
+            return [(payload, trace_count)]
+        else:
+            # Payload is too large, split in half recursively
+            mid_idx = start_idx + (trace_count + 1) // 2
+
+            # Process both halves recursively
+            left_payloads = self._build_payloads_recursive(traces, start_idx, mid_idx, new_parent_session_span_id)
+            right_payloads = self._build_payloads_recursive(traces, mid_idx, end_idx, new_parent_session_span_id)
+
+            # Combine results
+            return left_payloads + right_payloads
+
+    def _convert_traces_to_spans_indexed(
+        self, traces: List[List[Span]], start_idx: int, end_idx: int, new_parent_session_span_id: int
+    ) -> List[Tuple[int, List[Dict[str, Any]]]]:
+        """Convert traces to spans with xdist filtering applied, using indexes to avoid slicing."""
+        all_spans_with_trace_info = []
+        for trace_idx in range(start_idx, end_idx):
+            trace = traces[trace_idx]
+            trace_spans = [
+                self._convert_span(span, trace[0].context.dd_origin, new_parent_session_span_id)
+                for span in trace
+                if (not self._is_xdist_worker) or (span.get_tag(EVENT_TYPE) != SESSION_TYPE)
+            ]
+            all_spans_with_trace_info.append((trace_idx, trace_spans))
+
+        return all_spans_with_trace_info
+
+    def _create_payload_from_spans(self, spans: List[Dict[str, Any]]) -> bytes:
+        """Create a payload from the given spans."""
         return CIVisibilityEncoderV01._pack_payload(
-            {"version": self.PAYLOAD_FORMAT_VERSION, "metadata": self._metadata, "events": normalized_spans}
+            {
+                "version": self.PAYLOAD_FORMAT_VERSION,
+                "metadata": self._metadata,
+                "events": spans,
+            }
         )
 
     @staticmethod
     def _pack_payload(payload):
         return msgpack_packb(payload)
 
-    def _convert_span(self, span, dd_origin, new_parent_session_span_id=0):
-        # type: (Span, str, Optional[int]) -> Dict[str, Any]
+    def _convert_span(
+        self, span: Span, dd_origin: Optional[str] = None, new_parent_session_span_id: int = 0
+    ) -> Dict[str, Any]:
         sp = JSONEncoderV2._span_to_dict(span)
         sp = JSONEncoderV2._normalize_span(sp)
         sp["type"] = span.get_tag(EVENT_TYPE) or span.span_type
@@ -177,18 +258,17 @@ class CIVisibilityCoverageEncoderV02(CIVisibilityEncoderV01):
     def _set_itr_suite_skipping_mode(self, new_value):
         self.itr_suite_skipping_mode = new_value
 
-    def put(self, spans):
+    def put(self, item):
         spans_with_coverage = [
             span
-            for span in spans
+            for span in item
             if COVERAGE_TAG_NAME in span.get_tags() or span.get_struct_tag(COVERAGE_TAG_NAME) is not None
         ]
         if not spans_with_coverage:
             raise NoEncodableSpansError()
         return super(CIVisibilityCoverageEncoderV02, self).put(spans_with_coverage)
 
-    def _build_coverage_attachment(self, data):
-        # type: (bytes) -> List[bytes]
+    def _build_coverage_attachment(self, data: bytes) -> List[bytes]:
         return [
             b"--%s" % self.boundary.encode("utf-8"),
             b'Content-Disposition: form-data; name="coverage1"; filename="coverage1.msgpack"',
@@ -197,8 +277,7 @@ class CIVisibilityCoverageEncoderV02(CIVisibilityEncoderV01):
             data,
         ]
 
-    def _build_event_json_attachment(self):
-        # type: () -> List[bytes]
+    def _build_event_json_attachment(self) -> List[bytes]:
         return [
             b"--%s" % self.boundary.encode("utf-8"),
             b'Content-Disposition: form-data; name="event"; filename="event.json"',
@@ -207,18 +286,16 @@ class CIVisibilityCoverageEncoderV02(CIVisibilityEncoderV01):
             b'{"dummy":true}',
         ]
 
-    def _build_body(self, data):
-        # type: (bytes) -> List[bytes]
+    def _build_body(self, data: bytes) -> List[bytes]:
         return (
             self._build_coverage_attachment(data)
             + self._build_event_json_attachment()
             + [b"--%s--" % self.boundary.encode("utf-8")]
         )
 
-    def _build_data(self, traces):
-        # type: (List[List[Span]]) -> Optional[bytes]
+    def _build_data(self, traces: List[List[Span]]) -> Optional[bytes]:
         normalized_covs = [
-            self._convert_span(span, "")
+            self._convert_span(span)
             for trace in traces
             for span in trace
             if (COVERAGE_TAG_NAME in span.get_tags() or span.get_struct_tag(COVERAGE_TAG_NAME) is not None)
@@ -229,17 +306,17 @@ class CIVisibilityCoverageEncoderV02(CIVisibilityEncoderV01):
         # TODO: Split the events in several payloads as needed to avoid hitting the intake's maximum payload size.
         return msgpack_packb({"version": self.PAYLOAD_FORMAT_VERSION, "coverages": normalized_covs})
 
-    def _build_payload(self, traces):
-        # type: (List[List[Span]]) -> Optional[bytes]
+    def _build_payload(self, traces: List[List[Span]]) -> List[Tuple[Optional[bytes], int]]:
         data = self._build_data(traces)
         if not data:
-            return None
-        return b"\r\n".join(self._build_body(data))
+            return []
+        return [(b"\r\n".join(self._build_body(data)), len(data))]
 
-    def _convert_span(self, span, dd_origin, new_parent_session_span_id=0):
-        # type: (Span, str, Optional[int]) -> Dict[str, Any]
+    def _convert_span(
+        self, span: Span, dd_origin: Optional[str] = None, new_parent_session_span_id: int = 0
+    ) -> Dict[str, Any]:
         # DEV: new_parent_session_span_id is unused here, but it is used in super class
-        files: Dict[str, Any] = {}
+        files: dict[str, Any] = {}
 
         files_struct_tag_value = span.get_struct_tag(COVERAGE_TAG_NAME)
         if files_struct_tag_value is not None and "files" in files_struct_tag_value:

ddtrace/internal/ci_visibility/utils.py

@@ -112,11 +112,10 @@ def take_over_logger_stream_handler(remove_ddtrace_stream_handlers=True):
         log.debug("CIVisibility not taking over ddtrace logger because level is set to: %s", level)
         return
 
-    root_logger = logging.getLogger()
+    ddtrace_logger = logging.getLogger("ddtrace")
 
     if remove_ddtrace_stream_handlers:
         log.debug("CIVisibility removing DDTrace logger handler")
-        ddtrace_logger = logging.getLogger("ddtrace")
         for handler in list(ddtrace_logger.handlers):
             ddtrace_logger.removeHandler(handler)
     else:
@@ -136,8 +135,9 @@ def take_over_logger_stream_handler(remove_ddtrace_stream_handlers=True):
         log.warning("Invalid log level: %s", level)
         return
 
-    root_logger.addHandler(ci_visibility_handler)
-    root_logger.setLevel(min(root_logger.level, ci_visibility_handler.level))
+    ddtrace_logger.addHandler(ci_visibility_handler)
+    ddtrace_logger.setLevel(min(ddtrace_logger.level, ci_visibility_handler.level))
+    ddtrace_logger.propagate = False
 
     log.debug("logger setup complete")
 

ddtrace/internal/core/__init__.py

@@ -197,9 +197,12 @@ class ExecutionContext(object):
         self._parent = value
 
     def __exit__(
-        self, exc_type: Optional[type], exc_value: Optional[BaseException], traceback: Optional[types.TracebackType]
+        self,
+        exc_type: Optional[type],
+        exc_value: Optional[BaseException],
+        traceback: Optional[types.TracebackType],
     ) -> bool:
-        dispatch("context.ended.%s" % self.identifier, (self,))
+        dispatch("context.ended.%s" % self.identifier, (self, (exc_type, exc_value, traceback)))
         if self._span is None:
             try:
                 if self._token is not None:

ddtrace/internal/endpoints.py

@@ -0,0 +1,76 @@
+import dataclasses
+from time import monotonic
+from typing import List
+
+
+@dataclasses.dataclass(frozen=True)
+class HttpEndPoint:
+    method: str
+    path: str
+    resource_name: str = dataclasses.field(default="")
+    operation_name: str = dataclasses.field(default="http.request")
+
+    def __post_init__(self) -> None:
+        super().__setattr__("method", self.method.upper())
+        if not self.resource_name:
+            super().__setattr__("resource_name", f"{self.method} {self.path}")
+
+
+@dataclasses.dataclass()
+class HttpEndPointsCollection:
+    """A collection of HTTP endpoints that can be modified and flushed to a telemetry payload.
+
+    The collection collects HTTP endpoints at startup and can be flushed to a telemetry payload.
+    It maintains a maximum size and drops endpoints after a certain time period in case of a hot reload of the server.
+    """
+
+    endpoints: List[HttpEndPoint] = dataclasses.field(default_factory=list, init=False)
+    is_first: bool = dataclasses.field(default=True, init=False)
+    drop_time_seconds: float = dataclasses.field(default=90.0, init=False)
+    last_modification_time: float = dataclasses.field(default_factory=monotonic, init=False)
+    max_size_length: int = dataclasses.field(default=900, init=False)
+
+    def reset(self) -> None:
+        """Reset the collection to its initial state."""
+        self.endpoints.clear()
+        self.is_first = True
+        self.last_modification_time = monotonic()
+
+    def add_endpoint(
+        self, method: str, path: str, resource_name: str = "", operation_name: str = "http.request"
+    ) -> None:
+        """
+        Add an endpoint to the collection.
+        """
+        current_time = monotonic()
+        if current_time - self.last_modification_time > self.drop_time_seconds:
+            self.reset()
+            self.endpoints.append(
+                HttpEndPoint(method=method, path=path, resource_name=resource_name, operation_name=operation_name)
+            )
+        elif len(self.endpoints) < self.max_size_length:
+            self.last_modification_time = current_time
+            self.endpoints.append(
+                HttpEndPoint(method=method, path=path, resource_name=resource_name, operation_name=operation_name)
+            )
+
+    def flush(self, max_length: int) -> dict:
+        """
+        Flush the endpoints to a payload, returning the first `max` endpoints.
+        """
+        if max_length >= len(self.endpoints):
+            res = {
+                "is_first": self.is_first,
+                "endpoints": [dataclasses.asdict(ep) for ep in self.endpoints],
+            }
+            self.reset()
+            return res
+        else:
+            res = {
+                "is_first": self.is_first,
+                "endpoints": [dataclasses.asdict(ep) for ep in self.endpoints[:max_length]],
+            }
+            self.endpoints = self.endpoints[max_length:]
+            self.is_first = False
+            self.last_modification_time = monotonic()
+            return res

ddtrace/internal/schema/processor.py

@@ -1,5 +1,6 @@
 from ddtrace._trace.processor import TraceProcessor
 from ddtrace.constants import _BASE_SERVICE_KEY
+from ddtrace.internal.serverless import in_aws_lambda
 from ddtrace.settings._config import config
 
 from . import schematize_service_name
@@ -8,10 +9,13 @@ from . import schematize_service_name
 class BaseServiceProcessor(TraceProcessor):
     def __init__(self):
         self._global_service = schematize_service_name((config.service or "").lower())
+        self._in_aws_lambda = in_aws_lambda()
 
     def process_trace(self, trace):
-        if not trace:
-            return
+        # AWS Lambda spans receive unhelpful base_service value of runtime
+        # Remove base_service to prevent service overrides in Lambda spans
+        if not trace or self._in_aws_lambda:
+            return trace
 
         traces_to_process = filter(
             lambda x: x.service and x.service.lower() != self._global_service,

ddtrace/internal/telemetry/writer.py

@@ -418,6 +418,23 @@ class TelemetryWriter(PeriodicService):
             payload = {"dependencies": packages}
             self.add_event(payload, "app-dependencies-loaded")
 
+    def _add_endpoints_event(self):
+        """Adds a Telemetry event which sends the list of HTTP endpoints found at startup to the agent"""
+        import ddtrace.settings.asm as asm_config_module
+
+        if not asm_config_module.config._api_security_endpoint_collection or not self._enabled:
+            return
+
+        if not asm_config_module.endpoint_collection.endpoints:
+            return
+
+        with self._service_lock:
+            payload = asm_config_module.endpoint_collection.flush(
+                asm_config_module.config._api_security_endpoint_collection_limit
+            )
+
+        self.add_event(payload, "app-endpoints")
+
     def _app_product_change(self):
         # type: () -> None
         """Adds a Telemetry event which reports the enablement of an APM product"""
@@ -660,6 +677,7 @@ class TelemetryWriter(PeriodicService):
             self._app_client_configuration_changed_event(configurations)
 
         self._app_dependencies_loaded_event()
+        self._add_endpoints_event()
 
         if shutting_down:
             self._app_closing_event()

ddtrace/internal/test_visibility/coverage_lines.py

@@ -41,8 +41,8 @@ class CoverageLines:
     def add(self, line_number: int):
         lines_byte = line_number // 8
 
-        if lines_byte >= len(self._lines):
-            self._lines.extend(bytearray(lines_byte - len(self._lines) + 1))
+        if lines_byte >= self._lines.__len__():
+            self._lines.extend(bytearray(lines_byte - self._lines.__len__() + 1))
 
         # DEV this fun bit allows us to trick ourselves into little-endianness, which is what the backend wants to see
         # in bytes
@@ -62,8 +62,8 @@ class CoverageLines:
 
     def update(self, other: "CoverageLines"):
         # Extend our lines if the other coverage has more lines
-        if len(other._lines) > len(self._lines):
-            self._lines.extend(bytearray(len(other._lines) - len(self._lines)))
+        if other._lines.__len__() > self._lines.__len__():
+            self._lines.extend(bytearray(other._lines.__len__() - self._lines.__len__()))
 
         for _byte_idx, _byte in enumerate(other._lines):
             self._lines[_byte_idx] |= _byte

ddtrace/internal/writer/writer.py

@@ -385,13 +385,28 @@ class HTTPWriter(periodic.PeriodicService, TraceWriter):
     def _flush_queue_with_client(self, client: WriterClientBase, raise_exc: bool = False) -> None:
         n_traces = len(client.encoder)
         try:
-            encoded, n_traces = client.encoder.encode()
-
-            if encoded is None:
+            if not (encoded_traces := client.encoder.encode()):
                 return
 
-            # Should gzip the payload if intake accepts it
-            if self._intake_accepts_gzip:
+        except Exception:
+            # FIXME(munir): if client.encoder raises an Exception n_traces may not be accurate due to race conditions
+            log.error("failed to encode trace with encoder %r", client.encoder, exc_info=True)
+            self._metrics_dist("encoder.dropped.traces", n_traces)
+            return
+
+        for payload in encoded_traces:
+            encoded_data, n_traces = payload
+            self._flush_single_payload(encoded_data, n_traces, client=client, raise_exc=raise_exc)
+
+    def _flush_single_payload(
+        self, encoded: Optional[bytes], n_traces: int, client: WriterClientBase, raise_exc: bool = False
+    ) -> None:
+        if encoded is None:
+            return
+
+        # Should gzip the payload if intake accepts it
+        if self._intake_accepts_gzip:
+            try:
                 original_size = len(encoded)
                 # Replace the value to send with the gzipped the value
                 encoded = gzip.compress(encoded, compresslevel=6)
@@ -399,12 +414,10 @@ class HTTPWriter(periodic.PeriodicService, TraceWriter):
 
                 # And add the header
                 self._headers["Content-Encoding"] = "gzip"
-
-        except Exception:
-            # FIXME(munir): if client.encoder raises an Exception n_traces may not be accurate due to race conditions
-            log.error("failed to encode trace with encoder %r", client.encoder, exc_info=True)
-            self._metrics_dist("encoder.dropped.traces", n_traces)
-            return
+            except Exception:
+                log.error("failed to compress traces with encoder %r", client.encoder, exc_info=True)
+                self._metrics_dist("encoder.dropped.traces", n_traces)
+                return
 
         try:
             self._send_payload_with_backoff(encoded, n_traces, client)

ddtrace/llmobs/_constants.py

@@ -9,6 +9,7 @@ PARENT_ID_KEY = "_ml_obs.llmobs_parent_id"
 PROPAGATED_LLMOBS_TRACE_ID_KEY = "_dd.p.llmobs_trace_id"
 LLMOBS_TRACE_ID = "_ml_obs.llmobs_trace_id"
 TAGS = "_ml_obs.tags"
+AGENT_MANIFEST = "_ml_obs.meta.agent_manifest"
 
 MODEL_NAME = "_ml_obs.meta.model_name"
 MODEL_PROVIDER = "_ml_obs.meta.model_provider"
@@ -56,6 +57,7 @@ AGENTLESS_EXP_BASE_URL = "https://{}".format(EXP_SUBDOMAIN_NAME)
 EVP_PAYLOAD_SIZE_LIMIT = 5 << 20  # 5MB (actual limit is 5.1MB)
 EVP_EVENT_SIZE_LIMIT = (1 << 20) - 1024  # 999KB (actual limit is 1MB)
 
+EXPERIMENT_CSV_FIELD_MAX_SIZE = 10 * 1024 * 1024
 
 DROPPED_IO_COLLECTION_ERROR = "dropped_io"
 DROPPED_VALUE_TEXT = "[This value has been dropped because this span's size exceeds the 1MB size limit.]"
@@ -97,3 +99,4 @@ PROXY_REQUEST = "llmobs.proxy_request"
 
 EXPERIMENT_ID_KEY = "_ml_obs.experiment_id"
 EXPERIMENT_EXPECTED_OUTPUT = "_ml_obs.meta.input.expected_output"
+DEFAULT_PROJECT_NAME = "default-project"