ddapm-test-agent 1.31.1__py3-none-any.whl → 1.33.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddapm_test_agent/agent.py +239 -2
- ddapm_test_agent/client.py +63 -20
- ddapm_test_agent/logs.py +67 -0
- ddapm_test_agent/metrics.py +94 -0
- ddapm_test_agent/trace.py +399 -0
- ddapm_test_agent/vcr_proxy.py +80 -23
- {ddapm_test_agent-1.31.1.dist-info → ddapm_test_agent-1.33.0.dist-info}/METADATA +63 -2
- {ddapm_test_agent-1.31.1.dist-info → ddapm_test_agent-1.33.0.dist-info}/RECORD +13 -11
- {ddapm_test_agent-1.31.1.dist-info → ddapm_test_agent-1.33.0.dist-info}/WHEEL +0 -0
- {ddapm_test_agent-1.31.1.dist-info → ddapm_test_agent-1.33.0.dist-info}/entry_points.txt +0 -0
- {ddapm_test_agent-1.31.1.dist-info → ddapm_test_agent-1.33.0.dist-info}/licenses/LICENSE.BSD3 +0 -0
- {ddapm_test_agent-1.31.1.dist-info → ddapm_test_agent-1.33.0.dist-info}/licenses/LICENSE.apache2 +0 -0
- {ddapm_test_agent-1.31.1.dist-info → ddapm_test_agent-1.33.0.dist-info}/top_level.txt +0 -0
ddapm_test_agent/trace.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Tracing specific functions and types"""
|
|
2
2
|
|
|
3
|
+
from enum import IntEnum
|
|
3
4
|
import json
|
|
4
5
|
from typing import Any
|
|
5
6
|
from typing import Callable
|
|
@@ -114,6 +115,59 @@ v04TracePayload = List[List[Span]]
|
|
|
114
115
|
TraceMap = OrderedDict[int, Trace]
|
|
115
116
|
|
|
116
117
|
|
|
118
|
+
class V1ChunkKeys(IntEnum):
|
|
119
|
+
PRIORITY = 1
|
|
120
|
+
ORIGIN = 2
|
|
121
|
+
ATTRIBUTES = 3
|
|
122
|
+
SPANS = 4
|
|
123
|
+
DROPPED_TRACE = 5
|
|
124
|
+
TRACE_ID = 6
|
|
125
|
+
SAMPLING_MECHANISM = 7
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class V1SpanKeys(IntEnum):
|
|
129
|
+
SERVICE = 1
|
|
130
|
+
NAME = 2
|
|
131
|
+
RESOURCE = 3
|
|
132
|
+
SPAN_ID = 4
|
|
133
|
+
PARENT_ID = 5
|
|
134
|
+
START = 6
|
|
135
|
+
DURATION = 7
|
|
136
|
+
ERROR = 8
|
|
137
|
+
ATTRIBUTES = 9
|
|
138
|
+
TYPE = 10
|
|
139
|
+
SPAN_LINKS = 11
|
|
140
|
+
SPAN_EVENTS = 12
|
|
141
|
+
ENV = 13
|
|
142
|
+
VERSION = 14
|
|
143
|
+
COMPONENT = 15
|
|
144
|
+
SPAN_KIND = 16
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class V1SpanLinkKeys(IntEnum):
|
|
148
|
+
TRACE_ID = 1
|
|
149
|
+
SPAN_ID = 2
|
|
150
|
+
ATTRIBUTES = 3
|
|
151
|
+
TRACE_STATE = 4
|
|
152
|
+
FLAGS = 5
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class V1SpanEventKeys(IntEnum):
|
|
156
|
+
TIME = 1
|
|
157
|
+
NAME = 2
|
|
158
|
+
ATTRIBUTES = 3
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class V1AnyValueKeys(IntEnum):
|
|
162
|
+
STRING = 1
|
|
163
|
+
BOOL = 2
|
|
164
|
+
DOUBLE = 3
|
|
165
|
+
INT = 4
|
|
166
|
+
BYTES = 5
|
|
167
|
+
ARRAY = 6
|
|
168
|
+
KEY_VALUE_LIST = 7
|
|
169
|
+
|
|
170
|
+
|
|
117
171
|
# TODO:ban add extra tags to add to the span
|
|
118
172
|
# TODO:ban warn about dropping metastruct
|
|
119
173
|
def verify_span(d: Any) -> Span:
|
|
@@ -702,6 +756,351 @@ def decode_v07(data: bytes) -> v04TracePayload:
|
|
|
702
756
|
return _verify_v07_payload(payload)
|
|
703
757
|
|
|
704
758
|
|
|
759
|
+
def decode_v1(data: bytes) -> v04TracePayload:
|
|
760
|
+
"""Decode a v1 trace payload.
|
|
761
|
+
The v1 format is similar to the v07 format but in an optimized format and with a few changes:
|
|
762
|
+
- Strings are deduplicated and sent in a "Streaming" format where strings are referred to by their index in a string table
|
|
763
|
+
- Trace IDs are sent as 128 bit integers in a bytes array
|
|
764
|
+
- 'meta' and 'metrics' are now sent as typed 'attributes', more similar to how OTLP traces are sent
|
|
765
|
+
"""
|
|
766
|
+
payload = msgpack.unpackb(data, strict_map_key=False)
|
|
767
|
+
return _convert_v1_payload(payload)
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def _get_and_add_string(string_table: List[str], value: Union[int, str]) -> str:
|
|
771
|
+
if isinstance(value, str):
|
|
772
|
+
string_table.append(value)
|
|
773
|
+
return value
|
|
774
|
+
elif isinstance(value, int):
|
|
775
|
+
if value >= len(string_table) or value < 0:
|
|
776
|
+
raise ValueError(f"Value {value} is out of range for string table of length {len(string_table)}")
|
|
777
|
+
return string_table[value]
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def _convert_v1_payload(data: Any) -> v04TracePayload:
|
|
781
|
+
if not isinstance(data, dict):
|
|
782
|
+
raise TypeError("Trace payload must be a map, got type %r." % type(data))
|
|
783
|
+
|
|
784
|
+
string_table: List[str] = [""] # 0 is reserved for empty string
|
|
785
|
+
|
|
786
|
+
v04Payload: List[List[Span]] = []
|
|
787
|
+
|
|
788
|
+
for k, v in data.items():
|
|
789
|
+
if k == 1:
|
|
790
|
+
raise TypeError("Message pack representation of v1 trace payload must stream strings")
|
|
791
|
+
elif k > 1 and k < 10: # All keys from 2-9 are strings, for now we can just build the string table
|
|
792
|
+
# TODO: In the future we can assert on these keys
|
|
793
|
+
if isinstance(v, str):
|
|
794
|
+
string_table.append(v)
|
|
795
|
+
elif k == 11:
|
|
796
|
+
if not isinstance(v, list):
|
|
797
|
+
raise TypeError("Trace payload 'chunks' (11) must be a list.")
|
|
798
|
+
for chunk in v:
|
|
799
|
+
v04Payload.append(_convert_v1_chunk(chunk, string_table))
|
|
800
|
+
else:
|
|
801
|
+
raise TypeError("Unknown key %r in v1 trace payload" % k)
|
|
802
|
+
return cast(v04TracePayload, v04Payload)
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _convert_v1_chunk(chunk: Any, string_table: List[str]) -> List[Span]:
|
|
806
|
+
if not isinstance(chunk, dict):
|
|
807
|
+
raise TypeError("Chunk must be a map.")
|
|
808
|
+
|
|
809
|
+
priority, origin, sampling_mechanism = "", "", None
|
|
810
|
+
trace_id, trace_id_high = 0, 0
|
|
811
|
+
meta: Dict[str, str] = {}
|
|
812
|
+
metrics: Dict[str, MetricType] = {}
|
|
813
|
+
spans: List[Span] = []
|
|
814
|
+
for k, v in chunk.items():
|
|
815
|
+
if k == V1ChunkKeys.PRIORITY:
|
|
816
|
+
priority = v
|
|
817
|
+
elif k == V1ChunkKeys.ORIGIN:
|
|
818
|
+
origin = _get_and_add_string(string_table, v)
|
|
819
|
+
elif k == V1ChunkKeys.ATTRIBUTES:
|
|
820
|
+
if not isinstance(v, list):
|
|
821
|
+
raise TypeError("Chunk Attributes must be a list, got type %r." % type(v))
|
|
822
|
+
_convert_v1_attributes(v, meta, metrics, string_table)
|
|
823
|
+
elif k == V1ChunkKeys.SPANS:
|
|
824
|
+
if not isinstance(v, list):
|
|
825
|
+
raise TypeError("Chunk 'spans'(4) must be a list.")
|
|
826
|
+
for span in v:
|
|
827
|
+
converted_span = _convert_v1_span(span, string_table)
|
|
828
|
+
spans.append(converted_span)
|
|
829
|
+
elif k == V1ChunkKeys.DROPPED_TRACE:
|
|
830
|
+
raise TypeError("Tracers must not set the droppedTrace(5) flag.")
|
|
831
|
+
elif k == V1ChunkKeys.TRACE_ID:
|
|
832
|
+
if len(v) != 16:
|
|
833
|
+
raise TypeError("Trace ID must be 16 bytes, got %r." % len(v))
|
|
834
|
+
# trace_id is a 128 bit integer in a bytes array, so we need to get the last 64 bits
|
|
835
|
+
trace_id = int.from_bytes(v[8:], "big")
|
|
836
|
+
trace_id_high = int.from_bytes(v[:8], "big")
|
|
837
|
+
elif k == V1ChunkKeys.SAMPLING_MECHANISM:
|
|
838
|
+
sampling_mechanism = v
|
|
839
|
+
else:
|
|
840
|
+
raise TypeError("Unknown key %r in v1 trace chunk" % k)
|
|
841
|
+
|
|
842
|
+
for span in spans:
|
|
843
|
+
if "metrics" not in span:
|
|
844
|
+
span["metrics"] = {}
|
|
845
|
+
if "meta" not in span:
|
|
846
|
+
span["meta"] = {}
|
|
847
|
+
span["trace_id"] = trace_id
|
|
848
|
+
span["meta"]["_dd.p.tid"] = hex(trace_id_high)
|
|
849
|
+
if sampling_mechanism is not None:
|
|
850
|
+
span["meta"]["_dd.p.dm"] = "-" + str(sampling_mechanism)
|
|
851
|
+
if origin != "":
|
|
852
|
+
span["meta"]["_dd.origin"] = origin
|
|
853
|
+
if priority != "":
|
|
854
|
+
span["metrics"]["_sampling_priority_v1"] = priority
|
|
855
|
+
for k, v in meta.items():
|
|
856
|
+
span["meta"][k] = v
|
|
857
|
+
for k, v in metrics.items():
|
|
858
|
+
span["metrics"][k] = v
|
|
859
|
+
return spans
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def _convert_v1_span(span: Any, string_table: List[str]) -> Span:
|
|
863
|
+
if not isinstance(span, dict):
|
|
864
|
+
raise TypeError("Span must be a map.")
|
|
865
|
+
|
|
866
|
+
# Create a regular dict first, then cast to TypedDict
|
|
867
|
+
v4Span: Dict[str, Any] = {}
|
|
868
|
+
env, version, component, spanKind = "", "", "", ""
|
|
869
|
+
|
|
870
|
+
for k, v in span.items():
|
|
871
|
+
if k == V1SpanKeys.SERVICE:
|
|
872
|
+
v4Span["service"] = _get_and_add_string(string_table, v)
|
|
873
|
+
elif k == V1SpanKeys.NAME:
|
|
874
|
+
v4Span["name"] = _get_and_add_string(string_table, v)
|
|
875
|
+
elif k == V1SpanKeys.RESOURCE:
|
|
876
|
+
v4Span["resource"] = _get_and_add_string(string_table, v)
|
|
877
|
+
elif k == V1SpanKeys.SPAN_ID:
|
|
878
|
+
v4Span["span_id"] = v
|
|
879
|
+
elif k == V1SpanKeys.PARENT_ID:
|
|
880
|
+
v4Span["parent_id"] = v
|
|
881
|
+
elif k == V1SpanKeys.START:
|
|
882
|
+
v4Span["start"] = v
|
|
883
|
+
elif k == V1SpanKeys.DURATION:
|
|
884
|
+
v4Span["duration"] = v
|
|
885
|
+
elif k == V1SpanKeys.ERROR:
|
|
886
|
+
if not isinstance(v, bool):
|
|
887
|
+
raise TypeError("Error must be a boolean, got type %r." % type(v))
|
|
888
|
+
v4Span["error"] = 1 if v else 0
|
|
889
|
+
elif k == V1SpanKeys.ATTRIBUTES:
|
|
890
|
+
if not isinstance(v, list):
|
|
891
|
+
raise TypeError("Attributes must be a list, got type %r." % type(v))
|
|
892
|
+
meta: Dict[str, str] = {}
|
|
893
|
+
metrics: Dict[str, MetricType] = {}
|
|
894
|
+
_convert_v1_attributes(v, meta, metrics, string_table)
|
|
895
|
+
v4Span["meta"] = meta
|
|
896
|
+
v4Span["metrics"] = metrics
|
|
897
|
+
elif k == V1SpanKeys.TYPE:
|
|
898
|
+
v4Span["type"] = _get_and_add_string(string_table, v)
|
|
899
|
+
elif k == V1SpanKeys.SPAN_LINKS:
|
|
900
|
+
if not isinstance(v, list):
|
|
901
|
+
raise TypeError("Span links must be a list, got type %r." % type(v))
|
|
902
|
+
links: List[SpanLink] = []
|
|
903
|
+
for raw_link in v:
|
|
904
|
+
link = _convert_v1_span_link(raw_link, string_table)
|
|
905
|
+
links.append(link)
|
|
906
|
+
v4Span["span_links"] = links
|
|
907
|
+
elif k == V1SpanKeys.SPAN_EVENTS:
|
|
908
|
+
if not isinstance(v, list):
|
|
909
|
+
raise TypeError("Span events must be a list, got type %r." % type(v))
|
|
910
|
+
events: List[SpanEvent] = []
|
|
911
|
+
for raw_event in v:
|
|
912
|
+
event = _convert_v1_span_event(raw_event, string_table)
|
|
913
|
+
events.append(event)
|
|
914
|
+
v4Span["span_events"] = events
|
|
915
|
+
elif k == V1SpanKeys.ENV:
|
|
916
|
+
env = _get_and_add_string(string_table, v)
|
|
917
|
+
elif k == V1SpanKeys.VERSION:
|
|
918
|
+
version = _get_and_add_string(string_table, v)
|
|
919
|
+
elif k == V1SpanKeys.COMPONENT:
|
|
920
|
+
component = _get_and_add_string(string_table, v)
|
|
921
|
+
elif k == V1SpanKeys.SPAN_KIND:
|
|
922
|
+
if not isinstance(v, int):
|
|
923
|
+
raise TypeError("Span kind must be an integer, got type %r." % type(v))
|
|
924
|
+
if v == 1:
|
|
925
|
+
spanKind = "internal"
|
|
926
|
+
elif v == 2:
|
|
927
|
+
spanKind = "server"
|
|
928
|
+
elif v == 3:
|
|
929
|
+
spanKind = "client"
|
|
930
|
+
elif v == 4:
|
|
931
|
+
spanKind = "producer"
|
|
932
|
+
elif v == 5:
|
|
933
|
+
spanKind = "consumer"
|
|
934
|
+
else:
|
|
935
|
+
raise TypeError("Unknown span kind %r." % v)
|
|
936
|
+
|
|
937
|
+
if "meta" not in v4Span or v4Span["meta"] is None:
|
|
938
|
+
v4Span["meta"] = {}
|
|
939
|
+
if env != "":
|
|
940
|
+
v4Span["meta"]["env"] = env
|
|
941
|
+
if version != "":
|
|
942
|
+
v4Span["meta"]["version"] = version
|
|
943
|
+
if component != "":
|
|
944
|
+
v4Span["meta"]["component"] = component
|
|
945
|
+
if spanKind != "":
|
|
946
|
+
v4Span["meta"]["span.kind"] = spanKind
|
|
947
|
+
|
|
948
|
+
# Cast to TypedDict
|
|
949
|
+
return v4Span # type: ignore
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
def _convert_v1_span_event(event: Any, string_table: List[str]) -> SpanEvent:
|
|
953
|
+
if not isinstance(event, dict):
|
|
954
|
+
raise TypeError("Span event must be a map, got type %r." % type(event))
|
|
955
|
+
|
|
956
|
+
# Create a regular dict first, then cast to TypedDict
|
|
957
|
+
v4Event: Dict[str, Any] = {}
|
|
958
|
+
|
|
959
|
+
for k, v in event.items():
|
|
960
|
+
if k == V1SpanEventKeys.TIME:
|
|
961
|
+
v4Event["time_unix_nano"] = v
|
|
962
|
+
elif k == V1SpanEventKeys.NAME:
|
|
963
|
+
v4Event["name"] = _get_and_add_string(string_table, v)
|
|
964
|
+
elif k == V1SpanEventKeys.ATTRIBUTES:
|
|
965
|
+
v4Event["attributes"] = _convert_v1_span_event_attributes(v, string_table)
|
|
966
|
+
else:
|
|
967
|
+
raise TypeError("Unknown key %r in v1 span event" % k)
|
|
968
|
+
|
|
969
|
+
# Cast to TypedDict
|
|
970
|
+
return v4Event # type: ignore
|
|
971
|
+
|
|
972
|
+
|
|
973
|
+
def _convert_v1_span_link(link: Any, string_table: List[str]) -> SpanLink:
|
|
974
|
+
if not isinstance(link, dict):
|
|
975
|
+
raise TypeError("Span link must be a map, got type %r." % type(link))
|
|
976
|
+
|
|
977
|
+
# Create a regular dict first, then cast to TypedDict
|
|
978
|
+
v4Link: Dict[str, Any] = {}
|
|
979
|
+
|
|
980
|
+
for k, v in link.items():
|
|
981
|
+
if k == V1SpanLinkKeys.TRACE_ID:
|
|
982
|
+
if len(v) != 16:
|
|
983
|
+
raise TypeError("Trace ID must be 16 bytes, got %r." % len(v))
|
|
984
|
+
# trace_id is a 128 bit integer in a bytes array, so we need to get the last 64 bits
|
|
985
|
+
v4Link["trace_id"] = int.from_bytes(v[8:], "big")
|
|
986
|
+
v4Link["trace_id_high"] = int.from_bytes(v[:8], "big")
|
|
987
|
+
elif k == V1SpanLinkKeys.SPAN_ID:
|
|
988
|
+
v4Link["span_id"] = v
|
|
989
|
+
elif k == V1SpanLinkKeys.ATTRIBUTES:
|
|
990
|
+
v4Link["attributes"] = _convert_v1_span_link_attributes(v, string_table)
|
|
991
|
+
elif k == V1SpanLinkKeys.TRACE_STATE:
|
|
992
|
+
v4Link["tracestate"] = _get_and_add_string(string_table, v)
|
|
993
|
+
elif k == V1SpanLinkKeys.FLAGS:
|
|
994
|
+
v4Link["flags"] = v
|
|
995
|
+
else:
|
|
996
|
+
raise TypeError("Unknown key %r in v1 span link" % k)
|
|
997
|
+
|
|
998
|
+
# Cast to TypedDict
|
|
999
|
+
return v4Link # type: ignore
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def _convert_v1_span_link_attributes(attr: Any, string_table: List[str]) -> Dict[str, str]:
|
|
1003
|
+
"""
|
|
1004
|
+
Convert a v1 span link attributes to a v4 span link attributes. Unfortunately we need multiple implementations that
|
|
1005
|
+
convert "attributes" as the v0.4 representation of attributes is different between span links and span events.
|
|
1006
|
+
"""
|
|
1007
|
+
if not isinstance(attr, list):
|
|
1008
|
+
raise TypeError("Attribute must be a list, got type %r." % type(attr))
|
|
1009
|
+
if len(attr) % 3 != 0:
|
|
1010
|
+
raise TypeError("Attribute list must have a multiple of 3 elements, got %r." % len(attr))
|
|
1011
|
+
v4_attributes: Dict[str, str] = {}
|
|
1012
|
+
for i in range(0, len(attr), 3):
|
|
1013
|
+
key = _get_and_add_string(string_table, attr[i])
|
|
1014
|
+
value_type = attr[i + 1]
|
|
1015
|
+
value = attr[i + 2]
|
|
1016
|
+
if value_type == V1AnyValueKeys.STRING:
|
|
1017
|
+
v4_attributes[key] = _get_and_add_string(string_table, value)
|
|
1018
|
+
elif value_type == V1AnyValueKeys.BOOL:
|
|
1019
|
+
v4_attributes[key] = "true" if value else "false"
|
|
1020
|
+
elif value_type == V1AnyValueKeys.DOUBLE:
|
|
1021
|
+
v4_attributes[key] = str(value)
|
|
1022
|
+
elif value_type == V1AnyValueKeys.INT:
|
|
1023
|
+
v4_attributes[key] = str(value)
|
|
1024
|
+
elif value_type == V1AnyValueKeys.BYTES:
|
|
1025
|
+
raise NotImplementedError("Bytes values are not supported yet.")
|
|
1026
|
+
elif value_type == V1AnyValueKeys.ARRAY:
|
|
1027
|
+
raise NotImplementedError("Array of values are not supported yet.")
|
|
1028
|
+
elif value_type == V1AnyValueKeys.KEY_VALUE_LIST:
|
|
1029
|
+
raise NotImplementedError("Key value list values are not supported yet.")
|
|
1030
|
+
else:
|
|
1031
|
+
raise TypeError("Unknown attribute value type %r." % value_type)
|
|
1032
|
+
return v4_attributes
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
def _convert_v1_span_event_attributes(attr: Any, string_table: List[str]) -> Dict[str, Dict[str, Any]]:
|
|
1036
|
+
"""
|
|
1037
|
+
Convert a v1 span event attributes to a v4 span event attributes. Unfortunately we need multiple implementations that
|
|
1038
|
+
convert "attributes" as the v0.4 representation of attributes is different between span links and span events.
|
|
1039
|
+
"""
|
|
1040
|
+
if not isinstance(attr, list):
|
|
1041
|
+
raise TypeError("Attribute must be a list, got type %r." % type(attr))
|
|
1042
|
+
if len(attr) % 3 != 0:
|
|
1043
|
+
raise TypeError("Attribute list must have a multiple of 3 elements, got %r." % len(attr))
|
|
1044
|
+
attributes: Dict[str, Dict[str, Any]] = {}
|
|
1045
|
+
for i in range(0, len(attr), 3):
|
|
1046
|
+
v4_attr_value: Dict[str, Any] = {}
|
|
1047
|
+
key = _get_and_add_string(string_table, attr[i])
|
|
1048
|
+
value_type = attr[i + 1]
|
|
1049
|
+
value = attr[i + 2]
|
|
1050
|
+
if value_type == V1AnyValueKeys.STRING:
|
|
1051
|
+
v4_attr_value["type"] = 0
|
|
1052
|
+
v4_attr_value["string_value"] = _get_and_add_string(string_table, value)
|
|
1053
|
+
elif value_type == V1AnyValueKeys.BOOL:
|
|
1054
|
+
v4_attr_value["type"] = 1
|
|
1055
|
+
v4_attr_value["bool_value"] = value
|
|
1056
|
+
elif value_type == V1AnyValueKeys.DOUBLE:
|
|
1057
|
+
v4_attr_value["type"] = 3
|
|
1058
|
+
v4_attr_value["double_value"] = value
|
|
1059
|
+
elif value_type == V1AnyValueKeys.INT:
|
|
1060
|
+
v4_attr_value["type"] = 2 # Yes the constants are different here
|
|
1061
|
+
v4_attr_value["int_value"] = value
|
|
1062
|
+
elif value_type == V1AnyValueKeys.BYTES:
|
|
1063
|
+
raise NotImplementedError("Bytes values are not supported yet.")
|
|
1064
|
+
elif value_type == V1AnyValueKeys.ARRAY:
|
|
1065
|
+
raise NotImplementedError("Array of strings values are not supported yet.")
|
|
1066
|
+
elif value_type == V1AnyValueKeys.KEY_VALUE_LIST:
|
|
1067
|
+
raise NotImplementedError("Key value list values are not supported yet.")
|
|
1068
|
+
else:
|
|
1069
|
+
raise TypeError("Unknown attribute value type %r." % value_type)
|
|
1070
|
+
attributes[key] = v4_attr_value
|
|
1071
|
+
return attributes
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
def _convert_v1_attributes(
|
|
1075
|
+
attr: Any, meta: Dict[str, str], metrics: Dict[str, MetricType], string_table: List[str]
|
|
1076
|
+
) -> None:
|
|
1077
|
+
if not isinstance(attr, list):
|
|
1078
|
+
raise TypeError("Attribute must be a list, got type %r." % type(attr))
|
|
1079
|
+
if len(attr) % 3 != 0:
|
|
1080
|
+
raise TypeError("Attribute list must have a multiple of 3 elements, got %r." % len(attr))
|
|
1081
|
+
for i in range(0, len(attr), 3):
|
|
1082
|
+
key = _get_and_add_string(string_table, attr[i])
|
|
1083
|
+
value_type = attr[i + 1]
|
|
1084
|
+
value = attr[i + 2]
|
|
1085
|
+
if value_type == V1AnyValueKeys.STRING:
|
|
1086
|
+
meta[key] = _get_and_add_string(string_table, value)
|
|
1087
|
+
elif value_type == V1AnyValueKeys.BOOL:
|
|
1088
|
+
# Treat v1 boolean attributes as metrics with a value of 1 or 0
|
|
1089
|
+
metrics[key] = 1 if value else 0
|
|
1090
|
+
elif value_type == V1AnyValueKeys.DOUBLE:
|
|
1091
|
+
metrics[key] = value
|
|
1092
|
+
elif value_type == V1AnyValueKeys.INT:
|
|
1093
|
+
metrics[key] = value
|
|
1094
|
+
elif value_type == V1AnyValueKeys.BYTES:
|
|
1095
|
+
raise NotImplementedError("Bytes values are not supported yet.")
|
|
1096
|
+
elif value_type == V1AnyValueKeys.ARRAY:
|
|
1097
|
+
raise NotImplementedError("Array of strings values are not supported yet.")
|
|
1098
|
+
elif value_type == V1AnyValueKeys.KEY_VALUE_LIST:
|
|
1099
|
+
raise NotImplementedError("Key value list values are not supported yet.")
|
|
1100
|
+
else:
|
|
1101
|
+
raise TypeError("Unknown attribute value type %r." % value_type)
|
|
1102
|
+
|
|
1103
|
+
|
|
705
1104
|
def _verify_v07_payload(data: Any) -> v04TracePayload:
|
|
706
1105
|
if not isinstance(data, dict):
|
|
707
1106
|
raise TypeError("Trace payload must be a map, got type %r." % type(data))
|
ddapm_test_agent/vcr_proxy.py
CHANGED
|
@@ -1,21 +1,38 @@
|
|
|
1
1
|
import hashlib
|
|
2
2
|
import json
|
|
3
|
+
import logging
|
|
3
4
|
import os
|
|
4
5
|
import re
|
|
6
|
+
from typing import Any
|
|
7
|
+
from typing import Dict
|
|
5
8
|
from typing import Optional
|
|
6
9
|
from urllib.parse import urljoin
|
|
7
10
|
|
|
8
11
|
from aiohttp.web import Request
|
|
9
12
|
from aiohttp.web import Response
|
|
10
13
|
import requests
|
|
14
|
+
from requests_aws4auth import AWS4Auth
|
|
11
15
|
import vcr
|
|
12
16
|
|
|
13
17
|
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# Used for AWS signature recalculation for aws services initial proxying
|
|
22
|
+
AWS_REGION = os.environ.get("AWS_REGION", "us-east-1")
|
|
23
|
+
AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY")
|
|
24
|
+
|
|
25
|
+
|
|
14
26
|
def url_path_join(base_url: str, path: str) -> str:
|
|
15
27
|
"""Join a base URL with a path, handling slashes automatically."""
|
|
16
28
|
return urljoin(base_url.rstrip("/") + "/", path.lstrip("/"))
|
|
17
29
|
|
|
18
30
|
|
|
31
|
+
AWS_SERVICES = {
|
|
32
|
+
"bedrock-runtime": "bedrock",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
19
36
|
PROVIDER_BASE_URLS = {
|
|
20
37
|
"openai": "https://api.openai.com/v1",
|
|
21
38
|
"azure-openai": "https://dd.openai.azure.com/",
|
|
@@ -23,8 +40,25 @@ PROVIDER_BASE_URLS = {
|
|
|
23
40
|
"anthropic": "https://api.anthropic.com/",
|
|
24
41
|
"datadog": "https://api.datadoghq.com/",
|
|
25
42
|
"genai": "https://generativelanguage.googleapis.com/",
|
|
43
|
+
"bedrock-runtime": f"https://bedrock-runtime.{AWS_REGION}.amazonaws.com",
|
|
26
44
|
}
|
|
27
45
|
|
|
46
|
+
CASSETTE_FILTER_HEADERS = [
|
|
47
|
+
"authorization",
|
|
48
|
+
"OpenAI-Organization",
|
|
49
|
+
"api-key",
|
|
50
|
+
"x-api-key",
|
|
51
|
+
"dd-api-key",
|
|
52
|
+
"dd-application-key",
|
|
53
|
+
"x-goog-api-key",
|
|
54
|
+
"x-amz-security-token",
|
|
55
|
+
"x-amz-content-sha256",
|
|
56
|
+
"x-amz-date",
|
|
57
|
+
"x-amz-user-agent",
|
|
58
|
+
"amz-sdk-invocation-id",
|
|
59
|
+
"amz-sdk-request",
|
|
60
|
+
]
|
|
61
|
+
|
|
28
62
|
NORMALIZERS = [
|
|
29
63
|
(
|
|
30
64
|
r"--form-data-boundary-[^\r\n]+",
|
|
@@ -65,6 +99,21 @@ def normalize_multipart_body(body: bytes) -> str:
|
|
|
65
99
|
return f"[binary_data_{hex_digest}]"
|
|
66
100
|
|
|
67
101
|
|
|
102
|
+
def parse_authorization_header(auth_header: str) -> Dict[str, str]:
|
|
103
|
+
"""Parse AWS Authorization header to extract components"""
|
|
104
|
+
if not auth_header.startswith("AWS4-HMAC-SHA256 "):
|
|
105
|
+
return {}
|
|
106
|
+
|
|
107
|
+
auth_parts = auth_header[len("AWS4-HMAC-SHA256 ") :].split(",")
|
|
108
|
+
parsed = {}
|
|
109
|
+
|
|
110
|
+
for part in auth_parts:
|
|
111
|
+
key, value = part.split("=", 1)
|
|
112
|
+
parsed[key.strip()] = value.strip()
|
|
113
|
+
|
|
114
|
+
return parsed
|
|
115
|
+
|
|
116
|
+
|
|
68
117
|
def get_vcr(subdirectory: str, vcr_cassettes_directory: str) -> vcr.VCR:
|
|
69
118
|
cassette_dir = os.path.join(vcr_cassettes_directory, subdirectory)
|
|
70
119
|
|
|
@@ -72,15 +121,7 @@ def get_vcr(subdirectory: str, vcr_cassettes_directory: str) -> vcr.VCR:
|
|
|
72
121
|
cassette_library_dir=cassette_dir,
|
|
73
122
|
record_mode="once",
|
|
74
123
|
match_on=["path", "method"],
|
|
75
|
-
filter_headers=
|
|
76
|
-
"authorization",
|
|
77
|
-
"OpenAI-Organization",
|
|
78
|
-
"api-key",
|
|
79
|
-
"x-api-key",
|
|
80
|
-
"dd-api-key",
|
|
81
|
-
"dd-application-key",
|
|
82
|
-
"x-goog-api-key",
|
|
83
|
-
],
|
|
124
|
+
filter_headers=CASSETTE_FILTER_HEADERS,
|
|
84
125
|
)
|
|
85
126
|
|
|
86
127
|
|
|
@@ -125,31 +166,47 @@ async def proxy_request(request: Request, vcr_cassettes_directory: str) -> Respo
|
|
|
125
166
|
body_bytes = await request.read()
|
|
126
167
|
|
|
127
168
|
vcr_cassette_prefix = request.pop("vcr_cassette_prefix", None)
|
|
128
|
-
|
|
129
169
|
cassette_name = generate_cassette_name(path, request.method, body_bytes, vcr_cassette_prefix)
|
|
170
|
+
|
|
171
|
+
request_kwargs: Dict[str, Any] = {
|
|
172
|
+
"method": request.method,
|
|
173
|
+
"url": target_url,
|
|
174
|
+
"headers": headers,
|
|
175
|
+
"data": body_bytes,
|
|
176
|
+
"cookies": dict(request.cookies),
|
|
177
|
+
"allow_redirects": False,
|
|
178
|
+
"stream": True,
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if provider in AWS_SERVICES and not os.path.exists(os.path.join(vcr_cassettes_directory, provider, cassette_name)):
|
|
182
|
+
if not AWS_SECRET_ACCESS_KEY:
|
|
183
|
+
return Response(
|
|
184
|
+
body="AWS_SECRET_ACCESS_KEY environment variable not set for aws signature recalculation",
|
|
185
|
+
status=400,
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
auth_header = request.headers.get("Authorization", "")
|
|
189
|
+
auth_parts = parse_authorization_header(auth_header)
|
|
190
|
+
aws_access_key = auth_parts.get("Credential", "").split("/")[0]
|
|
191
|
+
|
|
192
|
+
auth = AWS4Auth(aws_access_key, AWS_SECRET_ACCESS_KEY, AWS_REGION, AWS_SERVICES[provider])
|
|
193
|
+
request_kwargs["auth"] = auth
|
|
194
|
+
|
|
130
195
|
with get_vcr(provider, vcr_cassettes_directory).use_cassette(f"{cassette_name}.yaml"):
|
|
131
|
-
|
|
132
|
-
method=request.method,
|
|
133
|
-
url=target_url,
|
|
134
|
-
headers=headers,
|
|
135
|
-
data=body_bytes,
|
|
136
|
-
cookies=dict(request.cookies),
|
|
137
|
-
allow_redirects=False,
|
|
138
|
-
stream=True,
|
|
139
|
-
)
|
|
196
|
+
provider_response = requests.request(**request_kwargs)
|
|
140
197
|
|
|
141
198
|
# Extract content type without charset
|
|
142
|
-
content_type =
|
|
199
|
+
content_type = provider_response.headers.get("content-type", "")
|
|
143
200
|
if ";" in content_type:
|
|
144
201
|
content_type = content_type.split(";")[0].strip()
|
|
145
202
|
|
|
146
203
|
response = Response(
|
|
147
|
-
body=
|
|
148
|
-
status=
|
|
204
|
+
body=provider_response.content,
|
|
205
|
+
status=provider_response.status_code,
|
|
149
206
|
content_type=content_type,
|
|
150
207
|
)
|
|
151
208
|
|
|
152
|
-
for key, value in
|
|
209
|
+
for key, value in provider_response.headers.items():
|
|
153
210
|
if key.lower() not in (
|
|
154
211
|
"content-length",
|
|
155
212
|
"transfer-encoding",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ddapm-test-agent
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.33.0
|
|
4
4
|
Summary: Test agent for Datadog APM client libraries
|
|
5
5
|
Home-page: https://github.com/Datadog/dd-apm-test-agent
|
|
6
6
|
Author: Kyle Verhoog
|
|
@@ -24,6 +24,10 @@ Requires-Dist: requests
|
|
|
24
24
|
Requires-Dist: typing_extensions
|
|
25
25
|
Requires-Dist: yarl
|
|
26
26
|
Requires-Dist: vcrpy
|
|
27
|
+
Requires-Dist: requests-aws4auth
|
|
28
|
+
Requires-Dist: opentelemetry-proto<1.37.0,>1.33.0
|
|
29
|
+
Requires-Dist: protobuf>=3.19.0
|
|
30
|
+
Requires-Dist: grpcio<2.0,>=1.66.2
|
|
27
31
|
Provides-Extra: testing
|
|
28
32
|
Requires-Dist: ddtrace==3.11.0; extra == "testing"
|
|
29
33
|
Requires-Dist: pytest; extra == "testing"
|
|
@@ -69,13 +73,16 @@ The test agent can be installed from PyPI:
|
|
|
69
73
|
|
|
70
74
|
pip install ddapm-test-agent
|
|
71
75
|
|
|
72
|
-
|
|
76
|
+
# HTTP on port 8126, OTLP HTTP on port 4318, OTLP GRPC on port 4317
|
|
77
|
+
ddapm-test-agent --port=8126 --otlp-http-port=4318 --otlp-grpc-port=4317
|
|
73
78
|
|
|
74
79
|
or from Docker:
|
|
75
80
|
|
|
76
81
|
# Run the test agent and mount the snapshot directory
|
|
77
82
|
docker run --rm\
|
|
78
83
|
-p 8126:8126\
|
|
84
|
+
-p 4318:4318\
|
|
85
|
+
-p 4317:4317\
|
|
79
86
|
-e SNAPSHOT_CI=0\
|
|
80
87
|
-v $PWD/tests/snapshots:/snapshots\
|
|
81
88
|
ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:latest
|
|
@@ -186,6 +193,13 @@ The cassettes are matched based on the path, method, and body of the request. To
|
|
|
186
193
|
|
|
187
194
|
Optionally specifying whatever mounted path is used for the cassettes directory. The test agent comes with a default set of cassettes for OpenAI, Azure OpenAI, and DeepSeek.
|
|
188
195
|
|
|
196
|
+
#### AWS Services
|
|
197
|
+
AWS service proxying, specifically recording cassettes for the first time, requires a `AWS_SECRET_ACCESS_KEY` environment variable to be set for the container running the test agent. This is used to recalculate the AWS signature for the request, as the one generated client-side likely used `{test-agent-host}:{test-agent-port}/vcr/{aws-service}` as the host, and the signature will mismatch that on the actual AWS service.
|
|
198
|
+
|
|
199
|
+
Additionally, the `AWS_REGION` environment variable can be set, defaulting to `us-east-1`.
|
|
200
|
+
|
|
201
|
+
To add a new AWS service to proxy, add an entry in the `PROVIDER_BASE_URLS` for its provider url, and an entry in the `AWS_SERVICES` dictionary for the service name, since they are not always a one-to-one mapping with the implied provider url (e.g, `https://bedrock-runtime.{AWS_REGION}.amazonaws.com` is the provider url, but the service name is `bedrock`, as `bedrock` also has multiple sub services, like `converse`).
|
|
202
|
+
|
|
189
203
|
#### Usage in clients
|
|
190
204
|
|
|
191
205
|
To use this feature in your client, you can use the `/vcr/{provider}` endpoint to proxy requests to the provider API.
|
|
@@ -422,6 +436,24 @@ Return stats that have been received by the agent for the given session token.
|
|
|
422
436
|
|
|
423
437
|
Stats are returned as a JSON list of the stats payloads received.
|
|
424
438
|
|
|
439
|
+
### /test/session/logs
|
|
440
|
+
|
|
441
|
+
Return OpenTelemetry logs that have been received by the agent for the given session token.
|
|
442
|
+
|
|
443
|
+
#### [optional] `?test_session_token=`
|
|
444
|
+
#### [optional] `X-Datadog-Test-Session-Token`
|
|
445
|
+
|
|
446
|
+
Logs are returned as a JSON list of the OTLP logs payloads received. The logs are in the standard OpenTelemetry Protocol (OTLP) v1.7.0 format, decoded from protobuf into JSON.
|
|
447
|
+
|
|
448
|
+
### /test/session/metrics
|
|
449
|
+
|
|
450
|
+
Return OpenTelemetry metrics that have been received by the agent for the given session token.
|
|
451
|
+
|
|
452
|
+
#### [optional] `?test_session_token=`
|
|
453
|
+
#### [optional] `X-Datadog-Test-Session-Token`
|
|
454
|
+
|
|
455
|
+
Metrics are returned as a JSON list of the OTLP metrics payloads received. The metrics are in the standard OpenTelemetry Protocol (OTLP) v1.7.0 format, decoded from protobuf into JSON.
|
|
456
|
+
|
|
425
457
|
### /test/session/responses/config (POST)
|
|
426
458
|
Create a Remote Config payload to retrieve in endpoint `/v0.7/config`
|
|
427
459
|
|
|
@@ -522,6 +554,35 @@ curl -X GET 'http://0.0.0.0:8126/test/integrations/tested_versions'
|
|
|
522
554
|
|
|
523
555
|
Mimics the pipeline_stats endpoint of the agent, but always returns OK, and logs a line everytime it's called.
|
|
524
556
|
|
|
557
|
+
### /v1/logs (HTTP)
|
|
558
|
+
|
|
559
|
+
Accepts OpenTelemetry Protocol (OTLP) v1.7.0 logs in protobuf format via HTTP. This endpoint validates and decodes OTLP logs payloads for testing OpenTelemetry logs exporters and libraries.
|
|
560
|
+
|
|
561
|
+
The HTTP endpoint accepts `POST` requests with `Content-Type: application/x-protobuf` and `Content-Type: application/json` and stores the decoded logs for retrieval via the `/test/session/logs` endpoint.
|
|
562
|
+
|
|
563
|
+
### /v1/metrics (HTTP)
|
|
564
|
+
|
|
565
|
+
Accepts OpenTelemetry Protocol (OTLP) v1.7.0 metrics in protobuf format via HTTP. This endpoint validates and decodes OTLP metrics payloads for testing OpenTelemetry metrics exporters and libraries.
|
|
566
|
+
|
|
567
|
+
The HTTP endpoint accepts `POST` requests with `Content-Type: application/x-protobuf` and `Content-Type: application/json` and stores the decoded metrics for retrieval via the `/test/session/metrics` endpoint.
|
|
568
|
+
|
|
569
|
+
### OTLP Logs and Metrics via GRPC
|
|
570
|
+
|
|
571
|
+
OTLP logs and metrics can also be sent via GRPC using the OpenTelemetry `LogsService.Export` and `MetricsService.Export` methods respectively. The GRPC server implements the standard OTLP service interfaces and forwards all requests to the HTTP server, ensuring consistent processing and session management.
|
|
572
|
+
|
|
573
|
+
**Note:** OTLP endpoints are served on separate ports from the main APM endpoints (default: 8126):
|
|
574
|
+
- **HTTP**: Port 4318 (default) - Use `--otlp-http-port` to configure
|
|
575
|
+
- **GRPC**: Port 4317 (default) - Use `--otlp-grpc-port` to configure
|
|
576
|
+
|
|
577
|
+
Both protocols store decoded data for retrieval via the `/test/session/logs` and `/test/session/metrics` HTTP endpoints respectively.
|
|
578
|
+
|
|
579
|
+
GRPC Client → GRPC Server → HTTP POST → HTTP Server → Agent Storage
|
|
580
|
+
↓ ↓
|
|
581
|
+
(forwards protobuf) (session management)
|
|
582
|
+
↓ ↓
|
|
583
|
+
HTTP Retrievable via
|
|
584
|
+
Response /test/session/{logs,metrics}
|
|
585
|
+
|
|
525
586
|
### /tracer_flare/v1
|
|
526
587
|
|
|
527
588
|
Mimics the tracer_flare endpoint of the agent. Returns OK if the flare contains the required form fields, otherwise `400`.
|