sf-veritas 0.11.10__cp314-cp314-manylinux_2_28_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sf_veritas/__init__.py +46 -0
- sf_veritas/_auto_preload.py +73 -0
- sf_veritas/_sfconfig.c +162 -0
- sf_veritas/_sfconfig.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sfcrashhandler.c +267 -0
- sf_veritas/_sfcrashhandler.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sffastlog.c +953 -0
- sf_veritas/_sffastlog.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sffastnet.c +994 -0
- sf_veritas/_sffastnet.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sffastnetworkrequest.c +727 -0
- sf_veritas/_sffastnetworkrequest.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sffuncspan.c +2791 -0
- sf_veritas/_sffuncspan.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sffuncspan_config.c +730 -0
- sf_veritas/_sffuncspan_config.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sfheadercheck.c +341 -0
- sf_veritas/_sfheadercheck.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sfnetworkhop.c +1454 -0
- sf_veritas/_sfnetworkhop.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sfservice.c +1223 -0
- sf_veritas/_sfservice.cpython-314-x86_64-linux-gnu.so +0 -0
- sf_veritas/_sfteepreload.c +6227 -0
- sf_veritas/app_config.py +57 -0
- sf_veritas/cli.py +336 -0
- sf_veritas/constants.py +10 -0
- sf_veritas/custom_excepthook.py +304 -0
- sf_veritas/custom_log_handler.py +146 -0
- sf_veritas/custom_output_wrapper.py +153 -0
- sf_veritas/custom_print.py +153 -0
- sf_veritas/django_app.py +5 -0
- sf_veritas/env_vars.py +186 -0
- sf_veritas/exception_handling_middleware.py +18 -0
- sf_veritas/exception_metaclass.py +69 -0
- sf_veritas/fast_frame_info.py +116 -0
- sf_veritas/fast_network_hop.py +293 -0
- sf_veritas/frame_tools.py +112 -0
- sf_veritas/funcspan_config_loader.py +693 -0
- sf_veritas/function_span_profiler.py +1313 -0
- sf_veritas/get_preload_path.py +34 -0
- sf_veritas/import_hook.py +62 -0
- sf_veritas/infra_details/__init__.py +3 -0
- sf_veritas/infra_details/get_infra_details.py +24 -0
- sf_veritas/infra_details/kubernetes/__init__.py +3 -0
- sf_veritas/infra_details/kubernetes/get_cluster_name.py +147 -0
- sf_veritas/infra_details/kubernetes/get_details.py +7 -0
- sf_veritas/infra_details/running_on/__init__.py +17 -0
- sf_veritas/infra_details/running_on/kubernetes.py +11 -0
- sf_veritas/interceptors.py +543 -0
- sf_veritas/libsfnettee.so +0 -0
- sf_veritas/local_env_detect.py +118 -0
- sf_veritas/package_metadata.py +6 -0
- sf_veritas/patches/__init__.py +0 -0
- sf_veritas/patches/_patch_tracker.py +74 -0
- sf_veritas/patches/concurrent_futures.py +19 -0
- sf_veritas/patches/constants.py +1 -0
- sf_veritas/patches/exceptions.py +82 -0
- sf_veritas/patches/multiprocessing.py +32 -0
- sf_veritas/patches/network_libraries/__init__.py +99 -0
- sf_veritas/patches/network_libraries/aiohttp.py +294 -0
- sf_veritas/patches/network_libraries/curl_cffi.py +363 -0
- sf_veritas/patches/network_libraries/http_client.py +670 -0
- sf_veritas/patches/network_libraries/httpcore.py +580 -0
- sf_veritas/patches/network_libraries/httplib2.py +315 -0
- sf_veritas/patches/network_libraries/httpx.py +557 -0
- sf_veritas/patches/network_libraries/niquests.py +218 -0
- sf_veritas/patches/network_libraries/pycurl.py +399 -0
- sf_veritas/patches/network_libraries/requests.py +595 -0
- sf_veritas/patches/network_libraries/ssl_socket.py +822 -0
- sf_veritas/patches/network_libraries/tornado.py +360 -0
- sf_veritas/patches/network_libraries/treq.py +270 -0
- sf_veritas/patches/network_libraries/urllib_request.py +483 -0
- sf_veritas/patches/network_libraries/utils.py +598 -0
- sf_veritas/patches/os.py +17 -0
- sf_veritas/patches/threading.py +231 -0
- sf_veritas/patches/web_frameworks/__init__.py +54 -0
- sf_veritas/patches/web_frameworks/aiohttp.py +798 -0
- sf_veritas/patches/web_frameworks/async_websocket_consumer.py +337 -0
- sf_veritas/patches/web_frameworks/blacksheep.py +532 -0
- sf_veritas/patches/web_frameworks/bottle.py +513 -0
- sf_veritas/patches/web_frameworks/cherrypy.py +683 -0
- sf_veritas/patches/web_frameworks/cors_utils.py +122 -0
- sf_veritas/patches/web_frameworks/django.py +963 -0
- sf_veritas/patches/web_frameworks/eve.py +401 -0
- sf_veritas/patches/web_frameworks/falcon.py +931 -0
- sf_veritas/patches/web_frameworks/fastapi.py +738 -0
- sf_veritas/patches/web_frameworks/flask.py +526 -0
- sf_veritas/patches/web_frameworks/klein.py +501 -0
- sf_veritas/patches/web_frameworks/litestar.py +616 -0
- sf_veritas/patches/web_frameworks/pyramid.py +440 -0
- sf_veritas/patches/web_frameworks/quart.py +841 -0
- sf_veritas/patches/web_frameworks/robyn.py +708 -0
- sf_veritas/patches/web_frameworks/sanic.py +874 -0
- sf_veritas/patches/web_frameworks/starlette.py +742 -0
- sf_veritas/patches/web_frameworks/strawberry.py +1446 -0
- sf_veritas/patches/web_frameworks/tornado.py +485 -0
- sf_veritas/patches/web_frameworks/utils.py +170 -0
- sf_veritas/print_override.py +13 -0
- sf_veritas/regular_data_transmitter.py +444 -0
- sf_veritas/request_interceptor.py +401 -0
- sf_veritas/request_utils.py +550 -0
- sf_veritas/segfault_handler.py +116 -0
- sf_veritas/server_status.py +1 -0
- sf_veritas/shutdown_flag.py +11 -0
- sf_veritas/subprocess_startup.py +3 -0
- sf_veritas/test_cli.py +145 -0
- sf_veritas/thread_local.py +1319 -0
- sf_veritas/timeutil.py +114 -0
- sf_veritas/transmit_exception_to_sailfish.py +28 -0
- sf_veritas/transmitter.py +132 -0
- sf_veritas/types.py +47 -0
- sf_veritas/unified_interceptor.py +1678 -0
- sf_veritas/utils.py +39 -0
- sf_veritas-0.11.10.dist-info/METADATA +97 -0
- sf_veritas-0.11.10.dist-info/RECORD +141 -0
- sf_veritas-0.11.10.dist-info/WHEEL +5 -0
- sf_veritas-0.11.10.dist-info/entry_points.txt +2 -0
- sf_veritas-0.11.10.dist-info/top_level.txt +1 -0
- sf_veritas.libs/libbrotlicommon-6ce2a53c.so.1.0.6 +0 -0
- sf_veritas.libs/libbrotlidec-811d1be3.so.1.0.6 +0 -0
- sf_veritas.libs/libcom_err-730ca923.so.2.1 +0 -0
- sf_veritas.libs/libcrypt-52aca757.so.1.1.0 +0 -0
- sf_veritas.libs/libcrypto-bdaed0ea.so.1.1.1k +0 -0
- sf_veritas.libs/libcurl-eaa3cf66.so.4.5.0 +0 -0
- sf_veritas.libs/libgssapi_krb5-323bbd21.so.2.2 +0 -0
- sf_veritas.libs/libidn2-2f4a5893.so.0.3.6 +0 -0
- sf_veritas.libs/libk5crypto-9a74ff38.so.3.1 +0 -0
- sf_veritas.libs/libkeyutils-2777d33d.so.1.6 +0 -0
- sf_veritas.libs/libkrb5-a55300e8.so.3.3 +0 -0
- sf_veritas.libs/libkrb5support-e6594cfc.so.0.1 +0 -0
- sf_veritas.libs/liblber-2-d20824ef.4.so.2.10.9 +0 -0
- sf_veritas.libs/libldap-2-cea2a960.4.so.2.10.9 +0 -0
- sf_veritas.libs/libnghttp2-39367a22.so.14.17.0 +0 -0
- sf_veritas.libs/libpcre2-8-516f4c9d.so.0.7.1 +0 -0
- sf_veritas.libs/libpsl-99becdd3.so.5.3.1 +0 -0
- sf_veritas.libs/libsasl2-7de4d792.so.3.0.0 +0 -0
- sf_veritas.libs/libselinux-d0805dcb.so.1 +0 -0
- sf_veritas.libs/libssh-c11d285b.so.4.8.7 +0 -0
- sf_veritas.libs/libssl-60250281.so.1.1.1k +0 -0
- sf_veritas.libs/libunistring-05abdd40.so.2.1.0 +0 -0
- sf_veritas.libs/libuuid-95b83d40.so.1.3.0 +0 -0
|
@@ -0,0 +1,670 @@
|
|
|
1
|
+
# sf_veritas/patches/network_libraries/http_client.py
|
|
2
|
+
import os
|
|
3
|
+
import time
|
|
4
|
+
import traceback
|
|
5
|
+
from typing import List, Optional, Tuple
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
import wrapt
|
|
9
|
+
|
|
10
|
+
HAS_WRAPT = True
|
|
11
|
+
except ImportError:
|
|
12
|
+
HAS_WRAPT = False
|
|
13
|
+
|
|
14
|
+
from ... import _sffastnetworkrequest as _fast # native module
|
|
15
|
+
from ...constants import SAILFISH_TRACING_HEADER, PARENT_SESSION_ID_HEADER
|
|
16
|
+
from ...env_vars import SF_DEBUG
|
|
17
|
+
from ...thread_local import is_network_recording_suppressed, trace_id_ctx
|
|
18
|
+
from .utils import init_fast_header_check, inject_headers_ultrafast, record_network_request, init_fast_network_tracking, is_ssl_socket_active, has_sailfish_header
|
|
19
|
+
|
|
20
|
+
# JSON serialization - try fast orjson first, fallback to stdlib json
|
|
21
|
+
try:
|
|
22
|
+
import orjson
|
|
23
|
+
|
|
24
|
+
HAS_ORJSON = True
|
|
25
|
+
except ImportError:
|
|
26
|
+
import json
|
|
27
|
+
|
|
28
|
+
HAS_ORJSON = False
|
|
29
|
+
|
|
30
|
+
TRACE_HEADER_LOWER = SAILFISH_TRACING_HEADER.lower()
|
|
31
|
+
PARENT_HEADER_LOWER = PARENT_SESSION_ID_HEADER.lower()
|
|
32
|
+
# --- Native fast path (C) readiness probe -------------------------
|
|
33
|
+
_FAST = None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _install_putheader_debug(_hc):
|
|
37
|
+
"""Instrument http.client putheader to trace Sailfish headers when SF_DEBUG is on."""
|
|
38
|
+
if not SF_DEBUG:
|
|
39
|
+
return
|
|
40
|
+
putheader = getattr(_hc.HTTPConnection, "putheader", None)
|
|
41
|
+
if not putheader or getattr(putheader, "_sf_debug_wrapped", False):
|
|
42
|
+
return
|
|
43
|
+
|
|
44
|
+
original_putheader = putheader
|
|
45
|
+
|
|
46
|
+
def debug_putheader(self, header, *values):
|
|
47
|
+
header_text = header
|
|
48
|
+
if isinstance(header_text, bytes):
|
|
49
|
+
header_text = header_text.decode("latin-1", "ignore")
|
|
50
|
+
header_lower = header_text.lower() if isinstance(header_text, str) else None
|
|
51
|
+
if header_lower in (TRACE_HEADER_LOWER, PARENT_HEADER_LOWER):
|
|
52
|
+
try:
|
|
53
|
+
value_preview = [
|
|
54
|
+
v.decode("latin-1", "ignore") if isinstance(v, (bytes, bytearray)) else str(v)
|
|
55
|
+
for v in values
|
|
56
|
+
]
|
|
57
|
+
print(
|
|
58
|
+
"[http.client.putheader] method="
|
|
59
|
+
f"{getattr(self, '_method', '?')} header={header_text} values={value_preview}",
|
|
60
|
+
log=False,
|
|
61
|
+
)
|
|
62
|
+
stack = "".join(traceback.format_stack(limit=4))
|
|
63
|
+
print(f"[http.client.putheader] stack:\n{stack}", log=False)
|
|
64
|
+
except Exception as exc:
|
|
65
|
+
print(f"[http.client.putheader] debug log failed: {exc}", log=False)
|
|
66
|
+
return original_putheader(self, header, *values)
|
|
67
|
+
|
|
68
|
+
debug_putheader._sf_debug_wrapped = True # type: ignore[attr-defined]
|
|
69
|
+
_hc.HTTPConnection.putheader = debug_putheader
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _fast_ready() -> bool:
|
|
73
|
+
global _FAST
|
|
74
|
+
if _FAST is None:
|
|
75
|
+
try:
|
|
76
|
+
_FAST = _fast
|
|
77
|
+
|
|
78
|
+
if SF_DEBUG:
|
|
79
|
+
try:
|
|
80
|
+
print(
|
|
81
|
+
"[http_client] _sffastnetworkrequest loaded successfully",
|
|
82
|
+
log=False,
|
|
83
|
+
)
|
|
84
|
+
except TypeError:
|
|
85
|
+
print("[http_client] _sffastnetworkrequest loaded successfully")
|
|
86
|
+
except Exception as e:
|
|
87
|
+
_FAST = False
|
|
88
|
+
|
|
89
|
+
if SF_DEBUG:
|
|
90
|
+
try:
|
|
91
|
+
print(
|
|
92
|
+
f"[http_client] _sffastnetworkrequest NOT available: {e}",
|
|
93
|
+
log=False,
|
|
94
|
+
)
|
|
95
|
+
except TypeError:
|
|
96
|
+
print(f"[http_client] _sffastnetworkrequest NOT available: {e}")
|
|
97
|
+
if _FAST is False:
|
|
98
|
+
return False
|
|
99
|
+
try:
|
|
100
|
+
ready = bool(_FAST.is_ready())
|
|
101
|
+
|
|
102
|
+
if SF_DEBUG:
|
|
103
|
+
try:
|
|
104
|
+
print(
|
|
105
|
+
f"[http_client] _sffastnetworkrequest.is_ready() = {ready}",
|
|
106
|
+
log=False,
|
|
107
|
+
)
|
|
108
|
+
except TypeError:
|
|
109
|
+
print(f"[http_client] _sffastnetworkrequest.is_ready() = {ready}")
|
|
110
|
+
return ready
|
|
111
|
+
except Exception as e:
|
|
112
|
+
if SF_DEBUG:
|
|
113
|
+
try:
|
|
114
|
+
print(f"[http_client] is_ready() check failed: {e}", log=False)
|
|
115
|
+
except TypeError:
|
|
116
|
+
print(f"[http_client] is_ready() check failed: {e}")
|
|
117
|
+
return False
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _split_headers_and_body_from_send_chunk(
|
|
121
|
+
chunk: memoryview, state
|
|
122
|
+
) -> Tuple[Optional[bytes], Optional[bytes]]:
|
|
123
|
+
if state["seen_hdr_end"]:
|
|
124
|
+
return None, bytes(chunk)
|
|
125
|
+
|
|
126
|
+
mv = chunk
|
|
127
|
+
pos = mv.tobytes().find(b"\r\n\r\n")
|
|
128
|
+
if pos == -1:
|
|
129
|
+
state["hdr_buf"].append(bytes(mv))
|
|
130
|
+
return None, None
|
|
131
|
+
|
|
132
|
+
hdr_part = bytes(mv[: pos + 4])
|
|
133
|
+
body_part = bytes(mv[pos + 4 :])
|
|
134
|
+
state["hdr_buf"].append(hdr_part)
|
|
135
|
+
state["seen_hdr_end"] = True
|
|
136
|
+
return b"".join(state["hdr_buf"]), body_part if body_part else None
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _parse_request_headers_from_block(block: bytes) -> dict:
|
|
140
|
+
headers = {}
|
|
141
|
+
lines = block.split(b"\r\n")
|
|
142
|
+
for raw in lines[1:]:
|
|
143
|
+
if not raw:
|
|
144
|
+
break
|
|
145
|
+
i = raw.find(b":")
|
|
146
|
+
if i <= 0:
|
|
147
|
+
continue
|
|
148
|
+
k = raw[:i].decode("latin1", "replace").strip()
|
|
149
|
+
v = raw[i + 1 :].decode("latin1", "replace").strip()
|
|
150
|
+
headers[k] = v
|
|
151
|
+
return headers
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _tee_preload_active() -> bool:
|
|
155
|
+
"""Detect if the LD_PRELOAD tee is active; if so, skip Python-level patch."""
|
|
156
|
+
if os.getenv("SF_TEE_PRELOAD_ONLY", "0") == "1":
|
|
157
|
+
return True
|
|
158
|
+
ld = os.getenv("LD_PRELOAD", "")
|
|
159
|
+
# match our shipped name
|
|
160
|
+
return "libsfnettee.so" in ld or "_sfteepreload" in ld
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def patch_http_client(domains_to_not_propagate_headers_to: Optional[List[str]] = None):
|
|
164
|
+
"""
|
|
165
|
+
ALWAYS patch for header injection (trace_id + funcspan_override).
|
|
166
|
+
Skip capture/emission if LD_PRELOAD tee is active (socket layer already captures).
|
|
167
|
+
|
|
168
|
+
This ensures headers propagate correctly regardless of capture mechanism.
|
|
169
|
+
"""
|
|
170
|
+
preload_active = _tee_preload_active()
|
|
171
|
+
|
|
172
|
+
# Initialize C extension for ultra-fast header checking (if available)
|
|
173
|
+
if preload_active:
|
|
174
|
+
init_fast_header_check(domains_to_not_propagate_headers_to or [])
|
|
175
|
+
if SF_DEBUG:
|
|
176
|
+
try:
|
|
177
|
+
print(
|
|
178
|
+
"[http_client] LD_PRELOAD tee active; patching for headers only (no capture)",
|
|
179
|
+
log=False,
|
|
180
|
+
)
|
|
181
|
+
except TypeError:
|
|
182
|
+
print(
|
|
183
|
+
"[http_client] LD_PRELOAD tee active; patching for headers only (no capture)"
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
# Initialize fast network tracking (collectNetworkRequest mutation sender)
|
|
187
|
+
# This must be called to set up the C extension that sends network requests
|
|
188
|
+
if not preload_active:
|
|
189
|
+
init_fast_network_tracking()
|
|
190
|
+
if SF_DEBUG:
|
|
191
|
+
try:
|
|
192
|
+
print("[http_client] Initialized fast network tracking for collectNetworkRequest", log=False)
|
|
193
|
+
except TypeError:
|
|
194
|
+
print("[http_client] Initialized fast network tracking for collectNetworkRequest")
|
|
195
|
+
|
|
196
|
+
# Check if C extension is available for capture (not required for header injection)
|
|
197
|
+
fast_available = False
|
|
198
|
+
if not preload_active:
|
|
199
|
+
fast_available = _fast_ready()
|
|
200
|
+
if not fast_available and SF_DEBUG:
|
|
201
|
+
try:
|
|
202
|
+
print(
|
|
203
|
+
"[http_client] C extension not ready - will patch for headers only (no capture)",
|
|
204
|
+
log=False,
|
|
205
|
+
)
|
|
206
|
+
except TypeError:
|
|
207
|
+
print(
|
|
208
|
+
"[http_client] C extension not ready - will patch for headers only (no capture)"
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
_fast = _FAST if (not preload_active and fast_available) else None # type: ignore[assignment]
|
|
212
|
+
if domains_to_not_propagate_headers_to is None:
|
|
213
|
+
domains_to_not_propagate_headers_to = []
|
|
214
|
+
|
|
215
|
+
if SF_DEBUG:
|
|
216
|
+
mode = "headers only" if preload_active else "full capture"
|
|
217
|
+
try:
|
|
218
|
+
print(
|
|
219
|
+
f"[http_client] Patching http.client ({mode})",
|
|
220
|
+
log=False,
|
|
221
|
+
)
|
|
222
|
+
except TypeError:
|
|
223
|
+
print(f"[http_client] Patching http.client ({mode})")
|
|
224
|
+
|
|
225
|
+
try:
|
|
226
|
+
import http.client as _hc
|
|
227
|
+
except ImportError:
|
|
228
|
+
if SF_DEBUG:
|
|
229
|
+
try:
|
|
230
|
+
print("[http_client] http.client not available to patch", log=False)
|
|
231
|
+
except TypeError:
|
|
232
|
+
print("[http_client] http.client not available to patch")
|
|
233
|
+
return
|
|
234
|
+
|
|
235
|
+
_install_putheader_debug(_hc)
|
|
236
|
+
|
|
237
|
+
# Body size limits (only needed if NOT using preload)
|
|
238
|
+
if not preload_active:
|
|
239
|
+
try:
|
|
240
|
+
SFF_MAX_REQ_BODY = getattr(_fast, "SFF_MAX_REQ_BODY", 8192)
|
|
241
|
+
SFF_MAX_RESP_BODY = getattr(_fast, "SFF_MAX_RESP_BODY", 8192)
|
|
242
|
+
except Exception:
|
|
243
|
+
SFF_MAX_REQ_BODY = 8192
|
|
244
|
+
SFF_MAX_RESP_BODY = 8192
|
|
245
|
+
else:
|
|
246
|
+
SFF_MAX_REQ_BODY = 0
|
|
247
|
+
SFF_MAX_RESP_BODY = 0
|
|
248
|
+
|
|
249
|
+
if SF_DEBUG:
|
|
250
|
+
print("SFF_MAX_REQ_BODY=", SFF_MAX_REQ_BODY, log=False)
|
|
251
|
+
print("SFF_MAX_RESP_BODY=", SFF_MAX_RESP_BODY, log=False)
|
|
252
|
+
|
|
253
|
+
original_request = _hc.HTTPConnection.request
|
|
254
|
+
original_send = _hc.HTTPConnection.send
|
|
255
|
+
original_getresponse = _hc.HTTPConnection.getresponse
|
|
256
|
+
original_close = _hc.HTTPConnection.close
|
|
257
|
+
original_response_read = _hc.HTTPResponse.read
|
|
258
|
+
|
|
259
|
+
def patched_request(
|
|
260
|
+
self, method, url, body=None, headers=None, *, encode_chunked=False
|
|
261
|
+
):
|
|
262
|
+
# CRITICAL: Clean any stale capture state from connection reuse
|
|
263
|
+
# httplib2 pools connections and reuses them across requests, which can
|
|
264
|
+
# leave stale _sf_req_capture state that corrupts subsequent requests
|
|
265
|
+
if hasattr(self, "_sf_req_capture"):
|
|
266
|
+
delattr(self, "_sf_req_capture")
|
|
267
|
+
if hasattr(self, "_sf_response_processed"):
|
|
268
|
+
delattr(self, "_sf_response_processed")
|
|
269
|
+
|
|
270
|
+
# Build full URL for domain checking (http.client uses relative paths)
|
|
271
|
+
full_url = url
|
|
272
|
+
if not url.startswith(("http://", "https://")):
|
|
273
|
+
# Relative path - build full URL from connection
|
|
274
|
+
scheme = "https" if isinstance(self, _hc.HTTPSConnection) else "http"
|
|
275
|
+
full_url = (
|
|
276
|
+
f"{scheme}://{self.host}:{self.port}{url}"
|
|
277
|
+
if self.port not in (80, 443)
|
|
278
|
+
else f"{scheme}://{self.host}{url}"
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
# ULTRA-FAST header injection using inject_headers_ultrafast() (~100ns)
|
|
282
|
+
# Create dict for injection, then check if we actually added anything
|
|
283
|
+
hdrs_dict = dict(headers) if headers else {}
|
|
284
|
+
original_keys = set(hdrs_dict.keys())
|
|
285
|
+
|
|
286
|
+
# Check if headers are already injected (avoid double-injection with httplib2/requests)
|
|
287
|
+
# Fast O(1) check for common cases, fallback to O(n) for rare mixed-case variants
|
|
288
|
+
has_trace = has_sailfish_header(hdrs_dict)
|
|
289
|
+
if SF_DEBUG:
|
|
290
|
+
print(f"[http_client dedup check] headers_keys={list(hdrs_dict.keys())}, has_trace={has_trace}", log=False)
|
|
291
|
+
|
|
292
|
+
if not has_trace:
|
|
293
|
+
inject_headers_ultrafast(
|
|
294
|
+
hdrs_dict, full_url, domains_to_not_propagate_headers_to
|
|
295
|
+
)
|
|
296
|
+
elif SF_DEBUG:
|
|
297
|
+
print(f"[http_client] SKIPPED injection - headers already present!", log=False)
|
|
298
|
+
|
|
299
|
+
# Only use dict if we added headers OR original had headers (preserve None if nothing to add)
|
|
300
|
+
if headers or set(hdrs_dict.keys()) != original_keys:
|
|
301
|
+
hdrs_out = hdrs_dict
|
|
302
|
+
else:
|
|
303
|
+
hdrs_out = None # Preserve None if no headers were originally provided and none were injected
|
|
304
|
+
|
|
305
|
+
# Only capture state if NOT using LD_PRELOAD (preload captures at socket layer)
|
|
306
|
+
# ALSO skip capture for HTTPS when ssl_socket.py is active (avoids double-capture)
|
|
307
|
+
is_https = isinstance(self, _hc.HTTPSConnection)
|
|
308
|
+
skip_capture = preload_active or (is_https and is_ssl_socket_active())
|
|
309
|
+
|
|
310
|
+
if not skip_capture:
|
|
311
|
+
# Get trace_id for capture (already injected in headers)
|
|
312
|
+
trace_id = trace_id_ctx.get(None) or ""
|
|
313
|
+
|
|
314
|
+
start_ts = int(time.time() * 1_000)
|
|
315
|
+
# Store state as list [start_ts, trace_id, url, method, req_hdr_buf, req_body_buf, seen_end]
|
|
316
|
+
# Lists are mutable so patched_send can append to buffers
|
|
317
|
+
self._sf_req_capture = [
|
|
318
|
+
start_ts,
|
|
319
|
+
trace_id,
|
|
320
|
+
url,
|
|
321
|
+
method,
|
|
322
|
+
bytearray(),
|
|
323
|
+
bytearray(),
|
|
324
|
+
False,
|
|
325
|
+
]
|
|
326
|
+
|
|
327
|
+
if SF_DEBUG:
|
|
328
|
+
trace_headers = {k: v for k, v in (hdrs_out or {}).items() if isinstance(k, str) and k.lower() in (TRACE_HEADER_LOWER, PARENT_HEADER_LOWER)}
|
|
329
|
+
print(f"[http_client] CALLING original_request with headers={trace_headers}", log=False)
|
|
330
|
+
|
|
331
|
+
return original_request(
|
|
332
|
+
self,
|
|
333
|
+
method,
|
|
334
|
+
url,
|
|
335
|
+
body=body,
|
|
336
|
+
headers=hdrs_out,
|
|
337
|
+
encode_chunked=encode_chunked,
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
def patched_send(self, data):
|
|
341
|
+
state = getattr(self, "_sf_req_capture", None)
|
|
342
|
+
# DEFENSIVE: Ensure state is valid and not from a previous request (httplib2 connection reuse)
|
|
343
|
+
if state is not None and isinstance(state, list) and len(state) == 7:
|
|
344
|
+
# FAST: Capture headers and body without parsing
|
|
345
|
+
# state = [start_ts, trace_id, url, method, req_hdr_buf, req_body_buf, seen_end]
|
|
346
|
+
hdr_buf, body_buf, seen_end = state[4], state[5], state[6]
|
|
347
|
+
|
|
348
|
+
if not seen_end:
|
|
349
|
+
# Look for \r\n\r\n to split headers from body
|
|
350
|
+
pos = data.find(b"\r\n\r\n")
|
|
351
|
+
if pos >= 0:
|
|
352
|
+
hdr_buf.extend(data[: pos + 4])
|
|
353
|
+
if len(data) > pos + 4:
|
|
354
|
+
cap = SFF_MAX_REQ_BODY - len(body_buf)
|
|
355
|
+
if cap > 0:
|
|
356
|
+
body_buf.extend(data[pos + 4 : pos + 4 + cap])
|
|
357
|
+
state[6] = True # Mark seen_end
|
|
358
|
+
else:
|
|
359
|
+
hdr_buf.extend(data)
|
|
360
|
+
else:
|
|
361
|
+
# Already saw headers, just capture body
|
|
362
|
+
cap = SFF_MAX_REQ_BODY - len(body_buf)
|
|
363
|
+
if cap > 0:
|
|
364
|
+
body_buf.extend(data[:cap])
|
|
365
|
+
|
|
366
|
+
return original_send(self, data)
|
|
367
|
+
|
|
368
|
+
def patched_getresponse(self):
|
|
369
|
+
response = original_getresponse(self)
|
|
370
|
+
|
|
371
|
+
state = getattr(self, "_sf_req_capture", None)
|
|
372
|
+
if not state:
|
|
373
|
+
return response
|
|
374
|
+
|
|
375
|
+
# CRITICAL: Prevent double-processing from httplib2 connection reuse
|
|
376
|
+
# If this connection was already processed, clean up and return early
|
|
377
|
+
if getattr(self, "_sf_response_processed", False):
|
|
378
|
+
if hasattr(self, "_sf_req_capture"):
|
|
379
|
+
delattr(self, "_sf_req_capture")
|
|
380
|
+
if hasattr(self, "_sf_response_processed"):
|
|
381
|
+
delattr(self, "_sf_response_processed")
|
|
382
|
+
return response
|
|
383
|
+
|
|
384
|
+
# Check if network recording is suppressed (e.g., by @skip_network_tracing decorator)
|
|
385
|
+
if is_network_recording_suppressed():
|
|
386
|
+
delattr(self, "_sf_req_capture")
|
|
387
|
+
return response
|
|
388
|
+
|
|
389
|
+
# Mark as processed BEFORE attempting capture (prevents re-entry if getresponse called again)
|
|
390
|
+
self._sf_response_processed = True
|
|
391
|
+
|
|
392
|
+
# CRITICAL: Set up response body capture buffer (httplib2 compatibility)
|
|
393
|
+
# Instead of peek() which hangs with SSL, we'll capture body in patched_response_read()
|
|
394
|
+
# when httplib2 calls response.read() to consume the body
|
|
395
|
+
self._sf_response_body_buf = bytearray() if SFF_MAX_RESP_BODY > 0 else None
|
|
396
|
+
# Store reference to connection so patched_response_read can find the buffer
|
|
397
|
+
response._orig_conn = self
|
|
398
|
+
# Mark that we haven't emitted capture yet (will emit after body is read or on close)
|
|
399
|
+
self._sf_capture_emitted = False
|
|
400
|
+
|
|
401
|
+
# ULTRA-FAST: Prepare captured data for later emission (after body is read)
|
|
402
|
+
try:
|
|
403
|
+
# state = [start_ts, trace_id, url, method, req_hdr_buf, req_body_buf, seen_end]
|
|
404
|
+
start_ts, trace_id, url, method, req_hdr_buf, req_body_buf, _ = state
|
|
405
|
+
delattr(self, "_sf_req_capture")
|
|
406
|
+
|
|
407
|
+
status = int(getattr(response, "status", 0))
|
|
408
|
+
ok = 1 if status < 400 else 0
|
|
409
|
+
end_ts = int(time.time() * 1_000)
|
|
410
|
+
|
|
411
|
+
# FAST: Parse request headers from buffer
|
|
412
|
+
req_headers_json = "{}"
|
|
413
|
+
hdr_dict = {}
|
|
414
|
+
if req_hdr_buf:
|
|
415
|
+
try:
|
|
416
|
+
hdr_dict = _parse_request_headers_from_block(bytes(req_hdr_buf))
|
|
417
|
+
except Exception:
|
|
418
|
+
pass
|
|
419
|
+
|
|
420
|
+
if HAS_ORJSON:
|
|
421
|
+
req_headers_json = orjson.dumps(hdr_dict).decode("utf-8")
|
|
422
|
+
else:
|
|
423
|
+
req_headers_json = json.dumps(hdr_dict).decode("utf-8")
|
|
424
|
+
|
|
425
|
+
# FAST: Get response headers
|
|
426
|
+
resp_headers_json = "{}"
|
|
427
|
+
if HAS_ORJSON:
|
|
428
|
+
resp_headers_json = orjson.dumps(
|
|
429
|
+
{str(k): str(v) for k, v in response.getheaders()}
|
|
430
|
+
).decode("utf-8")
|
|
431
|
+
else:
|
|
432
|
+
resp_headers_json = json.dumps(
|
|
433
|
+
{str(k): str(v) for k, v in response.getheaders()}
|
|
434
|
+
).decode("utf-8")
|
|
435
|
+
|
|
436
|
+
# Store capture data on connection for later emission (after body is read)
|
|
437
|
+
# This deferred emission allows httplib2 to call response.read() and fill
|
|
438
|
+
# _sf_response_body_buf via patched_response_read()
|
|
439
|
+
self._sf_pending_capture = {
|
|
440
|
+
"trace_id": trace_id,
|
|
441
|
+
"url": url,
|
|
442
|
+
"method": method,
|
|
443
|
+
"status": status,
|
|
444
|
+
"ok": ok,
|
|
445
|
+
"timestamp_start": start_ts,
|
|
446
|
+
"timestamp_end": end_ts,
|
|
447
|
+
"request_body": bytes(req_body_buf) if req_body_buf else b"",
|
|
448
|
+
"request_headers_json": req_headers_json,
|
|
449
|
+
"response_headers_json": resp_headers_json,
|
|
450
|
+
}
|
|
451
|
+
except Exception:
|
|
452
|
+
pass
|
|
453
|
+
|
|
454
|
+
return response
|
|
455
|
+
|
|
456
|
+
def patched_response_read(self, amt=None):
|
|
457
|
+
"""
|
|
458
|
+
Capture response body as httplib2 reads it (solves peek() hang issue).
|
|
459
|
+
|
|
460
|
+
Instead of peeking in getresponse() (which interferes with SSL socket state),
|
|
461
|
+
we capture the body when httplib2 calls response.read() to consume it.
|
|
462
|
+
This is non-invasive and works perfectly with connection pooling.
|
|
463
|
+
"""
|
|
464
|
+
# Call original read() first
|
|
465
|
+
data = original_response_read(self, amt)
|
|
466
|
+
|
|
467
|
+
# If this response belongs to a connection being captured, store the body
|
|
468
|
+
conn = getattr(self, "_orig_conn", None)
|
|
469
|
+
if conn:
|
|
470
|
+
# Capture body if buffer exists
|
|
471
|
+
if (
|
|
472
|
+
hasattr(conn, "_sf_response_body_buf")
|
|
473
|
+
and conn._sf_response_body_buf is not None
|
|
474
|
+
and data
|
|
475
|
+
):
|
|
476
|
+
buf = conn._sf_response_body_buf
|
|
477
|
+
# Only capture up to limit
|
|
478
|
+
remaining = SFF_MAX_RESP_BODY - len(buf)
|
|
479
|
+
if remaining > 0:
|
|
480
|
+
buf.extend(data[:remaining])
|
|
481
|
+
|
|
482
|
+
# Emit capture after first read (httplib2 typically reads entire body in one call)
|
|
483
|
+
# or when no more data (empty read signals EOF)
|
|
484
|
+
if hasattr(conn, "_sf_pending_capture") and not getattr(
|
|
485
|
+
conn, "_sf_capture_emitted", False
|
|
486
|
+
):
|
|
487
|
+
# Check if we should emit now (either got data or EOF)
|
|
488
|
+
should_emit = (data is not None) and (
|
|
489
|
+
len(data) == 0 or amt is None or (amt and len(data) > 0)
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
if should_emit or not data: # Emit on first read or EOF
|
|
493
|
+
try:
|
|
494
|
+
capture = conn._sf_pending_capture
|
|
495
|
+
resp_body = (
|
|
496
|
+
bytes(conn._sf_response_body_buf)
|
|
497
|
+
if conn._sf_response_body_buf
|
|
498
|
+
else b""
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
# Emit mutation collectNetworkRequest (not collectNetworkHop!)
|
|
502
|
+
record_network_request(
|
|
503
|
+
trace_id=capture["trace_id"],
|
|
504
|
+
url=capture["url"],
|
|
505
|
+
method=capture["method"],
|
|
506
|
+
status_code=capture["status"],
|
|
507
|
+
success=capture["ok"],
|
|
508
|
+
timestamp_start=capture["timestamp_start"],
|
|
509
|
+
timestamp_end=capture["timestamp_end"],
|
|
510
|
+
request_data=capture["request_body"],
|
|
511
|
+
response_data=resp_body,
|
|
512
|
+
request_headers=capture["request_headers_json"].encode('utf-8') if capture["request_headers_json"] else b"",
|
|
513
|
+
response_headers=capture["response_headers_json"].encode('utf-8') if capture["response_headers_json"] else b"",
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
conn._sf_capture_emitted = True
|
|
517
|
+
delattr(conn, "_sf_pending_capture")
|
|
518
|
+
except Exception:
|
|
519
|
+
pass
|
|
520
|
+
|
|
521
|
+
return data
|
|
522
|
+
|
|
523
|
+
def patched_close(self):
|
|
524
|
+
"""Clean up capture state when connection is closed (httplib2 connection pooling)."""
|
|
525
|
+
# CRITICAL: Emit any pending capture before closing (if body wasn't read)
|
|
526
|
+
if hasattr(self, "_sf_pending_capture") and not getattr(
|
|
527
|
+
self, "_sf_capture_emitted", False
|
|
528
|
+
):
|
|
529
|
+
try:
|
|
530
|
+
capture = self._sf_pending_capture
|
|
531
|
+
resp_body = (
|
|
532
|
+
bytes(self._sf_response_body_buf)
|
|
533
|
+
if hasattr(self, "_sf_response_body_buf")
|
|
534
|
+
and self._sf_response_body_buf
|
|
535
|
+
else b""
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
# Emit mutation collectNetworkRequest (not collectNetworkHop!)
|
|
539
|
+
record_network_request(
|
|
540
|
+
trace_id=capture["trace_id"],
|
|
541
|
+
url=capture["url"],
|
|
542
|
+
method=capture["method"],
|
|
543
|
+
status_code=capture["status"],
|
|
544
|
+
success=capture["ok"],
|
|
545
|
+
timestamp_start=capture["timestamp_start"],
|
|
546
|
+
timestamp_end=capture["timestamp_end"],
|
|
547
|
+
request_data=capture["request_body"],
|
|
548
|
+
response_data=resp_body,
|
|
549
|
+
request_headers=capture["request_headers_json"].encode('utf-8') if capture["request_headers_json"] else b"",
|
|
550
|
+
response_headers=capture["response_headers_json"].encode('utf-8') if capture["response_headers_json"] else b"",
|
|
551
|
+
)
|
|
552
|
+
except Exception:
|
|
553
|
+
pass
|
|
554
|
+
|
|
555
|
+
# Clean capture state before closing to prevent memory leaks
|
|
556
|
+
# and ensure proper SSL shutdown (prevents TimeoutError: SSL shutdown timed out)
|
|
557
|
+
if hasattr(self, "_sf_req_capture"):
|
|
558
|
+
delattr(self, "_sf_req_capture")
|
|
559
|
+
if hasattr(self, "_sf_response_processed"):
|
|
560
|
+
delattr(self, "_sf_response_processed")
|
|
561
|
+
if hasattr(self, "_sf_response_body_buf"):
|
|
562
|
+
delattr(self, "_sf_response_body_buf")
|
|
563
|
+
if hasattr(self, "_sf_pending_capture"):
|
|
564
|
+
delattr(self, "_sf_pending_capture")
|
|
565
|
+
if hasattr(self, "_sf_capture_emitted"):
|
|
566
|
+
delattr(self, "_sf_capture_emitted")
|
|
567
|
+
|
|
568
|
+
# Call original close() to perform actual connection teardown
|
|
569
|
+
return original_close(self)
|
|
570
|
+
|
|
571
|
+
# ALWAYS patch request() for header injection (even with LD_PRELOAD or no C extension)
|
|
572
|
+
if HAS_WRAPT:
|
|
573
|
+
|
|
574
|
+
def instrumented_request(wrapped, instance, args, kwargs):
|
|
575
|
+
"""Ultra-fast header injection using wrapt."""
|
|
576
|
+
method = args[0] if len(args) > 0 else kwargs.get("method", "GET")
|
|
577
|
+
url = args[1] if len(args) > 1 else kwargs.get("url", "")
|
|
578
|
+
body = args[2] if len(args) > 2 else kwargs.get("body", None)
|
|
579
|
+
headers = args[3] if len(args) > 3 else kwargs.get("headers", None)
|
|
580
|
+
encode_chunked = kwargs.get("encode_chunked", False)
|
|
581
|
+
return patched_request(
|
|
582
|
+
instance, method, url, body, headers, encode_chunked=encode_chunked
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
wrapt.wrap_function_wrapper(_hc.HTTPConnection, "request", instrumented_request)
|
|
586
|
+
else:
|
|
587
|
+
_hc.HTTPConnection.request = patched_request
|
|
588
|
+
|
|
589
|
+
# ONLY patch send/getresponse if NOT using LD_PRELOAD AND C extension is available (for capture/emission)
|
|
590
|
+
if not preload_active and fast_available:
|
|
591
|
+
if HAS_WRAPT:
|
|
592
|
+
|
|
593
|
+
def instrumented_send(wrapped, instance, args, kwargs):
|
|
594
|
+
"""Ultra-fast send wrapper using wrapt."""
|
|
595
|
+
data = args[0] if len(args) > 0 else kwargs.get("data", b"")
|
|
596
|
+
return patched_send(instance, data)
|
|
597
|
+
|
|
598
|
+
def instrumented_getresponse(wrapped, instance, args, kwargs):
|
|
599
|
+
"""Ultra-fast getresponse wrapper using wrapt."""
|
|
600
|
+
return patched_getresponse(instance)
|
|
601
|
+
|
|
602
|
+
wrapt.wrap_function_wrapper(_hc.HTTPConnection, "send", instrumented_send)
|
|
603
|
+
wrapt.wrap_function_wrapper(
|
|
604
|
+
_hc.HTTPConnection, "getresponse", instrumented_getresponse
|
|
605
|
+
)
|
|
606
|
+
else:
|
|
607
|
+
_hc.HTTPConnection.send = patched_send
|
|
608
|
+
_hc.HTTPConnection.getresponse = patched_getresponse
|
|
609
|
+
|
|
610
|
+
if SF_DEBUG:
|
|
611
|
+
try:
|
|
612
|
+
print("[http_client] Patched send/getresponse for capture", log=False)
|
|
613
|
+
except TypeError:
|
|
614
|
+
print("[http_client] Patched send/getresponse for capture")
|
|
615
|
+
else:
|
|
616
|
+
reason = (
|
|
617
|
+
"LD_PRELOAD handles capture"
|
|
618
|
+
if preload_active
|
|
619
|
+
else "C extension not available"
|
|
620
|
+
)
|
|
621
|
+
if SF_DEBUG:
|
|
622
|
+
try:
|
|
623
|
+
print(
|
|
624
|
+
f"[http_client] Skipped send/getresponse patches ({reason})",
|
|
625
|
+
log=False,
|
|
626
|
+
)
|
|
627
|
+
except TypeError:
|
|
628
|
+
print(f"[http_client] Skipped send/getresponse patches ({reason})")
|
|
629
|
+
|
|
630
|
+
# ALWAYS patch close() to clean up capture state (critical for httplib2 connection pooling)
|
|
631
|
+
# This prevents SSL shutdown timeouts when connections are reused
|
|
632
|
+
if HAS_WRAPT:
|
|
633
|
+
|
|
634
|
+
def instrumented_close(wrapped, instance, args, kwargs):
|
|
635
|
+
"""Ultra-fast close wrapper using wrapt."""
|
|
636
|
+
return patched_close(instance)
|
|
637
|
+
|
|
638
|
+
wrapt.wrap_function_wrapper(_hc.HTTPConnection, "close", instrumented_close)
|
|
639
|
+
else:
|
|
640
|
+
_hc.HTTPConnection.close = patched_close
|
|
641
|
+
|
|
642
|
+
# ALWAYS patch HTTPResponse.read() to capture response body (httplib2 compatibility)
|
|
643
|
+
# This allows us to capture body as httplib2 reads it, avoiding peek() hang issue
|
|
644
|
+
if HAS_WRAPT:
|
|
645
|
+
|
|
646
|
+
def instrumented_response_read(wrapped, instance, args, kwargs):
|
|
647
|
+
"""Ultra-fast response read wrapper using wrapt."""
|
|
648
|
+
amt = args[0] if len(args) > 0 else kwargs.get("amt", None)
|
|
649
|
+
return patched_response_read(instance, amt)
|
|
650
|
+
|
|
651
|
+
wrapt.wrap_function_wrapper(
|
|
652
|
+
_hc.HTTPResponse, "read", instrumented_response_read
|
|
653
|
+
)
|
|
654
|
+
else:
|
|
655
|
+
_hc.HTTPResponse.read = patched_response_read
|
|
656
|
+
|
|
657
|
+
if SF_DEBUG:
|
|
658
|
+
try:
|
|
659
|
+
print(
|
|
660
|
+
"[http_client] Patched close() and HTTPResponse.read() for httplib2 compatibility",
|
|
661
|
+
log=False,
|
|
662
|
+
)
|
|
663
|
+
except TypeError:
|
|
664
|
+
print(
|
|
665
|
+
"[http_client] Patched close() and HTTPResponse.read() for httplib2 compatibility"
|
|
666
|
+
)
|
|
667
|
+
def _install_putheader_debug(_hc):
|
|
668
|
+
"""Instrument http.client putheader to trace Sailfish headers when SF_DEBUG is on."""
|
|
669
|
+
if not (SF_DEBUG and getattr(_hc.HTTPConnection.putheader, "_sf_debug_wrapped", False) is False):
|
|
670
|
+
return
|