sf-veritas 0.11.10__cp314-cp314-manylinux_2_28_x86_64.whl

sf_veritas/patches/_patch_tracker.py

@@ -0,0 +1,74 @@
+"""
+Central tracking for patch application to prevent double-patching.
+
+This module maintains a registry of applied patches keyed by (patch_name, pid)
+to handle worker forks (gunicorn/uvicorn) and module reloading correctly.
+"""
+import os
+import threading
+
+# Global registry: {(patch_name, pid): True}
+_PATCHED = {}
+_lock = threading.Lock()
+
+
+def is_already_patched(patch_name: str) -> bool:
+    """
+    Check if a patch has already been applied in this process.
+
+    Args:
+        patch_name: Unique identifier for the patch (e.g., "requests", "httpx")
+
+    Returns:
+        True if already patched in current PID, False otherwise
+    """
+    key = (patch_name, os.getpid())
+    with _lock:
+        return key in _PATCHED
+
+
+def mark_as_patched(patch_name: str) -> bool:
+    """
+    Mark a patch as applied in the current process.
+
+    Args:
+        patch_name: Unique identifier for the patch
+
+    Returns:
+        True if this is the first application, False if already patched
+    """
+    key = (patch_name, os.getpid())
+    with _lock:
+        if key in _PATCHED:
+            return False
+        _PATCHED[key] = True
+        return True
+
+
+def once(patch_name: str):
+    """
+    Decorator/guard for ensuring a patch runs only once per process.
+
+    Usage:
+        @once("requests")
+        def patch_requests(...):
+            ...
+
+    Or as a guard:
+        if not once.check("requests"):
+            return  # Already patched
+    """
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            if is_already_patched(patch_name):
+                return  # Already patched, skip
+            mark_as_patched(patch_name)
+            return func(*args, **kwargs)
+        wrapper.__name__ = func.__name__
+        wrapper.__doc__ = func.__doc__
+        return wrapper
+    return decorator
+
+
+# Convenience method for checking
+once.check = lambda name: mark_as_patched(name)

sf_veritas/patches/concurrent_futures.py

@@ -0,0 +1,19 @@
+from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
+from ..thread_local import get_context, set_context
+
+_original_submit = ThreadPoolExecutor.submit
+
+
+def patched_submit(self, fn, *args, **kwargs):
+    current_context = get_context()
+
+    def wrapped_fn(*fn_args, **fn_kwargs):
+        set_context(current_context)
+        fn(*fn_args, **fn_kwargs)
+
+    return _original_submit(self, wrapped_fn, *args, **kwargs)
+
+
+def patch_concurrent_futures():
+    ThreadPoolExecutor.submit = patched_submit
+    ProcessPoolExecutor.submit = patched_submit

sf_veritas/patches/constants.py

@@ -0,0 +1 @@
+supported_network_verbs = ("get", "post", "put", "patch", "delete", "head", "options")

sf_veritas/patches/exceptions.py

@@ -0,0 +1,82 @@
+import importlib.util
+import sys
+import threading
+import time
+from importlib import abc
+
+from ..env_vars import PRINT_CONFIGURATION_STATUSES
+
+# Thread-local storage to avoid re-entry problems
+patch_lock = threading.local()
+
+
+def patch_exceptions(module):
+    if hasattr(patch_lock, "active"):
+        return
+    patch_lock.active = True
+
+
+class ExceptionPatchingFinder(abc.MetaPathFinder):
+    def find_spec(self, fullname, path, target=None):
+        if hasattr(patch_lock, "loading") and patch_lock.loading:
+            return None
+        try:
+            patch_lock.loading = True
+            original_spec = importlib.util.find_spec(fullname, path)
+            if original_spec:
+                return importlib.util.spec_from_loader(
+                    fullname,
+                    ExceptionPatchingLoader(original_spec.loader),
+                    origin=original_spec.origin,
+                )
+            return None
+        finally:
+            patch_lock.loading = False
+
+
+class ExceptionPatchingLoader(abc.Loader):
+    def __init__(self, loader):
+        self._original_loader = loader
+
+    def create_module(self, spec):
+        return self._original_loader.create_module(spec)
+
+    def exec_module(self, module):
+        self._original_loader.exec_module(module)
+        patch_exceptions(module)
+
+
+def install_import_hook():
+    if PRINT_CONFIGURATION_STATUSES:
+        print("EXCEPTIONS - install_import_hook", log=False)
+    sys.meta_path.insert(0, ExceptionPatchingFinder())
+    if PRINT_CONFIGURATION_STATUSES:
+        print("EXCEPTIONS - install_import_hook...DONE", log=False)
+
+
+# Initially store the current state of sys.excepthook
+original_excepthook = sys.excepthook
+
+
+def monitor_excepthook(interval=1):
+    global original_excepthook
+
+    while True:
+        current_hook = sys.excepthook
+        if current_hook != original_excepthook and PRINT_CONFIGURATION_STATUSES:
+            if PRINT_CONFIGURATION_STATUSES:
+                print("sys.excepthook has been modified!")
+            original_excepthook = current_hook
+            continue
+        if PRINT_CONFIGURATION_STATUSES:
+            print("No change detected in sys.excepthook.")
+
+        # Pause for the specified interval before the next check
+        time.sleep(interval)
+
+
+# Function to start monitoring in a separate thread
+def start_monitoring(interval=2):
+    thread = threading.Thread(target=monitor_excepthook, args=(interval,))
+    # thread.daemon = True  # This makes the thread exit when the main program exits
+    thread.start()

sf_veritas/patches/multiprocessing.py

@@ -0,0 +1,32 @@
+import multiprocessing
+
+from ..thread_local import get_context, set_context
+
+_original_process_init = multiprocessing.Process.__init__
+
+
+def patched_process_init(self, *args, **kwargs):
+    current_context = get_context()
+
+    original_target = kwargs.get("target")
+    if original_target:
+
+        def wrapped_target(*targs, **tkwargs):
+            set_context(current_context)
+            original_target(*targs, **tkwargs)
+
+        kwargs["target"] = wrapped_target
+    elif args and callable(args[0]):
+        original_target = args[0]
+
+        def wrapped_target(*targs, **tkwargs):
+            set_context(current_context)
+            original_target(*targs, **tkwargs)
+
+        args = (wrapped_target,) + args[1:]
+
+    _original_process_init(self, *args, **kwargs)
+
+
+def patch_multiprocessing():
+    multiprocessing.Process.__init__ = patched_process_init

sf_veritas/patches/network_libraries/__init__.py

@@ -0,0 +1,99 @@
+import os
+from typing import List, Optional
+
+from ...request_interceptor import get_domains_to_not_propagate_headers_to
+from ...env_vars import SF_DEBUG
+from .utils import init_fast_network_tracking
+from .ssl_socket import patch_ssl_sockets  # CRITICAL: Import SSL patching
+from .requests import patch_requests
+from .aiohttp import patch_aiohttp
+from .httpx import patch_httpx
+from .httpcore import patch_httpcore
+from .http_client import patch_http_client
+from .urllib_request import patch_urllib_request
+from .httplib2 import patch_httplib2
+from .pycurl import patch_pycurl
+from .niquests import patch_niquests
+from .curl_cffi import patch_curl_cffi
+from .tornado import patch_tornado
+from .treq import patch_treq
+
+# from .aioh2            import patch_aioh2           # Asynchronous HTTP/2 client, no clear extension hooks
+# from .http_prompt      import patch_http_prompt     # CLI HTTP client, minimal public API
+# from .mureq            import patch_mureq           # Specialized crawler client, little documentation
+# from .reqboost         import patch_reqboost        # High-performance batch client, docs scarce
+# from .impit            import (patch_impit)         # Used by Crawlee's ImpitHttpClient
+# from .h11              import patch_h11             # Low-level HTTP/1.1 protocol library
+# from .aioquic          import patch_aioquic         # QUIC/HTTP-3 client, no standard headers API
+# from .qh3              import patch_qh3             # Experimental HTTP/3 client, no docs found
+
+
+def patch_all_http_clients(
+    domains_to_not_propagate_headers_to: Optional[List[str]] = None,
+):
+    # ====================================================================
+    # CRITICAL: PATCH SSL FIRST - This captures ALL HTTPS traffic
+    # ====================================================================
+    # All HTTP libraries (requests, httpx, urllib3, aiohttp, http.client)
+    # use ssl.SSLSocket underneath. By patching at the SSL layer first,
+    # we automatically capture all HTTPS traffic with ~15-20ns overhead.
+    #
+    # This also avoids race conditions from C-level socket hooks.
+    # ====================================================================
+    if SF_DEBUG:
+        print(f"[patches] Calling patch_ssl_sockets() - SF_ENABLE_PYTHON_SSL_TEE={os.getenv('SF_ENABLE_PYTHON_SSL_TEE', 'NOT_SET')}", log=False)
+    try:
+        patch_ssl_sockets()
+        if SF_DEBUG:
+            print("[patches] ✓ SSL socket patching complete - all HTTPS captured automatically", log=False)
+    except Exception as e:
+        if SF_DEBUG:
+            print(f"[patches] WARNING: SSL patching failed: {e}", log=False)
+            import traceback
+            traceback.print_exc()
+
+    # Enable Python-level header injection (ULTRA-FAST: <100ns)
+    # This disables C-level header injection (291µs overhead) and lets Python patches handle it
+    os.environ["SF_PYTHON_HEADER_INJECTION"] = "0"
+
+    # Initialize fast C-based network tracking
+    # NOTE: When LD_PRELOAD is active (_sfteepreload), this only initializes the Python senders
+    # The actual socket capture is done by _sfteepreload automatically
+    init_fast_network_tracking()
+
+    # Send domains mutation ONCE before patching (not from within each patch function)
+    if domains_to_not_propagate_headers_to:
+        domains_to_not_propagate_headers_to = get_domains_to_not_propagate_headers_to(
+            domains_to_not_propagate_headers_to
+        )
+
+    # ULTRA-FAST Python-level header injection (<100ns)
+    # Enable core libraries that cover 95% of use cases:
+
+    # requests covers: requests → urllib3 → http.client (entire stack!)
+    patch_requests(domains_to_not_propagate_headers_to)
+
+    # aiohttp for async HTTP (standalone stack)
+    patch_aiohttp(domains_to_not_propagate_headers_to)
+
+    # Additional libraries (enable if needed):
+    patch_http_client(domains_to_not_propagate_headers_to)  # Covered by requests
+    patch_urllib_request(domains_to_not_propagate_headers_to)  # Covered by requests
+    patch_httplib2(domains_to_not_propagate_headers_to)
+    patch_httpx(domains_to_not_propagate_headers_to)
+    patch_httpcore(domains_to_not_propagate_headers_to)
+    patch_pycurl(domains_to_not_propagate_headers_to)
+    patch_treq(domains_to_not_propagate_headers_to)
+    patch_tornado(domains_to_not_propagate_headers_to)
+    patch_curl_cffi(domains_to_not_propagate_headers_to)
+    patch_niquests(domains_to_not_propagate_headers_to)
+
+    # # Lesser-used libraries
+    # patch_impit(domains_to_not_propagate_headers_to)
+    # patch_aioh2(domains_to_not_propagate_headers_to)
+    # patch_http_prompt(domains_to_not_propagate_headers_to)
+    # patch_mureq(domains_to_not_propagate_headers_to)
+    # patch_reqboost(domains_to_not_propagate_headers_to)
+    # patch_h11(domains_to_not_propagate_headers_to)
+    # patch_aioquic(domains_to_not_propagate_headers_to)
+    # patch_qh3(domains_to_not_propagate_headers_to)

sf_veritas/patches/network_libraries/aiohttp.py

@@ -0,0 +1,294 @@
+import os
+import time
+from typing import Any, List, Optional
+
+try:
+    import wrapt
+
+    HAS_WRAPT = True
+except ImportError:
+    HAS_WRAPT = False
+
+from ...constants import FUNCSPAN_OVERRIDE_HEADER, SAILFISH_TRACING_HEADER
+from ...thread_local import trace_id_ctx
+from .utils import (
+    init_fast_header_check,
+    inject_headers_ultrafast,
+    is_ssl_socket_active,
+    record_network_request,
+)
+
+# JSON serialization - try fast orjson first, fallback to stdlib json
+try:
+    import orjson
+
+    HAS_ORJSON = True
+except ImportError:
+    import json
+
+    HAS_ORJSON = False
+
+
+def _tee_preload_active() -> bool:
+    """Detect if LD_PRELOAD tee is active (same logic as http_client.py)."""
+    if os.getenv("SF_TEE_PRELOAD_ONLY", "0") == "1":
+        return True
+    ld = os.getenv("LD_PRELOAD", "")
+    return "libsfnettee.so" in ld or "_sfteepreload" in ld
+
+
+def patch_aiohttp(domains_to_not_propagate_headers_to: Optional[List[str]] = None):
+    """
+    Monkey-patch aiohttp so that every HTTP verb:
+        1) injects SAILFISH_TRACING_HEADER + FUNCSPAN_OVERRIDE_HEADER when allowed,
+        2) measures timing (only when LD_PRELOAD not active),
+        3) calls NetworkRequestTransmitter().do_send via record_network_request (UNLESS LD_PRELOAD active).
+
+    When LD_PRELOAD is active: ULTRA-FAST path using TraceConfig with wrapt (OTEL-style, minimal overhead).
+    When LD_PRELOAD is NOT active: Full capture path with body/header recording.
+    """
+    try:
+        import aiohttp
+    except:
+        return
+
+    skip = domains_to_not_propagate_headers_to or []
+    preload_active = _tee_preload_active()
+
+    # Initialize C extension for ultra-fast header checking (if available)
+    if preload_active:
+        init_fast_header_check(skip)
+
+    if preload_active:
+        # ========== ULTRA-FAST PATH: Direct wrapt on _request (bypass TraceConfig overhead!) ==========
+        # TraceConfig adds 15-20% overhead, so we patch _request directly like OTEL does for other libraries
+
+        if HAS_WRAPT:
+            # FASTEST: Use wrapt directly on _request method (OTEL-style for minimal overhead)
+            async def instrumented_request(wrapped, instance, args, kwargs):
+                """
+                Ultra-fast header injection using thread-local cache.
+                Bypasses TraceConfig machinery for <5% overhead.
+                """
+                # Extract verb and URL from args
+                verb_name = args[0] if args else kwargs.get("method", "GET")
+                url = str(args[1] if len(args) > 1 else kwargs.get("url", ""))
+
+                # Direct header mutation (no copy!)
+                headers = kwargs.get("headers")
+                if headers is None:
+                    headers = {}
+                    kwargs["headers"] = headers
+
+                # CRITICAL: Skip if already injected (prevents double injection)
+                if SAILFISH_TRACING_HEADER not in headers:
+                    # ULTRA-FAST: Thread-local cache + direct ContextVar.get() (<100ns!)
+                    inject_headers_ultrafast(headers, url, skip)
+
+                # NO timing, NO capture, NO threads - immediate return!
+                return await wrapped(*args, **kwargs)
+
+            wrapt.wrap_function_wrapper(
+                aiohttp.ClientSession, "_request", instrumented_request
+            )
+
+        else:
+            # Fallback: Direct patching if wrapt not available
+            orig_request = aiohttp.ClientSession._request
+
+            async def patched_request(self, verb_name: str, url: Any, **kwargs):
+                """Ultra-fast header injection without wrapt."""
+                headers = kwargs.get("headers")
+                if headers is None:
+                    headers = {}
+                    kwargs["headers"] = headers
+
+                # CRITICAL: Skip if already injected (prevents double injection)
+                if SAILFISH_TRACING_HEADER not in headers:
+                    # ULTRA-FAST: Thread-local cache + direct ContextVar.get() (<100ns!)
+                    inject_headers_ultrafast(headers, str(url), skip)
+
+                # NO timing, NO capture - immediate return!
+                return await orig_request(self, verb_name, url, **kwargs)
+
+            aiohttp.ClientSession._request = patched_request
+
+    else:
+        # ========== FULL CAPTURE PATH: When LD_PRELOAD is NOT active ==========
+        # Uses wrapper function to capture request/response data
+        orig_request = aiohttp.ClientSession._request
+
+        async def patched_request(self, verb_name: str, url: Any, **kwargs):
+            headers = kwargs.get("headers", {}) or {}
+            if not isinstance(headers, dict):
+                headers = dict(headers)
+
+            # CRITICAL: Skip if already injected (prevents double injection)
+            if SAILFISH_TRACING_HEADER not in headers:
+                # ULTRA-FAST: Thread-local cache + direct ContextVar.get() (<100ns!)
+                inject_headers_ultrafast(headers, str(url), skip)
+
+            kwargs["headers"] = headers
+
+            # Get trace_id for network recording (after injection)
+            trace_id = trace_id_ctx.get(None) or ""
+
+            # SLOW PATH: LD_PRELOAD not active, do full Python-level capture
+            # Capture request data as bytes - BEFORE request
+            req_data = b""
+            req_headers = b""
+            try:
+                if "json" in kwargs:
+                    if HAS_ORJSON:
+                        req_data = orjson.dumps(kwargs["json"])
+                    else:
+                        req_data = json.dumps(kwargs["json"]).encode("utf-8")
+                elif "data" in kwargs:
+                    data = kwargs["data"]
+                    if isinstance(data, bytes):
+                        req_data = data
+                    elif isinstance(data, str):
+                        req_data = data.encode("utf-8")
+
+                # Capture request headers
+                if HAS_ORJSON:
+                    # Convert keys to str (aiohttp may use istr which orjson doesn't accept)
+                    req_headers = orjson.dumps({str(k): str(v) for k, v in headers.items()})
+                else:
+                    req_headers = json.dumps({str(k): str(v) for k, v in headers.items()}).encode("utf-8")
+            except Exception:  # noqa: BLE001
+                pass
+
+            # 2) Perform & time the request
+            start = int(time.time() * 1_000)
+            response = await orig_request(self, verb_name, url, **kwargs)
+            end = int(time.time() * 1_000)
+
+            # Skip capture for HTTPS when ssl_socket.py is active (avoids double-capture)
+            url_str = str(url)
+            is_https = url_str.startswith("https://")
+            if is_https and is_ssl_socket_active():
+                return response
+
+            # 3) Capture response metadata immediately (before response can be closed)
+            status = getattr(response, "status", 0)
+            ok = status < 400
+
+            # Capture response headers immediately (cheap and safe)
+            resp_headers = b""
+            if HAS_ORJSON:
+                # Convert keys to str (aiohttp uses istr which orjson doesn't accept)
+                resp_headers = orjson.dumps({str(k): str(v) for k, v in response.headers.items()})
+            else:
+                resp_headers = json.dumps({str(k): str(v) for k, v in response.headers.items()}).encode("utf-8")
+
+            # Send to C extension immediately - no background task needed!
+            record_network_request(
+                trace_id,
+                str(url),
+                verb_name.upper(),
+                status,
+                ok,
+                None,
+                timestamp_start=start,
+                timestamp_end=end,
+                request_data=req_data,
+                response_data=b"",  # Skip body to avoid consuming stream
+                request_headers=req_headers,
+                response_headers=resp_headers,
+            )
+
+            # CRITICAL: Return response immediately!
+            return response
+
+        # Apply the wrapper to ClientSession._request
+        aiohttp.ClientSession._request = patched_request
+
+        # 2) Also patch the module-level aiohttp.request coroutine (for full-capture path)
+        orig_module_request = getattr(aiohttp, "request", None)
+        if orig_module_request:
+
+            async def patched_module_request(verb_name: str, url: str, **kwargs):
+                headers = kwargs.get("headers", {}) or {}
+                if not isinstance(headers, dict):
+                    headers = dict(headers)
+
+                # CRITICAL: Skip if already injected (prevents double injection)
+                if SAILFISH_TRACING_HEADER not in headers:
+                    # ULTRA-FAST: Thread-local cache + direct ContextVar.get() (<100ns!)
+                    inject_headers_ultrafast(headers, str(url), skip)
+
+                kwargs["headers"] = headers
+
+                # Get trace_id for network recording (after injection)
+                trace_id = trace_id_ctx.get(None) or ""
+
+                # SLOW PATH: LD_PRELOAD not active, do full Python-level capture
+                # Capture request data as bytes - BEFORE request
+                req_data = b""
+                req_headers = b""
+                try:
+                    if "json" in kwargs:
+                        if HAS_ORJSON:
+                            req_data = orjson.dumps(kwargs["json"])
+                        else:
+                            req_data = json.dumps(kwargs["json"]).encode("utf-8")
+                    elif "data" in kwargs:
+                        data = kwargs["data"]
+                        if isinstance(data, bytes):
+                            req_data = data
+                        elif isinstance(data, str):
+                            req_data = data.encode("utf-8")
+
+                    # Capture request headers
+                    if HAS_ORJSON:
+                        # Convert keys to str (aiohttp may use istr which orjson doesn't accept)
+                        req_headers = orjson.dumps({str(k): str(v) for k, v in headers.items()})
+                    else:
+                        req_headers = json.dumps({str(k): str(v) for k, v in headers.items()}).encode("utf-8")
+                except Exception:  # noqa: BLE001
+                    pass
+
+                start = int(time.time() * 1_000)
+                response = await orig_module_request(verb_name, url, **kwargs)
+                end = int(time.time() * 1_000)
+
+                # Skip capture for HTTPS when ssl_socket.py is active (avoids double-capture)
+                url_str = str(url)
+                is_https = url_str.startswith("https://")
+                if is_https and is_ssl_socket_active():
+                    return response
+
+                status = getattr(
+                    response, "status", getattr(response, "status_code", 0)
+                )
+                ok = status < 400
+
+                # Capture response headers immediately (cheap and safe)
+                resp_headers = b""
+                if HAS_ORJSON:
+                    resp_headers = orjson.dumps(dict(response.headers))
+                else:
+                    resp_headers = json.dumps(dict(response.headers)).encode("utf-8")
+
+                # Send to C extension immediately - no background task needed!
+                record_network_request(
+                    trace_id,
+                    str(url),
+                    verb_name.upper(),
+                    status,
+                    ok,
+                    None,
+                    timestamp_start=start,
+                    timestamp_end=end,
+                    request_data=req_data,
+                    response_data=b"",  # Skip body to avoid consuming stream
+                    request_headers=req_headers,
+                    response_headers=resp_headers,
+                )
+
+                # CRITICAL: Return response immediately!
+                return response
+
+            # Apply the wrapper to module-level request function
+            aiohttp.request = patched_module_request