sf-veritas 0.9.7__py3-none-any.whl

sf_veritas/patches/multiprocessing.py

@@ -0,0 +1,32 @@
+import multiprocessing
+
+from ..thread_local import get_context, set_context
+
+_original_process_init = multiprocessing.Process.__init__
+
+
+def patched_process_init(self, *args, **kwargs):
+    current_context = get_context()
+
+    original_target = kwargs.get("target")
+    if original_target:
+
+        def wrapped_target(*targs, **tkwargs):
+            set_context(current_context)
+            original_target(*targs, **tkwargs)
+
+        kwargs["target"] = wrapped_target
+    elif args and callable(args[0]):
+        original_target = args[0]
+
+        def wrapped_target(*targs, **tkwargs):
+            set_context(current_context)
+            original_target(*targs, **tkwargs)
+
+        args = (wrapped_target,) + args[1:]
+
+    _original_process_init(self, *args, **kwargs)
+
+
+def patch_multiprocessing():
+    multiprocessing.Process.__init__ = patched_process_init

sf_veritas/patches/network_libraries/__init__.py

@@ -0,0 +1,51 @@
+from typing import List, Optional
+
+from .aiohttp import patch_aiohttp
+from .curl_cffi import patch_curl_cffi
+from .http_client import patch_http_client
+from .httpcore import patch_httpcore
+from .httplib2 import patch_httplib2
+from .httpx import patch_httpx
+from .niquests import patch_niquests
+from .pycurl import patch_pycurl
+from .requests import patch_requests
+from .tornado import patch_tornado
+from .treq import patch_treq
+from .urllib_request import patch_urllib_request
+
+# from .aioh2            import patch_aioh2           # Asynchronous HTTP/2 client, no clear extension hooks
+# from .http_prompt      import patch_http_prompt     # CLI HTTP client, minimal public API
+# from .mureq            import patch_mureq           # Specialized crawler client, little documentation
+# from .reqboost         import patch_reqboost        # High-performance batch client, docs scarce
+# from .impit            import (patch_impit)         # Used by Crawlee's ImpitHttpClient
+# from .h11              import patch_h11             # Low-level HTTP/1.1 protocol library
+# from .aioquic          import patch_aioquic         # QUIC/HTTP-3 client, no standard headers API
+# from .qh3              import patch_qh3             # Experimental HTTP/3 client, no docs found
+
+
+def patch_all_http_clients(
+    domains_to_not_propagate_headers_to: Optional[List[str]] = None,
+):
+    # fully implemented patches
+    patch_requests(domains_to_not_propagate_headers_to)
+    patch_urllib_request(domains_to_not_propagate_headers_to)
+    patch_http_client(domains_to_not_propagate_headers_to)
+    patch_httplib2(domains_to_not_propagate_headers_to)
+    patch_pycurl(domains_to_not_propagate_headers_to)
+    patch_treq(domains_to_not_propagate_headers_to)
+    patch_httpx(domains_to_not_propagate_headers_to)
+    patch_aiohttp(domains_to_not_propagate_headers_to)
+    patch_tornado(domains_to_not_propagate_headers_to)
+    patch_curl_cffi(domains_to_not_propagate_headers_to)
+    patch_httpcore(domains_to_not_propagate_headers_to)
+    patch_niquests(domains_to_not_propagate_headers_to)
+
+    # # Lesser-used libraries
+    # patch_impit(domains_to_not_propagate_headers_to)
+    # patch_aioh2(domains_to_not_propagate_headers_to)
+    # patch_http_prompt(domains_to_not_propagate_headers_to)
+    # patch_mureq(domains_to_not_propagate_headers_to)
+    # patch_reqboost(domains_to_not_propagate_headers_to)
+    # patch_h11(domains_to_not_propagate_headers_to)
+    # patch_aioquic(domains_to_not_propagate_headers_to)
+    # patch_qh3(domains_to_not_propagate_headers_to)

sf_veritas/patches/network_libraries/aiohttp.py

@@ -0,0 +1,100 @@
+import time
+from typing import Any, List, Optional
+
+from ...constants import SAILFISH_TRACING_HEADER
+from .utils import get_trace_and_should_propagate, record_network_request
+
+
+def patch_aiohttp(domains_to_not_propagate_headers_to: Optional[List[str]] = None):
+    """
+    Monkey-patch aiohttp so that every HTTP verb:
+        1) injects SAILFISH_TRACING_HEADER when allowed,
+        2) measures timing,
+        3) calls NetworkRequestTransmitter().do_send via record_network_request.
+    """
+    try:
+        import aiohttp
+    except:
+        return
+
+    skip = domains_to_not_propagate_headers_to or []
+
+    # 1) Patch the core ClientSession._request coroutine
+    orig_request = aiohttp.ClientSession._request
+
+    async def patched_request(self, verb_name: str, url: Any, **kwargs):
+        trace_id, allow = get_trace_and_should_propagate(str(url), skip)
+        headers = kwargs.get("headers", {}) or {}
+        if allow:
+            headers[SAILFISH_TRACING_HEADER] = trace_id
+        kwargs["headers"] = headers
+
+        # 2) Perform & time the request
+        start = int(time.time() * 1_000)
+        response = await orig_request(self, verb_name, url, **kwargs)
+        end = int(time.time() * 1_000)
+
+        # 3) Record outcome
+        status = getattr(response, "status", 0)
+        ok = status < 400
+        error = None
+        if not ok:
+            try:
+                text = await response.text()
+                error = text[:255]
+            except Exception:
+                pass
+
+        record_network_request(
+            trace_id,
+            str(url),
+            verb_name.upper(),
+            status,
+            ok,
+            error,
+            timestamp_start=start,
+            timestamp_end=end,
+        )
+        return response
+
+    aiohttp.ClientSession._request = patched_request
+
+    # 2) Also patch the module-level aiohttp.request coroutine
+    orig_module_request = getattr(aiohttp, "request", None)
+    if orig_module_request:
+
+        async def patched_module_request(verb_name: str, url: str, **kwargs):
+            trace_id, allow = get_trace_and_should_propagate(str(url), skip)
+            headers = kwargs.get("headers", {}) or {}
+            if allow:
+                headers[SAILFISH_TRACING_HEADER] = trace_id
+            kwargs["headers"] = headers
+
+            start = int(time.time() * 1_000)
+            response = await orig_module_request(verb_name, url, **kwargs)
+            end = int(time.time() * 1_000)
+
+            status = getattr(response, "status", getattr(response, "status_code", 0))
+            ok = status < 400
+            error = None
+            if not ok:
+                try:
+                    body = await response.text()
+                    error = body[:255]
+                except Exception:
+                    pass
+
+            record_network_request(
+                trace_id,
+                str(url),
+                verb_name.upper(),
+                status,
+                ok,
+                error,
+                timestamp_start=start,
+                timestamp_end=end,
+            )
+
+            return response
+
+        aiohttp.request = patched_module_request

sf_veritas/patches/network_libraries/curl_cffi.py

@@ -0,0 +1,93 @@
+import time
+from typing import List, Optional
+
+from ...constants import SAILFISH_TRACING_HEADER
+from ..constants import supported_network_verbs as verbs
+from .utils import get_trace_and_should_propagate, record_network_request
+
+
+def patch_curl_cffi(domains_to_not_propagate_headers_to: Optional[List[str]] = None):
+    """
+    Monkey-patch curl_cffi.requests so that EVERY HTTP verb
+    injects SAILFISH_TRACING_HEADER (when allowed) and then records the request.
+    """
+    try:
+        import curl_cffi.requests as ccr
+    except ImportError:
+        return
+
+    skip = domains_to_not_propagate_headers_to or []
+
+    def make_wrapper(orig_fn, verb_name):
+        def wrapper(*args, **kwargs):
+            # 1) Determine HTTP method and URL safely
+            if verb_name == "request":
+                # support both request(url) and request(method, url, …)
+                if len(args) == 1 and isinstance(args[0], str):
+                    method, url = "GET", args[0]
+                elif len(args) >= 2 and isinstance(args[0], str):
+                    method, url = args[0].upper(), args[1]
+                elif len(args) >= 3:
+                    # bound Session.request(self, method, url, …)
+                    method, url = args[1].upper(), args[2]
+                else:
+                    method = kwargs.get("method", "").upper()
+                    url = kwargs.get("url", "")
+            else:
+                method = verb_name.upper()
+                # for module-level: args[0] == url
+                # for bound: args[1] == url
+                if len(args) >= 1 and isinstance(args[0], str):
+                    url = args[0]
+                elif len(args) >= 2:
+                    url = args[1]
+                else:
+                    url = kwargs.get("url", "")
+
+            # 2) Trace-id + skip-list check
+            trace_id, allow = get_trace_and_should_propagate(url, skip)
+            headers = kwargs.get("headers", {}) or {}
+            if allow:
+                headers[SAILFISH_TRACING_HEADER] = trace_id
+            kwargs["headers"] = headers
+
+            # 3) Perform the real call
+            start = int(time.time() * 1_000)
+            resp = orig_fn(*args, **kwargs)
+            end = int(time.time() * 1_000)
+
+            # 4) Record the network request
+            status = getattr(resp, "status_code", None) or getattr(resp, "status", 0)
+            ok = getattr(resp, "ok", status < 400)
+            error = None if ok else getattr(resp, "text", str(resp))[:255]
+
+            record_network_request(
+                trace_id,
+                url,
+                method,
+                status,
+                ok,
+                error,
+                timestamp_start=start,
+                timestamp_end=end,
+            )
+
+            return resp
+
+        return wrapper
+
+    # Patch module-level verbs
+    for verb in verbs:
+        orig = getattr(ccr, verb, None)
+        if orig:
+            setattr(ccr, verb, make_wrapper(orig, verb))
+
+    # Patch Session & AsyncSession methods
+    for cls_name in ("Session", "AsyncSession"):
+        cls = getattr(ccr, cls_name, None)
+        if not cls:
+            continue
+        for verb in verbs:
+            orig = getattr(cls, verb, None)
+            if orig:
+                setattr(cls, verb, make_wrapper(orig, verb))

sf_veritas/patches/network_libraries/http_client.py

@@ -0,0 +1,64 @@
+import time
+from typing import List, Optional
+
+from ...constants import SAILFISH_TRACING_HEADER
+from .utils import get_trace_and_should_propagate, record_network_request
+
+
+def patch_http_client(domains_to_not_propagate_headers_to: Optional[List[str]] = None):
+    # ensure we always have a list
+    if domains_to_not_propagate_headers_to is None:
+        domains_to_not_propagate_headers_to = []
+
+    try:
+        import http.client as _hc
+    except ImportError:
+        return
+
+    original_request = _hc.HTTPConnection.request
+
+    def patched_request(
+        self, method, url, body=None, headers=None, *, encode_chunked=False
+    ):
+        # timestamp for recording
+        start_ts = int(time.time() * 1_000)
+
+        # get the trace_id and check if we should propagate
+        trace_id, allow = get_trace_and_should_propagate(
+            url, domains_to_not_propagate_headers_to
+        )
+
+        # copy headers and inject only if allowed
+        headers = headers.copy() if headers else {}
+        if allow:
+            headers[SAILFISH_TRACING_HEADER] = trace_id
+
+        try:
+            # perform the real request
+            result = original_request(
+                self,
+                method,
+                url,
+                body=body,
+                headers=headers,
+                encode_chunked=encode_chunked,
+            )
+            # fire off our network-record GraphQL mutation
+            record_network_request(
+                trace_id, url, method, 0, True, timestamp_start=start_ts
+            )
+            return result
+        except Exception as e:
+            # record failures too
+            record_network_request(
+                trace_id,
+                url,
+                method,
+                0,
+                False,
+                error=str(e),
+                timestamp_start=start_ts,
+            )
+            raise
+
+    _hc.HTTPConnection.request = patched_request

sf_veritas/patches/network_libraries/httpcore.py

@@ -0,0 +1,152 @@
+import time
+from typing import List, Optional
+
+from ...constants import SAILFISH_TRACING_HEADER
+from .utils import get_trace_and_should_propagate, record_network_request
+
+
+def patch_httpcore(domains_to_not_propagate_headers_to: Optional[List[str]] = None):
+    """
+    Monkey-patch httpcore.ConnectionPool and AsyncConnectionPool
+    to inject SAILFISH_TRACING_HEADER (unless excluded)
+    and to record every outbound request.
+    """
+    try:
+        import httpcore
+    except ImportError:
+        return  # HTTP Core not present—skip patch
+
+    # Keep original methods
+    orig_sync_req = httpcore.ConnectionPool.request
+    orig_sync_stream = httpcore.ConnectionPool.stream
+    orig_async_req = httpcore.AsyncConnectionPool.request
+    orig_async_stream = httpcore.AsyncConnectionPool.stream
+
+    # Normalize exclude list
+    exclude = domains_to_not_propagate_headers_to or []
+
+    def _prepare_headers(url, existing_headers):
+        """
+        Returns (new_headers, trace_id).
+        Only injects if domain not in `exclude`.
+        """
+        trace_id, allow = get_trace_and_should_propagate(url, exclude)
+        if not allow:
+            return list(existing_headers or []), trace_id
+        hdrs = list(existing_headers or [])
+        hdrs.append((SAILFISH_TRACING_HEADER.encode(), trace_id.encode()))
+        return hdrs, trace_id
+
+    # 1. Sync .request(...)
+    def _patched_sync_request(self, method, url, **kwargs):
+        ts0 = int(time.time() * 1_000)
+        # prepare headers & trace
+        headers, trace_id = _prepare_headers(url, kwargs.get("headers"))
+        kwargs["headers"] = headers
+
+        error = None
+        try:
+            resp = orig_sync_req(self, method, url, **kwargs)
+            success = True
+            status = getattr(resp, "status_code", 0)
+            return resp
+        except Exception as e:
+            success = False
+            status = 0
+            error = str(e)[:255]
+            raise
+        finally:
+            ts1 = int(time.time() * 1_000)
+            record_network_request(
+                trace_id, url, method, status, success, error, ts0, ts1
+            )
+
+    # 2. Sync .stream(...)
+    def _patched_sync_stream(self, method, url, **kwargs):
+        ts0 = int(time.time() * 1_000)
+        headers, trace_id = _prepare_headers(url, kwargs.get("headers"))
+        kwargs["headers"] = headers
+
+        error = None
+        try:
+            stream = orig_sync_stream(self, method, url, **kwargs)
+            success = True
+            # stream itself yields the body; status often on returned object
+            status = 0
+            return stream
+        except Exception as e:
+            success = False
+            status = 0
+            error = str(e)[:255]
+            raise
+        finally:
+            ts1 = int(time.time() * 1_000)
+            record_network_request(
+                trace_id, url, method, status, success, error, ts0, ts1
+            )
+
+    # 3. Async .request(...)
+    async def _patched_async_request(self, method, url, **kwargs):
+        ts0 = int(time.time() * 1_000)
+        headers, trace_id = _prepare_headers(url, kwargs.get("headers"))
+        kwargs["headers"] = headers
+
+        error = None
+        try:
+            resp = await orig_async_req(self, method, url, **kwargs)
+            success = True
+            status = getattr(resp, "status_code", 0)
+            return resp
+        except Exception as e:
+            success = False
+            status = 0
+            error = str(e)[:255]
+            raise
+        finally:
+            ts1 = int(time.time() * 1_000)
+            record_network_request(
+                trace_id, url, method, status, success, error, ts0, ts1
+            )
+
+    # 4. Async .stream(...)
+    def _patched_async_stream(self, method, url, **kwargs):
+        ts0 = int(time.time() * 1_000)
+        headers, trace_id = _prepare_headers(url, kwargs.get("headers"))
+        kwargs["headers"] = headers
+        original_cm = orig_async_stream(self, method, url, **kwargs)
+
+        class _StreamCM:
+            def __init__(self, cm):
+                self._cm = cm
+                self._status = 0
+
+            async def __aenter__(self):
+                response = await self._cm.__aenter__()  # now a single Response
+                # capture status (httpcore.Response.status or status_code)
+                self._status = getattr(
+                    response, "status_code", getattr(response, "status", 0)
+                )
+                return response
+
+            async def __aexit__(self, exc_type, exc, tb):
+                success = exc_type is None
+                ts1 = int(time.time() * 1_000)
+                record_network_request(
+                    trace_id,
+                    url,
+                    method,
+                    self._status,
+                    success,
+                    None if success else str(exc)[:255],
+                    ts0,
+                    ts1,
+                )
+                return await self._cm.__aexit__(exc_type, exc, tb)
+
+        return _StreamCM(original_cm)
+
+    # Apply patches
+    httpcore.ConnectionPool.request = _patched_sync_request
+    httpcore.ConnectionPool.stream = _patched_sync_stream
+    httpcore.AsyncConnectionPool.request = _patched_async_request
+    httpcore.AsyncConnectionPool.stream = _patched_async_stream

sf_veritas/patches/network_libraries/httplib2.py

@@ -0,0 +1,76 @@
+import time
+from typing import List, Optional
+
+from ...constants import SAILFISH_TRACING_HEADER
+from .utils import get_trace_and_should_propagate, record_network_request
+
+
+def patch_httplib2(domains_to_not_propagate_headers_to: Optional[List[str]] = None):
+    """
+    Monkey-patch httplib2.Http.request so that:
+    1. We skip header injection for configured domains.
+    2. We call NetworkRequestTransmitter().do_send via record_network_request().
+    3. All HTTP methods (GET, POST, etc.) continue to work as before.
+    """
+    try:
+        import httplib2
+    except ImportError:
+        return
+
+    # default to an empty blocklist
+    if domains_to_not_propagate_headers_to is None:
+        domains_to_not_propagate_headers_to = []
+
+    original_request = httplib2.Http.request
+
+    def patched_request(self, uri, method="GET", body=None, headers=None, **kwargs):
+        start_ts = int(time.time() * 1_000)
+        # decide whether to inject header
+        trace_id, allow = get_trace_and_should_propagate(
+            uri, domains_to_not_propagate_headers_to
+        )
+        # prepare headers
+        headers = headers.copy() if headers else {}
+        if allow:
+            headers[SAILFISH_TRACING_HEADER] = trace_id
+
+        try:
+            # perform the actual HTTP call
+            response, content = original_request(
+                self, uri, method, body=body, headers=headers, **kwargs
+            )
+            status_code = getattr(response, "status", None) or getattr(
+                response, "status_code", None
+            )
+            success = isinstance(status_code, int) and 200 <= status_code < 400
+            return response, content
+
+        except Exception as e:
+            # record failures
+            record_network_request(
+                trace_id,
+                uri,
+                method,
+                0,
+                False,
+                error=str(e)[:255],
+                timestamp_start=start_ts,
+                timestamp_end=int(time.time() * 1_000),
+            )
+            raise
+
+        finally:
+            # record successes
+            if "status_code" in locals():
+                record_network_request(
+                    trace_id,
+                    uri,
+                    method,
+                    status_code,
+                    success,
+                    timestamp_start=start_ts,
+                    timestamp_end=int(time.time() * 1_000),
+                )
+
+    # apply our patch
+    httplib2.Http.request = patched_request

sf_veritas/patches/network_libraries/httpx.py

@@ -0,0 +1,123 @@
+from typing import List, Optional
+
+from ...constants import SAILFISH_TRACING_HEADER
+from .utils import get_trace_and_should_propagate, record_network_request
+
+
+def patch_httpx(domains_to_not_propagate_headers_to: Optional[List[str]] = None):
+    """
+    Monkey-patch HTTPX to inject SAILFISH_TRACING_HEADER into
+    all outbound requests (sync + async + streams), skipping any
+    domains in domains_to_not_propagate_headers_to, and recording each.
+    Safe to call even if HTTPX is not installed.
+    """
+    try:
+        import httpx
+    except ImportError:
+        return  # No HTTPX installed—nothing to patch
+
+    domains = domains_to_not_propagate_headers_to or []
+
+    # Preserve originals
+    orig_request = httpx.request
+    orig_client_request = httpx.Client.request
+    orig_async_request = httpx.AsyncClient.request
+    orig_stream = httpx.stream
+    orig_client_stream = httpx.Client.stream
+    orig_async_client_stream = httpx.AsyncClient.stream
+
+    # Shared header + record prep
+    def _prepare(method: str, url: str, headers: Optional[dict]):
+        trace_id, allow = get_trace_and_should_propagate(url, domains)
+        hdrs = dict(headers or {})
+        if allow:
+            hdrs[SAILFISH_TRACING_HEADER] = trace_id
+        return trace_id, hdrs
+
+    # 1) Module-level request
+    def _patched_request(method, url, *args, headers=None, **kwargs):
+        trace_id, hdrs = _prepare(method, str(url), headers)
+        resp = orig_request(method, url, *args, headers=hdrs, **kwargs)
+        record_network_request(trace_id, str(url), method, resp.status_code, resp.is_success)
+        return resp
+
+    # 2) Sync Client.request
+    def _patched_client_request(self, method, url, *args, headers=None, **kwargs):
+        trace_id, hdrs = _prepare(method, str(url), headers)
+        resp = orig_client_request(self, method, url, *args, headers=hdrs, **kwargs)
+        record_network_request(trace_id, str(url), method, resp.status_code, resp.is_success)
+        return resp
+
+    # 3) AsyncClient.request
+    async def _patched_async_request(self, method, url, *args, headers=None, **kwargs):
+        trace_id, hdrs = _prepare(method, str(url), headers)
+        resp = await orig_async_request(
+            self, method, url, *args, headers=hdrs, **kwargs
+        )
+        record_network_request(trace_id, str(url), method, resp.status_code, resp.is_success)
+        return resp
+
+    # 4a) Module-level streaming
+    def _patched_stream(method, url, *args, headers=None, **kwargs):
+        trace_id, hdrs = _prepare(method, str(url), headers)
+        cm = orig_stream(method, url, *args, headers=hdrs, **kwargs)
+
+        class StreamCM:
+            def __enter__(self):
+                resp = cm.__enter__()
+                resp.read()  # ensure .content
+                record_network_request(
+                    trace_id, url, method, resp.status_code, resp.is_success
+                )
+                return resp
+
+            def __exit__(self, exc_type, exc, tb):
+                return cm.__exit__(exc_type, exc, tb)
+
+        return StreamCM()
+
+    # 4b) Sync Client.stream()
+    def _patched_client_stream(self, method, url, *args, headers=None, **kwargs):
+        trace_id, hdrs = _prepare(method, str(url), headers)
+        cm = orig_client_stream(self, method, url, *args, headers=hdrs, **kwargs)
+
+        class ClientStreamCM:
+            def __enter__(self):
+                resp = cm.__enter__()
+                resp.read()
+                record_network_request(
+                    trace_id, str(url), method, resp.status_code, resp.is_success
+                )
+                return resp
+
+            def __exit__(self, exc_type, exc, tb):
+                return cm.__exit__(exc_type, exc, tb)
+
+        return ClientStreamCM()
+
+    # 4c) AsyncClient.stream()
+    def _patched_async_client_stream(self, method, url, *args, headers=None, **kwargs):
+        trace_id, hdrs = _prepare(method, str(url), headers)
+        cm = orig_async_client_stream(self, method, url, *args, headers=hdrs, **kwargs)
+
+        class AsyncClientStreamCM:
+            async def __aenter__(self):
+                resp = await cm.__aenter__()
+                await resp.aread()  # ensure .content
+                record_network_request(
+                    trace_id, str(url), method, resp.status_code, resp.is_success
+                )
+                return resp
+
+            async def __aexit__(self, exc_type, exc, tb):
+                return await cm.__aexit__(exc_type, exc, tb)
+
+        return AsyncClientStreamCM()
+
+    # Apply monkey-patches
+    httpx.request = _patched_request
+    httpx.Client.request = _patched_client_request
+    httpx.AsyncClient.request = _patched_async_request
+    httpx.stream = _patched_stream
+    httpx.Client.stream = _patched_client_stream
+    httpx.AsyncClient.stream = _patched_async_client_stream