sf-veritas 0.9.7__py3-none-any.whl

sf_veritas/.gitignore

@@ -0,0 +1,2 @@
+not_ready
+bootstrap_internal_local.py

sf_veritas/__init__.py

@@ -0,0 +1,4 @@
+from .transmit_exception_to_sailfish import transmit_exception_to_sailfish
+from .unified_interceptor import setup_interceptors
+
+__all__ = ["setup_interceptors", "transmit_exception_to_sailfish"]

sf_veritas/app_config.py

@@ -0,0 +1,49 @@
+import os
+import uuid
+from typing import Dict, List, Optional, Union
+
+from .constants import SAILFISH_DEFAULT_GRAPHQL_ENDPOINT
+from .infra_details import get_infra_details
+
+_service_uuid = str(uuid.uuid4())
+_service_identifier = None
+_service_version = None
+_git_sha = None
+_service_identification_received = False
+_service_additional_metadata: Optional[Dict[str, Union[str, int, float, None]]] = None
+
+_setup_interceptors_call_filename = None
+_setup_interceptors_call_lineno = None
+
+_interceptors_initialized = False
+
+_profiling_mode_enabled: bool = False
+_profiling_max_depth: int = 5
+
+_site_and_dist_packages_to_collect_local_variables_on: Optional[List[str]] = []
+
+_sailfish_api_key = None
+_sailfish_graphql_endpoint: str = os.getenv(
+    "SAILFISH_GRAPHQL_ENDPOINT", SAILFISH_DEFAULT_GRAPHQL_ENDPOINT
+)
+
+_infra_details = get_infra_details()
+
+
+def _set_site_and_dist_packages_to_collect_local_variables_on(
+    site_and_dist_packages_to_collect_local_variables_on: Optional[List[str]] = None,
+):
+    if site_and_dist_packages_to_collect_local_variables_on is None:
+        _site_and_dist_packages_to_collect_local_variables_on = []
+    site_and_dist_packages_to_collect_local_variables_on_final: List[str] = (
+        site_and_dist_packages_to_collect_local_variables_on.copy()
+    )
+    for package in site_and_dist_packages_to_collect_local_variables_on:
+        if "-" not in package:
+            continue
+        site_and_dist_packages_to_collect_local_variables_on_final.append(
+            package.replace("-", "_")
+        )
+    _site_and_dist_packages_to_collect_local_variables_on = (
+        site_and_dist_packages_to_collect_local_variables_on_final
+    )

sf_veritas/cli.py

@@ -0,0 +1,336 @@
+import os
+import subprocess
+import sys
+import tempfile
+from typing import Dict, List, Optional, Union
+
+from fire import Fire
+
+from .env_vars import SF_DEBUG
+from .unified_interceptor import setup_interceptors
+
+METHOD = "os.execvp"
+
+
+def inject_code(file_path, **kwargs):
+    """
+    Injects the given code snippet at the top of the target Python file.
+    """
+    code_to_inject = f"""
+from sf_veritas import setup_interceptors
+
+setup_interceptors(
+    api_key={kwargs.get('api_key')!r},
+    service_identifier={kwargs.get('service_identifier', None)!r},
+    service_version={kwargs.get('service_version', None)!r},
+    service_additional_metadata={kwargs.get('service_additional_metadata', None)!r},
+    profiling_mode_enabled={kwargs.get('profiling_mode_enabled', False)},
+    profiling_max_depth={kwargs.get('profiling_max_depth', 5)},
+    domains_to_not_propagate_headers_to={kwargs.get('domains_to_not_propagate_headers_to', None)!r},
+    site_and_dist_packages_to_collect_local_variables_on={kwargs.get('site_and_dist_packages_to_collect_local_variables_on', None)!r}
+)
+"""
+
+    # Read the original file content
+    with open(file_path, "r") as f:
+        original_content = f.read()
+
+    # Combine the injected code with the original content
+    new_content = code_to_inject + "\n" + original_content
+
+    # Create a temporary file in the same directory as the original script
+    temp_dir = os.path.dirname(file_path)
+    temp_file = tempfile.NamedTemporaryFile(
+        delete=False, dir=temp_dir, prefix="temp-", suffix=".py"
+    )
+
+    # Write the new content to the temporary file
+    with open(temp_file.name, "w") as f:
+        f.write(new_content)
+
+    return temp_file.name
+
+
+def get_sf_veritas_index(argv=sys.argv):
+    for i, arg in enumerate(argv):
+        if "sf-veritas" in arg:
+            return i
+    return None
+
+
+def get_python_index(argv=sys.argv):
+    if "python" in argv:
+        return argv.index("python")
+    if "python3" in argv:
+        return argv.index("python3")
+    return None
+
+
+def get_command_index(command_name, argv=sys.argv):
+    """
+    Returns the index of a given command in sys.argv or None if not found.
+    """
+    if command_name in argv:
+        return argv.index(command_name)
+    return None
+
+
+def handle_framework_command(
+    command_index,
+    module_delimiter=":",
+    argv=sys.argv,
+    app_path_offset: int = 1,
+    **kwargs,
+):
+    """
+    Handles framework commands like uvicorn, gunicorn, sanic, waitress-serve, daphne, etc.
+    """
+    app_path = argv[command_index + app_path_offset]
+    module_path = app_path.split(module_delimiter)[0].replace(".", "/") + ".py"
+
+    # Inject code into the module
+    modified_module_path = inject_code(module_path, **kwargs)
+
+    # Turn the modified file path into a dotted module path
+    modified_module_relpath = os.path.relpath(modified_module_path, start=os.getcwd())
+    new_module_path = os.path.splitext(modified_module_relpath)[0].replace("/", ".")
+
+    # Strip off any leading dots that appear because of ../ or ./ references
+    new_module_path = new_module_path.lstrip(".")
+
+    # If there's still nothing, you could do a sanity check here:
+    # if not new_module_path:
+    #     raise ValueError("Unable to determine a valid module name from path")
+
+    # Reconstruct the final "module:app" string
+    if module_delimiter in app_path:
+        # e.g., "backend.temp-abcdef:application"
+        modified_app_path = (
+            new_module_path + module_delimiter + app_path.split(module_delimiter)[1]
+        )
+    else:
+        modified_app_path = new_module_path
+
+    # Build the run_command
+    run_command = [argv[command_index], *argv[command_index + 1 :]]
+    run_command[app_path_offset] = modified_app_path
+
+    return run_command
+
+
+def find_application_arg(command_index, argv=sys.argv):
+    """
+    Finds the application argument for daphne and granian commands, ignoring any flags.
+    """
+    if SF_DEBUG:
+        print(
+            f"Checking arguments after index {command_index}: {argv[command_index + 1:]}",
+            log=False,
+        )
+    should_skip = False
+    for i in range(command_index + 1, len(argv)):
+        if should_skip:
+            should_skip = False
+            continue
+        if argv[i].startswith("-"):
+            should_skip = True
+            continue
+        return argv[i]
+    return None
+
+
+def main(
+    api_key: str,
+    service_identifier: str = None,
+    service_version: Union[str, int] = None,  # Allow both string and int
+    service_additional_metadata: Dict[str, Union[str, int, float, None]] = None,
+    profiling_mode_enabled: bool = False,
+    profiling_max_depth: int = 5,
+    domains_to_not_propagate_headers_to: Optional[List[str]] = None,
+    site_and_dist_packages_to_collect_local_variables_on: Optional[List[str]] = None,
+    *args,
+):
+    """
+    Main function to handle CLI and set up interceptors.
+
+    Args:
+        api_key: (Required) API key for authentication. This must be provided as a CLI argument.
+        service_identifier: Identifier for the service.
+        service_version: Version of the service.
+        service_additional_metadata: Additional metadata to associate with the service.
+        profiling_mode_enabled: Whether profiling mode is enabled.
+        profiling_max_depth: Maximum depth for profiling.
+        domains_to_not_propagate_headers_to: Domains to which headers should not be propagated.
+        site_and_dist_packages_to_collect_local_variables_on: If set to None, defaults to [], excluding all installed packages.
+        *args: Additional command-line arguments to be passed through.
+    """
+
+    # Convert service_version to string if it's numeric
+    if service_version is not None:
+        service_version = str(service_version)
+
+    # Collect setup parameters
+    setup_params = {
+        "api_key": api_key,
+        "service_identifier": service_identifier,
+        "service_version": service_version,
+        "service_additional_metadata": service_additional_metadata,
+        "profiling_mode_enabled": profiling_mode_enabled,
+        "profiling_max_depth": profiling_max_depth,
+        "domains_to_not_propagate_headers_to": domains_to_not_propagate_headers_to,
+        "site_and_dist_packages_to_collect_local_variables_on": site_and_dist_packages_to_collect_local_variables_on,
+    }
+
+    # Remaining arguments are captured in *args
+    remaining_args = sys.argv[1:]
+    try:
+        separator_index = sys.argv.index("--")
+        remaining_args = sys.argv[separator_index + 1 :]
+    except ValueError:
+        pass
+
+    if SF_DEBUG:
+        print("[[ DEBUG ]] sys.argv [[ /DEBUG ]]", sys.argv)
+        print("[[ DEBUG ]] setup_params [[ /DEBUG ]]", setup_params)
+        print("[[ DEBUG ]] remaining_args [[ /DEBUG ]]", remaining_args)
+
+    setup_interceptors(**setup_params)
+
+    if not remaining_args:
+        print("Usage: sf-veritas [options] <command> [args]")
+        sys.exit(1)
+
+    # Detect CLI command and adjust injection based on the framework
+    python_index = get_python_index(remaining_args)
+    uvicorn_index = get_command_index("uvicorn", remaining_args)
+    gunicorn_index = get_command_index("gunicorn", remaining_args)
+    flask_index = get_command_index("flask", remaining_args)
+    django_admin_index = get_command_index("django-admin", remaining_args)
+    sanic_index = get_command_index("sanic", remaining_args)
+    pserve_index = get_command_index("pserve", remaining_args)
+    waitress_index = get_command_index("waitress-serve", remaining_args)
+    daphne_index = get_command_index("daphne", remaining_args)
+    granian_index = get_command_index("granian", remaining_args)
+
+    if python_index is not None and python_index >= 0:
+        if "-m" in remaining_args:
+            module_index = remaining_args.index("-m", python_index)
+
+            # Ensure there's a module specified after `-m`
+            if module_index + 1 < len(remaining_args):
+                module_dot_path = remaining_args[module_index + 1]
+                module_path = module_dot_path.replace(".", "/") + ".py"
+
+                # Inject code into the module, passing setup_params
+                modified_module_path = inject_code(module_path, **setup_params)
+
+                # Calculate the relative path for the new temporary file's dot-path
+                new_module_dot_path = os.path.splitext(
+                    os.path.relpath(modified_module_path, start=os.getcwd())
+                )[0].replace("/", ".")
+
+                # Rebuild the command with the modified module dot-path
+                code_after_module_index = module_index + 2
+                run_command = [
+                    sys.executable,
+                    "-m",
+                    new_module_dot_path,
+                ] + remaining_args[code_after_module_index:]
+            else:
+                print("Error: No module specified after '-m'.")
+                sys.exit(1)
+        else:
+            script_path = remaining_args[python_index + 1]
+
+            # Inject code into the script file directly, passing setup_params
+            modified_script_path = inject_code(script_path, **setup_params)
+
+            # Rebuild the command to use the path to the modified script file
+            run_command = [sys.executable, modified_script_path] + remaining_args[
+                python_index + 2 :
+            ]
+    elif uvicorn_index is not None:
+        run_command = handle_framework_command(
+            uvicorn_index, module_delimiter=".", argv=remaining_args, **setup_params
+        )
+    elif gunicorn_index is not None:
+        run_command = handle_framework_command(
+            gunicorn_index, argv=remaining_args, **setup_params
+        )
+    elif sanic_index is not None:
+        run_command = handle_framework_command(
+            sanic_index, module_delimiter=".", argv=remaining_args, **setup_params
+        )
+    elif waitress_index is not None:
+        run_command = handle_framework_command(
+            waitress_index, module_delimiter=":", argv=remaining_args, **setup_params
+        )
+    elif flask_index is not None:
+        # Flask expects the FLASK_APP environment variable
+        app_path = os.environ.get("FLASK_APP", None)
+        if app_path:
+            module_path = app_path.replace(".", "/") + ".py"
+
+            # Inject code into the module, passing setup_params
+            modified_module_path = inject_code(module_path, **setup_params)
+
+            # Set the modified path back to FLASK_APP
+            os.environ["FLASK_APP"] = modified_module_path.replace("/", ".").replace(
+                ".py", ""
+            )
+
+        run_command = remaining_args
+    elif django_admin_index is not None:
+        # Handle django-admin commands
+        script_path = remaining_args[django_admin_index]
+
+        # Inject code into the script file directly, passing setup_params
+        modified_script_path = inject_code(script_path, **setup_params)
+
+        # Rebuild the command to use the path to the modified script file
+        run_command = (
+            remaining_args[:django_admin_index]
+            + [modified_script_path]
+            + remaining_args[django_admin_index + 1 :]
+        )
+    elif daphne_index is not None or granian_index is not None:
+        command_index = daphne_index if daphne_index is not None else granian_index
+        app_arg = find_application_arg(command_index, argv=remaining_args)
+        app_path_offset = remaining_args.index(app_arg)
+
+        if not app_arg:
+            print("Error: No application module specified for daphne or granian.")
+            sys.exit(1)
+
+        run_command = handle_framework_command(
+            command_index,
+            module_delimiter=":",
+            argv=remaining_args,
+            app_path_offset=app_path_offset,
+            **setup_params,
+        )
+    elif pserve_index is not None:
+        # Rebuild the command to use pserve
+        run_command = remaining_args
+    else:
+        # For all other cases, pass the command as-is
+        run_command = remaining_args
+
+    if SF_DEBUG:
+        print("Run command is now:", run_command, log=False)
+
+    env = os.environ.copy()
+
+    if METHOD == "subprocess.run":
+        result = subprocess.run(run_command, env=env)
+        sys.exit(result.returncode)
+    else:
+        os.execvpe(run_command[0], run_command, env)
+
+
+def fire_main():
+    return Fire(main)
+
+
+if __name__ == "__main__":
+    fire_main()

sf_veritas/constants.py

@@ -0,0 +1,3 @@
+NONSESSION_APPLOGS = "nonsession-applogs"
+SAILFISH_DEFAULT_GRAPHQL_ENDPOINT = "https://api-service.sailfishqa.com/graphql/"
+SAILFISH_TRACING_HEADER = "X-Sf3-Rid"

sf_veritas/custom_excepthook.py

@@ -0,0 +1,285 @@
+import inspect
+
+# import linecache
+import logging
+import re
+import sys
+import threading
+from types import FrameType, TracebackType
+from typing import List, Optional
+
+from .local_env_detect import sf_is_local_dev_environment
+from . import app_config
+from .env_vars import (
+    SAILFISH_EXCEPTION_FETCH_BEYOND_OFFENDER_DEPTH,
+    SAILFISH_EXCEPTION_FETCH_LOCALS_BEYOND_OFFENDER_DEPTH,
+    SF_DEBUG,
+    SF_DEBUG_TRACES,
+)
+
+# from .frame_tools import get_locals
+from .interceptors import ExceptionInterceptor
+from .thread_local import (
+    get_or_set_sf_trace_id,
+    has_handled_exception,
+    mark_exception_handled,
+)
+from .types import FrameInfo
+
+REGEX_EVERYTHING_BEFORE_PYTHON_PACKAGE_NAME_WHEN_EXAMINING_FILENAME = (
+    r".*\/(site|dist)-packages\/"
+)
+REGEX_EVERYTHING_AFTER_NEXT_SLASH = r"\/.*"
+
+logger = logging.getLogger(__name__)
+
+# Profiling data structure to collect function call information
+call_hierarchy = []
+
+# Lock to manage thread-safe access to profiling data
+call_hierarchy_lock = threading.Lock()
+
+_original_excepthook = sys.excepthook
+
+
+def custom_thread_excepthook(args):
+    sys.excepthook(args.exc_type, args.exc_value, args.exc_traceback)
+
+
+def process_traceback_only(
+    exc_value: Exception, exc_traceback: TracebackType, was_caught: bool = True
+):
+    """
+    Processes only the traceback information for an exception.
+
+    Args:
+        exc_value (Exception): The exception value.
+        exc_traceback (TracebackType): The traceback object for the exception.
+    """
+    trace = extract_trace(exc_traceback)
+    except_interceptor = ExceptionInterceptor()
+    _, trace_id = get_or_set_sf_trace_id()
+    if SF_DEBUG:
+        print(
+            "process_traceback_only...SENDING DATA...do_send args=",
+            (str(exc_value), trace, trace_id, was_caught),
+            trace_id,
+            log=False,
+        )
+    except_interceptor.do_send(
+        (str(exc_value), trace, trace_id, was_caught, sf_is_local_dev_environment()[0]), trace_id
+    )
+
+
+# Usage example in the custom_excepthook function:
+def custom_excepthook(
+    exc_type, exc_value, exc_traceback: TracebackType, raise_original: bool = True
+):
+    if has_handled_exception(exc_value):
+        return
+    mark_exception_handled(exc_value)
+
+    if SF_DEBUG:
+        print(f"custom_excepthook -> raise_original={raise_original}", log=False)
+
+    transmit_exception(exc_type, exc_value, exc_traceback)
+
+    if raise_original:
+        _original_excepthook(exc_type, exc_value, exc_traceback)
+
+
+def transmit_exception(exc_type, exc_value, exc_traceback, was_caught: bool = True):
+    """
+    Handles transmitting exception information based on profiling mode.
+
+    Args:
+        exc_type: The type of the exception.
+        exc_value: The exception instance.
+        exc_traceback: The traceback object for the exception.
+    """
+    if app_config._profiling_mode_enabled:
+        threading.Thread(
+            target=process_profiling_data,
+            args=(exc_type, exc_value, exc_traceback, was_caught),
+        ).start()
+        return
+    process_traceback_only(exc_value, exc_traceback, was_caught)
+
+
+def profiling_function(frame: FrameType, event: str, arg):
+    if not app_config._profiling_mode_enabled:
+        return
+
+    if event in ["call", "return"]:
+        code = frame.f_code
+        function_name = code.co_name
+        filename = code.co_filename
+        line_no = frame.f_lineno
+        class_name = None
+
+        # Try to determine if we're inside a method of a class
+        if "self" in frame.f_locals:
+            class_name = type(frame.f_locals["self"]).__name__
+
+        call_info = {
+            "event": event,
+            "function": function_name,
+            "class": class_name,
+            "file": filename,
+            "line_no": line_no,
+        }
+
+        # Add profiling data to call hierarchy with a lock for thread safety
+        with call_hierarchy_lock:
+            if len(call_hierarchy) < app_config._profiling_max_depth:
+                call_hierarchy.append(call_info)
+
+
+def process_profiling_data(exc_type, exc_value, exc_traceback, was_caught: bool = True):
+    # Extract the traceback and process detailed information
+    trace = extract_trace(exc_traceback)
+
+    # Process profiling data
+    with call_hierarchy_lock:
+        if SF_DEBUG and SF_DEBUG_TRACES:
+            print("=" * 40, log=False)
+            print("Printing the complete call hierarchy after exception...", log=False)
+            print("=" * 40, log=False)
+
+        for entry in call_hierarchy:
+            if entry["event"] == "call":
+                class_name = f"{entry['class']}." if entry["class"] else ""
+                if SF_DEBUG and SF_DEBUG_TRACES:
+                    print(
+                        f"Called: {class_name}{entry['function']} (File: {entry['file']}, Line: {entry['line_no']})",
+                        log=False,
+                    )
+            elif entry["event"] == "return":
+                if SF_DEBUG and SF_DEBUG_TRACES:
+                    print(f"Return from: {entry['function']}", log=False)
+
+        if SF_DEBUG and SF_DEBUG_TRACES:
+            print("-" * 40, log=False)
+
+        # Clear profiling data after processing
+        call_hierarchy.clear()
+
+    # Send the extracted traceback data to the exception interceptor
+    except_interceptor = ExceptionInterceptor()
+    _, trace_id = get_or_set_sf_trace_id()
+    if SF_DEBUG:
+        print(
+            "process_traceback_only...SENDING DATA...do_send args=",
+            (str(exc_value), trace, trace_id, was_caught),
+            trace_id,
+            log=False,
+        )
+    except_interceptor.do_send(
+        (str(exc_value), trace, trace_id, was_caught, sf_is_local_dev_environment()[0]), trace_id
+    )
+
+
+def safe_repr(value):
+    try:
+        return repr(value)
+    except Exception:
+        return f"<unrepresentable: {type(value).__name__}>"
+
+
+def extract_trace(tb: Optional[TracebackType] = None) -> List[FrameInfo]:
+    tb_list = []
+
+    # Collect all traceback frames in natural order (most recent last)
+    while tb is not None:
+        tb_list.append(tb)
+        tb = tb.tb_next
+
+    if not tb_list:
+        return []
+
+    frame_stack = []
+
+    # Iterate through the traceback, capturing frame information
+    i = 0
+    for tb in tb_list:
+        frame = tb.tb_frame
+        frame_info = inspect.getframeinfo(frame)
+        filename = frame_info.filename
+        function_name = frame_info.function
+        line_no = frame_info.lineno
+        code_context = (
+            frame_info.code_context[0].strip()
+            if frame_info.code_context
+            else "No code context"
+        )
+
+        frame_info_obj = FrameInfo(
+            file=filename,
+            line=line_no,
+            function=function_name,
+            code=code_context,
+        )
+
+        should_collect_local_variables_for_stack_item_value = (
+            should_collect_local_variables_for_stack_item(filename)
+        )
+
+        # Extract locals if within the configured depth limits
+        if should_collect_local_variables_for_stack_item_value and (
+            i < SAILFISH_EXCEPTION_FETCH_LOCALS_BEYOND_OFFENDER_DEPTH
+            or i >= len(tb_list) - SAILFISH_EXCEPTION_FETCH_BEYOND_OFFENDER_DEPTH - 1
+        ):
+
+            frame_info_obj.locals = {k: safe_repr(v) for k, v in frame.f_locals.items()}
+
+        frame_stack.append(frame_info_obj)
+        i += int(should_collect_local_variables_for_stack_item_value)
+
+    frame_stack[-1].offender = True
+    return frame_stack
+
+
+def start_profiling():
+    """Start profiling to collect call hierarchy information if profiling is enabled."""
+    if app_config._profiling_mode_enabled:
+        sys.setprofile(profiling_function)
+
+
+def stop_profiling():
+    """Stop profiling to prevent collecting more call hierarchy data."""
+    sys.setprofile(None)
+
+
+def should_collect_local_variables_for_stack_item(filename: str) -> bool:
+    if "__all__" in app_config._site_and_dist_packages_to_collect_local_variables_on:
+        return True
+    return is_allowed_package(filename)
+
+
+def is_installed_package(filename: str) -> bool:
+    return bool(
+        re.search(
+            REGEX_EVERYTHING_BEFORE_PYTHON_PACKAGE_NAME_WHEN_EXAMINING_FILENAME,
+            filename,
+        )
+    )
+
+
+def get_package_name_from_site_or_dist_package(filename: str):
+    site_dist_packages_removed = re.sub(
+        REGEX_EVERYTHING_BEFORE_PYTHON_PACKAGE_NAME_WHEN_EXAMINING_FILENAME,
+        "",
+        filename,
+    )
+    package_installed_name = re.sub(
+        REGEX_EVERYTHING_AFTER_NEXT_SLASH, "", site_dist_packages_removed
+    )
+    return package_installed_name
+
+
+def is_allowed_package(filename: str) -> bool:
+    if not is_installed_package(filename):
+        return True
+
+    package = get_package_name_from_site_or_dist_package(filename)
+    return package in app_config._site_and_dist_packages_to_collect_local_variables_on