scout-apm 3.3.0__cp312-cp312-macosx_10_13_x86_64.whl

scout_apm/core/error_service.py

@@ -0,0 +1,167 @@
+# coding=utf-8
+
+import json
+import logging
+import os
+import threading
+import time
+
+from scout_apm.compat import (
+    escape,
+    gzip_compress,
+    queue,
+    urlencode,
+    urljoin,
+    urllib3_cert_pool_manager,
+)
+from scout_apm.core.config import scout_config
+from scout_apm.core.threading import SingletonThread
+
+# Time unit - monkey-patched in tests to make them run faster
+SECOND = 1
+
+logger = logging.getLogger(__name__)
+
+
+class ErrorServiceThread(SingletonThread):
+    _instance_lock = threading.Lock()
+    _stop_event = threading.Event()
+    _queue = queue.Queue(maxsize=500)
+
+    @classmethod
+    def _on_stop(cls):
+        super(ErrorServiceThread, cls)._on_stop()
+        # Unblock _queue.get()
+        try:
+            cls._queue.put(None, False)
+        except queue.Full as exc:
+            logger.debug("ErrorServiceThread full for stop: %r", exc, exc_info=exc)
+            pass
+
+    @classmethod
+    def send(cls, error):
+        try:
+            cls._queue.put(error, False)
+        except queue.Full as exc:
+            logger.debug("ErrorServiceThread full for send: %r", exc, exc_info=exc)
+
+        cls.ensure_started()
+
+    @classmethod
+    def wait_until_drained(cls, timeout_seconds=2.0, callback=None):
+        interval_seconds = min(timeout_seconds, 0.05)
+        start = time.time()
+        while True:
+            queue_size = cls._queue.qsize()
+            queue_empty = queue_size == 0
+            elapsed = time.time() - start
+            if queue_empty or elapsed >= timeout_seconds:
+                break
+
+            if callback is not None:
+                callback(queue_size)
+                callback = None
+
+            cls.ensure_started()
+
+            time.sleep(interval_seconds)
+        return queue_empty
+
+    def run(self):
+        batch_size = scout_config.value("errors_batch_size") or 1
+        http = urllib3_cert_pool_manager()
+        try:
+            while True:
+                errors = []
+                try:
+                    # Attempt to fetch the batch size off of the queue.
+                    for _ in range(batch_size):
+                        error = self._queue.get(block=True, timeout=1 * SECOND)
+                        if error:
+                            errors.append(error)
+                except queue.Empty:
+                    pass
+
+                if errors and self._send(http, errors):
+                    for _ in range(len(errors)):
+                        self._queue.task_done()
+
+                # Check for stop event after each read. This allows opening,
+                # sending, and then immediately stopping.
+                if self._stop_event.is_set():
+                    logger.debug("ErrorServiceThread stopping.")
+                    break
+        except Exception as exc:
+            logger.debug("ErrorServiceThread exception: %r", exc, exc_info=exc)
+        finally:
+            http.clear()
+            logger.debug("ErrorServiceThread stopped.")
+
+    def _send(self, http, errors):
+        try:
+            data = json.dumps(
+                {
+                    "notifier": "scout_apm_python",
+                    "environment": scout_config.value("environment"),
+                    "root": scout_config.value("application_root"),
+                    "problems": errors,
+                }
+            ).encode("utf-8")
+        except (ValueError, TypeError) as exc:
+            logger.debug(
+                "Exception when serializing error message: %r", exc, exc_info=exc
+            )
+            return False
+
+        params = {
+            "key": scout_config.value("key"),
+            "name": escape(scout_config.value("name"), quote=False),
+        }
+        headers = {
+            "Agent-Hostname": scout_config.value("hostname") or "",
+            "Content-Encoding": "gzip",
+            "Content-Type": "application/json",
+            "X-Error-Count": "{}".format(len(errors)),
+        }
+
+        encoded_args = urlencode(params)
+        full_url = urljoin(
+            scout_config.value("errors_host"), "apps/error.scout"
+        ) + "?{}".format(encoded_args)
+
+        try:
+            # urllib3 requires all parameters to be the same type for
+            # python 2.7.
+            # Since gzip can only return a str, convert all unicode instances
+            # to str.
+            response = http.request(
+                str("POST"),
+                str(full_url),
+                body=gzip_compress(data),
+                headers={str(key): str(value) for key, value in headers.items()},
+            )
+            if response.status >= 400:
+                logger.debug(
+                    (
+                        "ErrorServiceThread %r response error on _send:"
+                        + " %r on PID: %s on thread: %s"
+                    ),
+                    response.status,
+                    response.data,
+                    os.getpid(),
+                    threading.current_thread(),
+                )
+                return False
+        except Exception as exc:
+            logger.debug(
+                (
+                    "ErrorServiceThread exception on _send:"
+                    + " %r on PID: %s on thread: %s"
+                ),
+                exc,
+                os.getpid(),
+                threading.current_thread(),
+                exc_info=exc,
+            )
+            return False
+        return True

scout_apm/core/metadata.py

@@ -0,0 +1,66 @@
+# coding=utf-8
+
+import datetime as dt
+import sys
+from os import getpid
+
+from scout_apm.core.agent.commands import ApplicationEvent, format_dt_for_core_agent
+from scout_apm.core.agent.socket import CoreAgentSocketThread
+from scout_apm.core.config import scout_config
+
+
+def report_app_metadata():
+    CoreAgentSocketThread.send(
+        ApplicationEvent(
+            event_type="scout.metadata",
+            event_value=get_metadata(),
+            source="Pid: " + str(getpid()),
+            timestamp=dt.datetime.now(dt.timezone.utc),
+        )
+    )
+
+
+def get_metadata():
+    data = {
+        "language": "python",
+        "language_version": "{}.{}.{}".format(*sys.version_info[:3]),
+        "server_time": format_dt_for_core_agent(dt.datetime.now(dt.timezone.utc)),
+        "framework": scout_config.value("framework"),
+        "framework_version": scout_config.value("framework_version"),
+        "environment": "",
+        "app_server": scout_config.value("app_server"),
+        "hostname": scout_config.value("hostname"),
+        "database_engine": "",
+        "database_adapter": "",
+        "application_name": "",
+        "libraries": get_python_packages_versions(),
+        "paas": "",
+        "application_root": scout_config.value("application_root"),
+        "scm_subdirectory": scout_config.value("scm_subdirectory"),
+        "git_sha": scout_config.value("revision_sha"),
+    }
+    # Deprecated - see #327:
+    data["version"] = data["language_version"]
+    return data
+
+
+def get_python_packages_versions():
+    try:
+        from importlib.metadata import distributions
+    except ImportError:
+        # For some reason it is unavailable
+        return []
+
+    return sorted(
+        (
+            distribution.metadata["Name"],
+            (distribution.metadata["Version"] or "Unknown"),
+        )
+        for distribution in distributions()
+        # Filter out distributions wtih None for name or value. This can be the
+        # case for packages without a METADATA or PKG-INFO file in their relevant
+        # distribution directory. According to comments in importlib.metadata
+        # internals this is possible for certain old packages, but I could only
+        # recreate it by deliberately deleting said files.
+        if distribution.metadata["Name"]
+    )

scout_apm/core/n_plus_one_tracker.py

@@ -0,0 +1,41 @@
+# coding=utf-8
+
+from collections import defaultdict
+
+
+class NPlusOneTrackedItem(object):
+    __slots__ = ("count", "duration", "captured")
+
+    def __init__(self):
+        self.count = 0
+        self.duration = 0.0
+        self.captured = False
+
+
+class NPlusOneTracker(object):
+    # Fetch backtraces on this number of same SQL calls
+    COUNT_THRESHOLD = 5
+
+    # Minimum time in seconds before we start performing any work.
+    DURATION_THRESHOLD = 0.150
+
+    __slots__ = ("_map",)
+
+    def __init__(self):
+        self._map = defaultdict(NPlusOneTrackedItem)
+
+    def should_capture_backtrace(self, sql, duration, count=1):
+        item = self._map[sql]
+        if item.captured:
+            return False
+
+        item.duration += duration
+        item.count += count
+
+        if (
+            item.duration >= self.DURATION_THRESHOLD
+            and item.count >= self.COUNT_THRESHOLD
+        ):
+            item.captured = True
+            return True
+        return False

scout_apm/core/objtrace.py

@@ -0,0 +1,24 @@
+# coding=utf-8
+
+# Try use the C extension, but if it isn't available, provide a dummy
+# implementation.
+
+try:
+    from scout_apm.core._objtrace import disable, enable, get_counts, reset_counts
+except ImportError:  # pragma: no cover
+
+    def enable():
+        pass
+
+    def disable():
+        pass
+
+    def get_counts():
+        return (0, 0, 0, 0)
+
+    def reset_counts():
+        pass
+
+    is_extension = False
+else:  # pragma: no cover
+    is_extension = True

scout_apm/core/platform_detection.py

@@ -0,0 +1,66 @@
+# coding=utf-8
+
+import platform
+
+
+def is_valid_triple(triple):
+    values = triple.split("-", 1)
+    return (
+        len(values) == 2
+        and values[0] in ("i686", "x86_64", "aarch64", "unknown")
+        and values[1]
+        in ("unknown-linux-gnu", "unknown-linux-musl", "apple-darwin", "unknown")
+        # Validate _apple_darwin_aarch64_override was applied.
+        and triple != "aarch64-apple-darwin"
+    )
+
+
+def _apple_darwin_aarch64_override(triple):
+    """
+    If using M1 ARM64 machine, still use x86_64.
+    See https://github.com/scoutapp/scout_apm_python/issues/683
+    """
+    if triple == "aarch64-apple-darwin":
+        return "x86_64-apple-darwin"
+    return triple
+
+
+def get_triple():
+    return _apple_darwin_aarch64_override(
+        "{arch}-{platform}".format(arch=get_arch(), platform=get_platform())
+    )
+
+
+def get_arch():
+    """
+    What CPU are we on?
+    """
+    arch = platform.machine()
+    if arch == "i686":
+        return "i686"
+    elif arch == "x86_64":
+        return "x86_64"
+    elif arch == "aarch64":
+        return "aarch64"
+    elif arch == "arm64":
+        # We treat these as synonymous and name them "aarch64" for consistency
+        # Mac OS, for example, uses "arm64"
+        return "aarch64"
+    else:
+        return "unknown"
+
+
+def get_platform():
+    """
+    What Operating System (and sub-system like glibc / musl)
+    """
+    system_name = platform.system()
+    if system_name == "Linux":
+        # Previously we'd use either "-gnu" or "-musl" indicate which version
+        # of libc we were built against. We now default to musl since it
+        # reliably works on all platforms.
+        return "unknown-linux-musl"
+    elif system_name == "Darwin":
+        return "apple-darwin"
+    else:
+        return "unknown"

scout_apm/core/queue_time.py

@@ -0,0 +1,99 @@
+# coding=utf-8
+
+import datetime as dt
+import logging
+import time
+import typing
+
+from scout_apm.compat import datetime_to_timestamp
+from scout_apm.core.tracked_request import TrackedRequest
+
+logger = logging.getLogger(__name__)
+
+# Cutoff epoch is used for determining ambiguous timestamp boundaries
+CUTOFF_EPOCH_S = time.mktime((dt.date.today().year - 10, 1, 1, 0, 0, 0, 0, 0, 0))
+CUTOFF_EPOCH_MS = CUTOFF_EPOCH_S * 1000.0
+CUTOFF_EPOCH_US = CUTOFF_EPOCH_S * 1000000.0
+CUTOFF_EPOCH_NS = CUTOFF_EPOCH_S * 1000000000.0
+
+
+def _convert_ambiguous_timestamp_to_ns(timestamp: float) -> float:
+    """
+    Convert an ambiguous float timestamp that could be in nanoseconds,
+    microseconds, milliseconds, or seconds to nanoseconds. Return 0.0 for
+    values in the more than 10 years ago.
+    """
+    if timestamp > CUTOFF_EPOCH_NS:
+        converted_timestamp = timestamp
+    elif timestamp > CUTOFF_EPOCH_US:
+        converted_timestamp = timestamp * 1000.0
+    elif timestamp > CUTOFF_EPOCH_MS:
+        converted_timestamp = timestamp * 1000000.0
+    elif timestamp > CUTOFF_EPOCH_S:
+        converted_timestamp = timestamp * 1000000000.0
+    else:
+        return 0.0
+    return converted_timestamp
+
+
+def track_request_queue_time(
+    header_value: typing.Any, tracked_request: TrackedRequest
+) -> bool:
+    """
+    Attempt to parse a queue time header and store the result in the tracked request.
+
+    Returns:
+        bool: Whether we succeeded in marking queue time. Used for testing.
+    """
+    if header_value.startswith("t="):
+        header_value = header_value[2:]
+
+    try:
+        first_char = header_value[0]
+    except IndexError:
+        return False
+
+    if not first_char.isdigit():  # filter out negatives, nan, inf, etc.
+        return False
+
+    try:
+        ambiguous_start_timestamp = float(header_value)
+    except ValueError:
+        return False
+
+    start_timestamp_ns = _convert_ambiguous_timestamp_to_ns(ambiguous_start_timestamp)
+    if start_timestamp_ns == 0.0:
+        return False
+
+    tr_start_timestamp_ns = datetime_to_timestamp(tracked_request.start_time) * 1e9
+
+    # Ignore if in the future
+    if start_timestamp_ns > tr_start_timestamp_ns:
+        return False
+
+    queue_time_ns = int(tr_start_timestamp_ns - start_timestamp_ns)
+    tracked_request.tag("scout.queue_time_ns", queue_time_ns)
+    return True
+
+
+def track_job_queue_time(
+    header_value: typing.Any, tracked_request: TrackedRequest
+) -> bool:
+    """
+    Attempt to parse a queue/latency time header and store the result in the request.
+
+    Returns:
+        bool: Whether we succeeded in marking queue time for the job. Used for testing.
+    """
+    if header_value is not None:
+        now = datetime_to_timestamp(dt.datetime.now(dt.timezone.utc)) * 1e9
+        try:
+            ambiguous_float_start = typing.cast(float, header_value)
+            start = _convert_ambiguous_timestamp_to_ns(ambiguous_float_start)
+            queue_time_ns = int(now - start)
+        except TypeError:
+            logger.debug("Invalid job queue time header: %r", header_value)
+            return False
+        else:
+            tracked_request.tag("scout.job_queue_time_ns", queue_time_ns)
+            return True

scout_apm/core/sampler.py

@@ -0,0 +1,149 @@
+# coding=utf-8
+
+import random
+from typing import Dict, Optional, Tuple
+
+
+class Sampler:
+    """
+    Handles sampling decision logic for Scout APM.
+
+    This class encapsulates all sampling-related functionality including:
+    - Loading and managing sampling configuration
+    - Pattern matching for operations (endpoints and jobs)
+    - Making sampling decisions based on operation type and patterns
+    """
+
+    # Constants for operation type detection
+    CONTROLLER_PREFIX = "Controller/"
+    JOB_PREFIX = "Job/"
+
+    def __init__(self, config):
+        """
+        Initialize sampler with Scout configuration.
+
+        Args:
+            config: ScoutConfig instance containing sampling configuration
+        """
+        self.config = config
+        self.sample_rate = config.value("sample_rate")
+        self.sample_endpoints = config.value("sample_endpoints")
+        self.sample_jobs = config.value("sample_jobs")
+        self.ignore_endpoints = set(
+            config.value("ignore_endpoints") + config.value("ignore")
+        )
+        self.ignore_jobs = set(config.value("ignore_jobs"))
+        self.endpoint_sample_rate = config.value("endpoint_sample_rate")
+        self.job_sample_rate = config.value("job_sample_rate")
+
+    def _any_sampling(self):
+        """
+        Check if any sampling is enabled.
+
+        Returns:
+            Boolean indicating if any sampling is enabled
+        """
+        return (
+            self.sample_rate < 100
+            or self.sample_endpoints
+            or self.sample_jobs
+            or self.ignore_endpoints
+            or self.ignore_jobs
+            or self.endpoint_sample_rate is not None
+            or self.job_sample_rate is not None
+        )
+
+    def _find_matching_rate(
+        self, name: str, patterns: Dict[str, float]
+    ) -> Optional[str]:
+        """
+        Finds the matching sample rate for a given operation name.
+
+        Args:
+            name: The operation name to match
+            patterns: Dictionary of pattern to sample rate mappings
+
+        Returns:
+            The sample rate for the matching pattern or None if no match found
+        """
+
+        for pattern, rate in patterns.items():
+            if name.startswith(pattern):
+                return rate
+        return None
+
+    def _get_operation_type_and_name(
+        self, operation: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """
+        Determines if an operation is an endpoint or job and extracts its name.
+
+        Args:
+            operation: The full operation string (e.g. "Controller/users/show")
+
+        Returns:
+            Tuple of (type, name) where type is either 'endpoint' or 'job',
+            and name is the operation name without the prefix
+        """
+        if operation.startswith(self.CONTROLLER_PREFIX):
+            return "endpoint", operation[len(self.CONTROLLER_PREFIX) :]
+        elif operation.startswith(self.JOB_PREFIX):
+            return "job", operation[len(self.JOB_PREFIX) :]
+        else:
+            return None, None
+
+    def get_effective_sample_rate(self, operation: str, is_ignored: bool) -> int:
+        """
+        Determines the effective sample rate for a given operation.
+
+        Prioritization:
+        1. Sampling rate for specific endpoint or job
+        2. Specified ignore pattern or flag for operation
+        3. Global endpoint or job sample rate
+        4. Global sample rate
+
+        Args:
+            operation: The operation string (e.g. "Controller/users/show")
+            is_ignored: boolean for if the specific transaction is ignored
+
+        Returns:
+            Integer between 0 and 100 representing sample rate
+        """
+        op_type, name = self._get_operation_type_and_name(operation)
+        patterns = self.sample_endpoints if op_type == "endpoint" else self.sample_jobs
+        ignores = self.ignore_endpoints if op_type == "endpoint" else self.ignore_jobs
+        default_operation_rate = (
+            self.endpoint_sample_rate if op_type == "endpoint" else self.job_sample_rate
+        )
+
+        if not op_type or not name:
+            return self.sample_rate
+        matching_rate = self._find_matching_rate(name, patterns)
+        if matching_rate is not None:
+            return matching_rate
+        for prefix in ignores:
+            if name.startswith(prefix) or is_ignored:
+                return 0
+        if default_operation_rate is not None:
+            return default_operation_rate
+
+        # Fall back to global sample rate
+        return self.sample_rate
+
+    def should_sample(self, operation: str, is_ignored: bool) -> bool:
+        """
+        Determines if an operation should be sampled.
+        If no sampling is enabled, always return True.
+
+        Args:
+            operation: The operation string (e.g. "Controller/users/show"
+                   or "Job/mailer")
+
+        Returns:
+            Boolean indicating whether to sample this operation
+        """
+        if not self._any_sampling():
+            return True
+        return random.randint(1, 100) <= self.get_effective_sample_rate(
+            operation, is_ignored
+        )

scout_apm/core/samplers/cpu.py

@@ -0,0 +1,76 @@
+# coding=utf-8
+
+import datetime as dt
+import logging
+
+import psutil
+
+logger = logging.getLogger(__name__)
+
+
+class Cpu(object):
+    metric_type = "CPU"
+    metric_name = "Utilization"
+    human_name = "Process CPU"
+
+    def __init__(self):
+        self.last_run = dt.datetime.now(dt.timezone.utc)
+        self.last_cpu_times = psutil.Process().cpu_times()
+        self.num_processors = psutil.cpu_count()
+        if self.num_processors is None:
+            logger.debug("Could not determine CPU count - assuming there is one.")
+            self.num_processors = 1
+
+    def run(self):
+        now = dt.datetime.now(dt.timezone.utc)
+        process = psutil.Process()  # get a handle on the current process
+        cpu_times = process.cpu_times()
+
+        wall_clock_elapsed = (now - self.last_run).total_seconds()
+        if wall_clock_elapsed < 0:
+            self.save_times(now, cpu_times)
+            logger.debug(
+                "%s: Negative time elapsed. now: %s, last_run: %s.",
+                self.human_name,
+                now,
+                self.last_run,
+            )
+            return None
+
+        utime_elapsed = cpu_times.user - self.last_cpu_times.user
+        stime_elapsed = cpu_times.system - self.last_cpu_times.system
+        process_elapsed = utime_elapsed + stime_elapsed
+
+        # This can happen right after a fork.  This class starts up in
+        # pre-fork, records {u,s}time, then forks. This resets {u,s}time to 0
+        if process_elapsed < 0:
+            self.save_times(now, cpu_times)
+            logger.debug(
+                "%s: Negative process time elapsed. "
+                "utime: %s, stime: %s, total time: %s. "
+                "This is normal to see when starting a forking web server.",
+                self.human_name,
+                utime_elapsed,
+                stime_elapsed,
+                process_elapsed,
+            )
+            return None
+
+        # Normalized to # of processors
+        normalized_wall_clock_elapsed = wall_clock_elapsed * self.num_processors
+
+        # If somehow we run for 0 seconds between calls, don't try to divide by 0
+        if normalized_wall_clock_elapsed == 0:
+            res = 0
+        else:
+            res = (process_elapsed / normalized_wall_clock_elapsed) * 100
+
+        self.save_times(now, cpu_times)
+
+        logger.debug("%s: %s [%s CPU(s)]", self.human_name, res, self.num_processors)
+
+        return res
+
+    def save_times(self, now, cpu_times):
+        self.last_run = now
+        self.cpu_times = cpu_times