datadog-checks-base 36.5.0__py2.py3-none-any.whl → 37.4.0__py2.py3-none-any.whl

datadog_checks/base/__about__.py

@@ -1,4 +1,4 @@
 # (C) Datadog, Inc. 2018-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
-__version__ = "36.5.0"
+__version__ = "37.4.0"

datadog_checks/base/__init__.py

@@ -1,18 +1,20 @@
 # (C) Datadog, Inc. 2018-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
+from datadog_checks.base.agent import datadog_agent
+
 from .__about__ import __version__
 from .checks import AgentCheck
 from .checks.openmetrics import OpenMetricsBaseCheck
+from .checks.openmetrics.v2.base import OpenMetricsBaseCheckV2
 from .config import is_affirmative
 from .errors import ConfigurationError
 from .utils.common import ensure_bytes, ensure_unicode, to_native_string, to_string
 
-# Python 3+
-try:
-    from .checks.openmetrics.v2.base import OpenMetricsBaseCheckV2
-except ImportError:
-    OpenMetricsBaseCheckV2 = None
+if datadog_agent.get_config('use_boringssl'):
+    import urllib3.contrib.pyopenssl
+
+    urllib3.contrib.pyopenssl.inject_into_urllib3()
 
 # Windows-only
 try:

datadog_checks/base/agent.py

@@ -0,0 +1,15 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+try:
+    import aggregator
+    import datadog_agent
+
+    AGENT_RUNNING = True
+except ImportError:
+    from .stubs import aggregator, datadog_agent
+
+    AGENT_RUNNING = False
+
+
+__all__ = ['AGENT_RUNNING', 'aggregator', 'datadog_agent']

datadog_checks/base/checks/base.py

@@ -5,8 +5,8 @@ import copy
 import functools
 import importlib
 import inspect
-import json
 import logging
+import os
 import re
 import traceback
 import unicodedata
@@ -28,7 +28,9 @@ from typing import (  # noqa: F401
 )
 
 import yaml
-from six import PY2, binary_type, iteritems, raise_from, text_type
+from pydantic import BaseModel, ValidationError
+
+from datadog_checks.base.agent import AGENT_RUNNING, aggregator, datadog_agent
 
 from ..config import is_affirmative
 from ..constants import ServiceCheck
@@ -45,43 +47,44 @@ from ..types import (
 from ..utils.agent.utils import should_profile_memory
 from ..utils.common import ensure_bytes, to_native_string
 from ..utils.diagnose import Diagnosis
+from ..utils.fips import enable_fips
 from ..utils.http import RequestsWrapper
 from ..utils.limiter import Limiter
 from ..utils.metadata import MetadataManager
 from ..utils.secrets import SecretsSanitizer
+from ..utils.serialization import from_json, to_json
 from ..utils.tagging import GENERIC_TAGS
 from ..utils.tls import TlsContextWrapper
 from ..utils.tracing import traced_class
 
-try:
-    import datadog_agent
-
+if AGENT_RUNNING:
     from ..log import CheckLoggingAdapter, init_logging
 
-    init_logging()
-except ImportError:
-    from ..stubs import datadog_agent
+else:
     from ..stubs.log import CheckLoggingAdapter, init_logging
 
-    init_logging()
-
-try:
-    import aggregator
-
-    using_stub_aggregator = False
-except ImportError:
-    from ..stubs import aggregator
-
-    using_stub_aggregator = True
-
+init_logging()
 
 if datadog_agent.get_config('disable_unsafe_yaml'):
     from ..ddyaml import monkey_patch_pyyaml
 
     monkey_patch_pyyaml()
 
-if not PY2:
-    from pydantic import BaseModel, ValidationError
+if datadog_agent.get_config('integration_tracing'):
+    from ddtrace import patch
+
+    # handle thread monitoring as an additional option
+    # See: http://pypi.datadoghq.com/trace/docs/other_integrations.html#futures
+    if datadog_agent.get_config('integration_tracing_futures'):
+        patch(logging=True, requests=True, futures=True)
+    else:
+        patch(logging=True, requests=True)
+
+if is_affirmative(datadog_agent.get_config('integration_profiling')):
+    from ddtrace.profiling import Profiler
+
+    prof = Profiler(service='datadog-agent-integrations')
+    prof.start()
 
 if TYPE_CHECKING:
     import ssl  # noqa: F401
@@ -174,7 +177,6 @@ class AgentCheck(object):
         # type: (*Any, **Any) -> None
         """
         Parameters:
-
             name (str):
                 the name of the check
             init_config (dict):
@@ -302,8 +304,13 @@ class AgentCheck(object):
         # Functions that will be called exactly once (if successful) before the first check run
         self.check_initializations = deque()  # type: Deque[Callable[[], None]]
 
-        if not PY2:
-            self.check_initializations.append(self.load_configuration_models)
+        self.check_initializations.append(self.load_configuration_models)
+
+        self.__formatted_tags = None
+        self.__logs_enabled = None
+
+        if os.environ.get("GOFIPS", "0") == "1":
+            enable_fips()
 
     def _create_metrics_pattern(self, metric_patterns, option_name):
         all_patterns = metric_patterns.get(option_name, [])
@@ -394,6 +401,36 @@ class AgentCheck(object):
 
         return self._http
 
+    @property
+    def logs_enabled(self):
+        # type: () -> bool
+        """
+        Returns True if logs are enabled, False otherwise.
+        """
+        if self.__logs_enabled is None:
+            self.__logs_enabled = bool(datadog_agent.get_config('logs_enabled'))
+
+        return self.__logs_enabled
+
+    @property
+    def formatted_tags(self):
+        # type: () -> str
+        if self.__formatted_tags is None:
+            normalized_tags = set()
+            for tag in self.instance.get('tags', []):
+                key, _, value = tag.partition(':')
+                if not value:
+                    continue
+
+                if self.disable_generic_tags and key in GENERIC_TAGS:
+                    key = '{}_{}'.format(self.name, key)
+
+                normalized_tags.add('{}:{}'.format(key, value))
+
+            self.__formatted_tags = ','.join(sorted(normalized_tags))
+
+        return self.__formatted_tags
+
     @property
     def diagnosis(self):
         # type: () -> Diagnosis
@@ -430,7 +467,7 @@ class AgentCheck(object):
         Used for sending metadata via Go bindings.
         """
         if not hasattr(self, '_metadata_manager'):
-            if not self.check_id and not using_stub_aggregator:
+            if not self.check_id and AGENT_RUNNING:
                 raise RuntimeError('Attribute `check_id` must be set')
 
             self._metadata_manager = MetadataManager(self.name, self.check_id, self.log, self.METADATA_TRANSFORMERS)
@@ -470,11 +507,9 @@ class AgentCheck(object):
 
         known_options = {k for k, _ in models_config}  # type: Set[str]
 
-        if not PY2:
-
-            if isinstance(models_config, BaseModel):
-                # Also add aliases, if any
-                known_options.update(set(models_config.model_dump(by_alias=True)))
+        if isinstance(models_config, BaseModel):
+            # Also add aliases, if any
+            known_options.update(set(models_config.model_dump(by_alias=True)))
 
         unknown_options = [option for option in user_configs.keys() if option not in known_options]  # type: List[str]
 
@@ -526,8 +561,7 @@ class AgentCheck(object):
     def load_configuration_model(import_path, model_name, config, context):
         try:
             package = importlib.import_module(import_path)
-        # TODO: remove the type ignore when we drop Python 2
-        except ModuleNotFoundError as e:  # type: ignore
+        except ModuleNotFoundError as e:
             # Don't fail if there are no models
             if str(e).startswith('No module named '):
                 return
@@ -538,8 +572,7 @@ class AgentCheck(object):
         if model is not None:
             try:
                 config_model = model.model_validate(config, context=context)
-            # TODO: remove the type ignore when we drop Python 2
-            except ValidationError as e:  # type: ignore
+            except ValidationError as e:
                 errors = e.errors()
                 num_errors = len(errors)
                 message_lines = [
@@ -558,7 +591,7 @@ class AgentCheck(object):
                     )
                     message_lines.append('  {}'.format(error['msg']))
 
-                raise_from(ConfigurationError('\n'.join(message_lines)), None)
+                raise ConfigurationError('\n'.join(message_lines)) from None
             else:
                 return config_model
 
@@ -608,7 +641,7 @@ class AgentCheck(object):
             err_msg = 'Histogram: {} has non integer value: {}. Only integer are valid bucket values (count).'.format(
                 repr(name), repr(value)
             )
-            if using_stub_aggregator:
+            if not AGENT_RUNNING:
                 raise ValueError(err_msg)
             self.warning(err_msg)
             return
@@ -658,6 +691,20 @@ class AgentCheck(object):
 
         aggregator.submit_event_platform_event(self, self.check_id, to_native_string(raw_event), "dbm-metadata")
 
+    def event_platform_event(self, raw_event, event_track_type):
+        # type: (str, str) -> None
+        """Send an event platform event.
+
+        Parameters:
+            raw_event (str):
+                JSON formatted string representing the event to send
+            event_track_type (str):
+                type of event ingested and processed by the event platform
+        """
+        if raw_event is None:
+            return
+        aggregator.submit_event_platform_event(self, self.check_id, to_native_string(raw_event), event_track_type)
+
     def should_send_metric(self, metric_name):
         return not self._metric_excluded(metric_name) and self._metric_included(metric_name)
 
@@ -706,7 +753,7 @@ class AgentCheck(object):
             err_msg = 'Metric: {} has non float value: {}. Only float values can be submitted as metrics.'.format(
                 repr(name), repr(value)
             )
-            if using_stub_aggregator:
+            if not AGENT_RUNNING:
                 raise ValueError(err_msg)
             self.warning(err_msg)
             return
@@ -718,7 +765,6 @@ class AgentCheck(object):
         """Sample a gauge metric.
 
         Parameters:
-
             name (str):
                 the name of the metric
             value (float):
@@ -741,7 +787,6 @@ class AgentCheck(object):
         """Sample a raw count metric.
 
         Parameters:
-
             name (str):
                 the name of the metric
             value (float):
@@ -766,7 +811,6 @@ class AgentCheck(object):
         """Sample an increasing counter metric.
 
         Parameters:
-
             name (str):
                 the name of the metric
             value (float):
@@ -798,7 +842,6 @@ class AgentCheck(object):
         """Sample a point, with the rate calculated at the end of the check.
 
         Parameters:
-
             name (str):
                 the name of the metric
             value (float):
@@ -821,7 +864,6 @@ class AgentCheck(object):
         """Sample a histogram metric.
 
         Parameters:
-
             name (str):
                 the name of the metric
             value (float):
@@ -844,7 +886,6 @@ class AgentCheck(object):
         """Sample a histogram based on rate metrics.
 
         Parameters:
-
             name (str):
                 the name of the metric
             value (float):
@@ -867,7 +908,6 @@ class AgentCheck(object):
         """Increment a counter metric.
 
         Parameters:
-
             name (str):
                 the name of the metric
             value (float):
@@ -891,7 +931,6 @@ class AgentCheck(object):
         """Decrement a counter metric.
 
         Parameters:
-
             name (str):
                 the name of the metric
             value (float):
@@ -915,7 +954,6 @@ class AgentCheck(object):
         """Send the status of a service.
 
         Parameters:
-
             name (str):
                 the name of the service check
             status (int):
@@ -941,6 +979,42 @@ class AgentCheck(object):
             self, self.check_id, self._format_namespace(name, raw), status, tags, hostname, message
         )
 
+    def send_log(self, data, cursor=None, stream='default'):
+        # type: (dict[str, str], dict[str, Any] | None, str) -> None
+        """Send a log for submission.
+
+        Parameters:
+            data (dict[str, str]):
+                The log data to send. The following keys are treated specially, if present:
+
+                - timestamp: should be an integer or float representing the number of seconds since the Unix epoch
+                - ddtags: if not defined, it will automatically be set based on the instance's `tags` option
+            cursor (dict[str, Any] or None):
+                Metadata associated with the log which will be saved to disk. The most recent value may be
+                retrieved with the `get_log_cursor` method.
+            stream (str):
+                The stream associated with this log, used for accurate cursor persistence.
+                Has no effect if `cursor` argument is `None`.
+        """
+        attributes = data.copy()
+        if 'ddtags' not in attributes and self.formatted_tags:
+            attributes['ddtags'] = self.formatted_tags
+
+        timestamp = attributes.get('timestamp')
+        if timestamp is not None:
+            # convert seconds to milliseconds
+            attributes['timestamp'] = int(timestamp * 1000)
+
+        datadog_agent.send_log(to_json(attributes), self.check_id)
+        if cursor is not None:
+            self.write_persistent_cache('log_cursor_{}'.format(stream), to_json(cursor))
+
+    def get_log_cursor(self, stream='default'):
+        # type: (str) -> dict[str, Any] | None
+        """Returns the most recent log cursor from disk."""
+        data = self.read_persistent_cache('log_cursor_{}'.format(stream))
+        return from_json(data) if data else None
+
     def _log_deprecation(self, deprecation_key, *args):
         # type: (str, *str) -> None
         """
@@ -963,7 +1037,6 @@ class AgentCheck(object):
         """Updates the cached metadata `name` with `value`, which is then sent by the Agent at regular intervals.
 
         Parameters:
-
             name (str):
                 the name of the metadata
             value (Any):
@@ -1015,7 +1088,6 @@ class AgentCheck(object):
         """Returns the value previously stored with `write_persistent_cache` for the same `key`.
 
         Parameters:
-
             key (str):
                 the key to retrieve
         """
@@ -1030,7 +1102,6 @@ class AgentCheck(object):
         The cache is persistent between agent restarts but will be rebuilt if the check instance configuration changes.
 
         Parameters:
-
             key (str):
                 the key to retrieve
             value (str):
@@ -1049,7 +1120,7 @@ class AgentCheck(object):
             new_tags = []
             for hostname, source_map in external_tags:
                 new_tags.append((to_native_string(hostname), source_map))
-                for src_name, tags in iteritems(source_map):
+                for src_name, tags in source_map.items():
                     source_map[src_name] = self._normalize_tags_type(tags)
             datadog_agent.set_external_tags(new_tags)
         except IndexError:
@@ -1076,7 +1147,6 @@ class AgentCheck(object):
         and make it compliant with flake8 logging format linter.
 
         Parameters:
-
             warning_message (str):
                 the warning message
             args (Any):
@@ -1114,7 +1184,7 @@ class AgentCheck(object):
         The agent calls this method to retrieve diagnostics from integrations. This method
         runs explicit diagnostics if available.
         """
-        return json.dumps([d._asdict() for d in (self.diagnosis.diagnoses + self.diagnosis.run_explicit())])
+        return to_json([d._asdict() for d in (self.diagnosis.diagnoses + self.diagnosis.run_explicit())])
 
     def _get_requests_proxy(self):
         # type: () -> ProxySettings
@@ -1142,13 +1212,14 @@ class AgentCheck(object):
     def normalize(self, metric, prefix=None, fix_case=False):
         # type: (Union[str, bytes], Union[str, bytes], bool) -> str
         """
-        Turn a metric into a well-formed metric name
-        prefix.b.c
-        :param metric The metric name to normalize
-        :param prefix A prefix to to add to the normalized name, default None
-        :param fix_case A boolean, indicating whether to make sure that the metric name returned is in "snake_case"
+        Turn a metric into a well-formed metric name prefix.b.c
+
+        Parameters:
+            metric: The metric name to normalize
+            prefix: A prefix to to add to the normalized name, default None
+            fix_case: A boolean, indicating whether to make sure that the metric name returned is in "snake_case"
         """
-        if isinstance(metric, text_type):
+        if isinstance(metric, str):
             metric = unicodedata.normalize('NFKD', metric).encode('ascii', 'ignore')
 
         if fix_case:
@@ -1173,7 +1244,7 @@ class AgentCheck(object):
         This happens for legacy reasons, when we cleaned up some characters (like '-')
         which are allowed in tags.
         """
-        if isinstance(tag, text_type):
+        if isinstance(tag, str):
             tag = tag.encode('utf-8', 'ignore')
         tag = self.TAG_REPLACEMENT.sub(br'_', tag)
         tag = self.MULTIPLE_UNDERSCORE_CLEANUP.sub(br'_', tag)
@@ -1219,7 +1290,13 @@ class AgentCheck(object):
 
                     enter_pdb(self.check, line=self.init_config['set_breakpoint'], args=(instance,))
                 elif self.should_profile_memory():
-                    self.profile_memory(self.check, self.init_config, args=(instance,))
+                    # self.init_config['profile_memory'] could be `/tmp/datadog-agent-memory-profiler*`
+                    # that is generated by Datadog Agent.
+                    # If we use `--m-dir` for `agent check` command, a hidden flag, it should be same as a given value.
+                    namespaces = [self.init_config['profile_memory']]
+                    for id in self.check_id.split(":"):
+                        namespaces.append(id)
+                    self.profile_memory(func=self.check, namespaces=namespaces, args=(instance,))
                 else:
                     self.check(instance)
 
@@ -1227,7 +1304,7 @@ class AgentCheck(object):
         except Exception as e:
             message = self.sanitize(str(e))
             tb = self.sanitize(traceback.format_exc())
-            error_report = json.dumps([{'message': message, 'traceback': tb}])
+            error_report = to_json([{'message': message, 'traceback': tb}])
         finally:
             if self.metric_limiter:
                 if is_affirmative(self.debug_metrics.get('metric_contexts', False)):
@@ -1267,13 +1344,12 @@ class AgentCheck(object):
         ```
 
         Parameters:
-
             event (dict[str, Any]):
                 the event to be sent
         """
         # Enforce types of some fields, considerably facilitates handling in go bindings downstream
-        for key, value in iteritems(event):
-            if not isinstance(value, (text_type, binary_type)):
+        for key, value in event.items():
+            if not isinstance(value, (str, bytes)):
                 continue
 
             try:

datadog_checks/base/checks/kube_leader/mixins.py

@@ -1,11 +1,7 @@
 # (C) Datadog, Inc. 2018-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
-
-try:
-    import datadog_agent
-except ImportError:
-    from ...stubs import datadog_agent
+from datadog_checks.base.agent import datadog_agent
 
 from .. import AgentCheck
 from .record import ElectionRecordAnnotation, ElectionRecordLease

datadog_checks/base/checks/libs/prometheus.py

@@ -2,15 +2,29 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
+from itertools import tee
+
 from prometheus_client.metrics_core import Metric
 from prometheus_client.parser import _parse_sample, _replace_help_escaping
 
 
+def text_fd_to_metric_families(fd):
+    raw_lines, input_lines = tee(fd, 2)
+    # It's important to start parsing outside of the for-loop.
+    # This way we treat crashes before we yield the first parsed line differently than crashes while yielding.
+    parsed_lines = _parse_payload(input_lines)
+    for raw_line, metric_family in zip(raw_lines, parsed_lines):
+        try:
+            yield metric_family
+        except Exception as e:
+            raise ValueError("Failed to parse the metric response '{}': {}".format(raw_line, e))
+
+
 # This copies most of the code from upstream at that version:
 # https://github.com/prometheus/client_python/blob/049744296d216e6be65dc8f3d44650310f39c384/prometheus_client/parser.py#L144
 # but reverting the behavior to a compatible version, which doesn't change counters to have a total suffix. See
 # https://github.com/prometheus/client_python/commit/a4dd93bcc6a0422e10cfa585048d1813909c6786#diff-0adf47ea7f99c66d4866ccb4e557a865L158
-def text_fd_to_metric_families(fd):
+def _parse_payload(fd):
     """Parse Prometheus text format from a file descriptor.
 
     This is a laxer parser than the main Go parser, so successful parsing does

datadog_checks/base/checks/libs/thread_pool.py

@@ -25,8 +25,7 @@
 import sys
 import threading
 import traceback
-
-from six.moves import queue, range
+import queue
 
 # Item pushed on the work queue to tell the worker threads to terminate
 SENTINEL = "QUIT"

datadog_checks/base/checks/logs/__init__.py

@@ -0,0 +1,3 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)

datadog_checks/base/checks/logs/crawler/__init__.py

@@ -0,0 +1,3 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)

datadog_checks/base/checks/logs/crawler/base.py

@@ -0,0 +1,34 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Iterable
+
+from datadog_checks.base import AgentCheck
+
+if TYPE_CHECKING:
+    from datadog_checks.base.checks.logs.crawler.stream import LogStream
+
+
+class LogCrawlerCheck(AgentCheck, ABC):
+    @abstractmethod
+    def get_log_streams(self) -> Iterable[LogStream]:
+        """
+        Yields the log streams associated with this check.
+        """
+
+    def process_streams(self) -> None:
+        """
+        Process the log streams and send the collected logs.
+
+        Crawler checks that need more functionality can implement the `check` method and call this directly.
+        """
+        for stream in self.get_log_streams():
+            last_cursor = self.get_log_cursor(stream.name)
+            for record in stream.records(cursor=last_cursor):
+                self.send_log(record.data, cursor=record.cursor, stream=stream.name)
+
+    def check(self, _) -> None:
+        self.process_streams()

datadog_checks/base/checks/logs/crawler/stream.py

@@ -0,0 +1,52 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Iterable
+
+if TYPE_CHECKING:
+    from datadog_checks.base import AgentCheck
+
+
+class LogRecord:
+    __slots__ = ('cursor', 'data')
+
+    def __init__(self, data: dict[str, str], *, cursor: dict[str, Any] | None):
+        self.data = data
+        self.cursor = cursor
+
+
+class LogStream(ABC):
+    def __init__(self, *, check: AgentCheck, name: str):
+        self.__check = check
+        self.__name = name
+
+    @property
+    def check(self) -> AgentCheck:
+        """
+        The AgentCheck instance associated with this LogStream.
+        """
+        return self.__check
+
+    @property
+    def name(self) -> str:
+        """
+        The name of this LogStream.
+        """
+        return self.__name
+
+    def construct_tags(self, tags: list[str]) -> list[str]:
+        """
+        Returns a formatted string of tags which may be used directly as the `ddtags` field of logs.
+        This will include the `tags` from the integration instance config.
+        """
+        formatted_tags = ','.join(tags)
+        return f'{self.check.formatted_tags},{formatted_tags}' if self.check.formatted_tags else formatted_tags
+
+    @abstractmethod
+    def records(self, *, cursor: dict[str, Any] | None = None) -> Iterable[LogRecord]:
+        """
+        Yields log records as they are received.
+        """

datadog_checks/base/checks/openmetrics/base_check.py

@@ -4,7 +4,6 @@
 from copy import deepcopy
 
 import requests
-from six import PY2
 
 from ...errors import CheckException
 from ...utils.tracing import traced_class
@@ -183,5 +182,4 @@ class StandardFields(object):
     pass
 
 
-if not PY2:
-    StandardFields.__doc__ = '\n'.join('- `{}`'.format(field) for field in STANDARD_FIELDS)
+StandardFields.__doc__ = '\n'.join('- `{}`'.format(field) for field in STANDARD_FIELDS)