dt-extensions-sdk 1.1.19__py3-none-any.whl → 1.1.20__py3-none-any.whl

{dt_extensions_sdk-1.1.19.dist-info → dt_extensions_sdk-1.1.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dt-extensions-sdk
-Version: 1.1.19
+Version: 1.1.20
 Project-URL: Documentation, https://github.com/dynatrace-extensions/dt-extensions-python-sdk#readme
 Project-URL: Issues, https://github.com/dynatrace-extensions/dt-extensions-python-sdk/issues
 Project-URL: Source, https://github.com/dynatrace-extensions/dt-extensions-python-sdk

{dt_extensions_sdk-1.1.19.dist-info → dt_extensions_sdk-1.1.20.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dynatrace_extension/__about__.py,sha256=eK_1QX-n9dlWHMtuWO8671EwIG7rKIdY_7SC5tYSNe0,110
+dynatrace_extension/__about__.py,sha256=-8wSDjpuHMK14QbY11SSVFtSWsTg2XYDKWCAQc82ob4,110
 dynatrace_extension/__init__.py,sha256=BvQuknmA7ti3WJi3zEXZfY7aAxJrie37VNitWICsUvI,752
 dynatrace_extension/cli/__init__.py,sha256=HCboY_eJPoqjFmoPDsBL8Jk6aNvank8K7JpkVrgwzUM,123
 dynatrace_extension/cli/main.py,sha256=wi3xxji3WbVXViebzLCB5UGNSADUcdYUzGI02EYbsaM,16965
@@ -16,7 +16,7 @@ dynatrace_extension/cli/create/extension_template/extension_name/__main__.py.tem
 dynatrace_extension/sdk/__init__.py,sha256=RsqQ1heGyCmSK3fhuEKAcxQIRCg4gEK0-eSkIehL5Nc,86
 dynatrace_extension/sdk/activation.py,sha256=goTbT1tD2kn8xfyXFdTy_cTZNcFPJpgbvQM8HOzKECA,1480
 dynatrace_extension/sdk/callback.py,sha256=1P9uQ388j6q6a-8VgmV5FxcOMH-Mr3zvWXk2ftVD8j4,5828
-dynatrace_extension/sdk/communication.py,sha256=Z8Gbvd18ISf1pqXCDsBEfFeNU996J16aPgKUhJ082Zc,18804
+dynatrace_extension/sdk/communication.py,sha256=dfnafZvpBCSQDGHGuNl70Ym1W1RlIuJMynDxMOfnFCs,17592
 dynatrace_extension/sdk/event.py,sha256=J261imbFKpxfuAQ6Nfu3RRcsIQKKivy6fme1nww2g-8,388
 dynatrace_extension/sdk/extension.py,sha256=3883WIlA5hM-0kNbdFaOtxYC_KzW-jPLMMaf0EgHRBo,40642
 dynatrace_extension/sdk/helper.py,sha256=ZNrO9ao2hE3KQ934vAYD74k0fCr6QTG-_bAvbk9-hi8,6562
@@ -26,8 +26,8 @@ dynatrace_extension/sdk/vendor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 dynatrace_extension/sdk/vendor/mureq/LICENSE,sha256=8AVcgZgiT_mvK1fOofXtRRr2f1dRXS_K21NuxQgP4VM,671
 dynatrace_extension/sdk/vendor/mureq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dynatrace_extension/sdk/vendor/mureq/mureq.py,sha256=TQ3xcpfwLEYIU1TU65A9OqWqwIKqyO8SSUFeuCvE60Y,14655
-dt_extensions_sdk-1.1.19.dist-info/METADATA,sha256=BjBxbVeBUwaGKVSQ_C_G1sQJLOOi7CuJj5eBEUoY3JM,2794
-dt_extensions_sdk-1.1.19.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-dt_extensions_sdk-1.1.19.dist-info/entry_points.txt,sha256=pweyOCgENGHjOlT6_kXYaBPOrE3p18K0UettqnNlnoE,55
-dt_extensions_sdk-1.1.19.dist-info/licenses/LICENSE.txt,sha256=3Zihv0lOVYHNfDkJC-tUAU6euP9r2NexsDW4w-zqgVk,1078
-dt_extensions_sdk-1.1.19.dist-info/RECORD,,
+dt_extensions_sdk-1.1.20.dist-info/METADATA,sha256=TSHy4wLRnheyQx1cHs9GaOmaIDsqMZ5PxZ2GNoAbu_w,2794
+dt_extensions_sdk-1.1.20.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+dt_extensions_sdk-1.1.20.dist-info/entry_points.txt,sha256=pweyOCgENGHjOlT6_kXYaBPOrE3p18K0UettqnNlnoE,55
+dt_extensions_sdk-1.1.20.dist-info/licenses/LICENSE.txt,sha256=3Zihv0lOVYHNfDkJC-tUAU6euP9r2NexsDW4w-zqgVk,1078
+dt_extensions_sdk-1.1.20.dist-info/RECORD,,

dynatrace_extension/__about__.py

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: MIT
 
-__version__ = "1.1.19"
+__version__ = "1.1.20"

dynatrace_extension/sdk/communication.py

@@ -6,25 +6,25 @@ from __future__ import annotations
 
 import json
 import logging
-import random
 import sys
-import time
 from abc import ABC, abstractmethod
-from collections import deque
 from dataclasses import dataclass
 from enum import Enum
-from itertools import islice
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, TypeVar, Union
+from typing import Any, Generator, List, Sequence, TypeVar, Union
 
 from .vendor.mureq.mureq import HTTPException, Response, request
 
 CONTENT_TYPE_JSON = "application/json;charset=utf-8"
 CONTENT_TYPE_PLAIN = "text/plain;charset=utf-8"
 COUNT_METRIC_ITEMS_DICT = TypeVar("COUNT_METRIC_ITEMS_DICT", str, List[str])
+
+# TODO - I believe these can be adjusted via RuntimeConfig, they can't be constants
 MAX_MINT_LINES_PER_REQUEST = 1000
 MAX_LOG_EVENTS_PER_REQUEST = 50_000
-MAX_LOG_REQUEST_SIZE = 5_000_000
+MAX_LOG_REQUEST_SIZE = 5_000_000  # actually 5_242_880
+MAX_METRIC_REQUEST_SIZE = 1_000_000  # actually 1_048_576
+
 HTTP_BAD_REQUEST = 400
 
 
@@ -264,22 +264,13 @@ class HttpClient(CommunicationClient):
         return self.send_status(Status())
 
     def send_metrics(self, mint_lines: list[str]) -> list[MintResponse]:
-        total_lines = len(mint_lines)
-        lines_sent = 0
-
-        self.logger.debug(f"Start sending {total_lines} metrics to the EEC")
         responses = []
 
-        # We divide into chunks of MAX_MINT_LINES_PER_REQUEST lines to avoid hitting the body size limit
-        chunks = divide_into_chunks(mint_lines, MAX_MINT_LINES_PER_REQUEST)
-
-        for chunk in chunks:
-            lines_in_chunk = len(chunk)
-            lines_sent += lines_in_chunk
-            self.logger.debug(f"Sending chunk with {lines_in_chunk} metric lines. ({lines_sent}/{total_lines})")
-            mint_data = "\n".join(chunk).encode("utf-8")
+        # We divide into batches of MAX_METRIC_REQUEST_SIZE bytes to avoid hitting the body size limit
+        batches = divide_into_batches(mint_lines, MAX_METRIC_REQUEST_SIZE, "\n")
+        for batch in batches:
             response = self._make_request(
-                self._metric_url, "POST", mint_data, extra_headers={"Content-Type": CONTENT_TYPE_PLAIN}
+                self._metric_url, "POST", batch, extra_headers={"Content-Type": CONTENT_TYPE_PLAIN}
             ).json()
             self.logger.debug(f"{self._metric_url}: {response}")
             mint_response = MintResponse.from_json(response)
@@ -290,15 +281,16 @@ class HttpClient(CommunicationClient):
         self.logger.debug(f"Sending log events: {events}")
 
         responses = []
-        batches = divide_logs_into_batches([events] if isinstance(events, dict) else events)
+        if isinstance(events, dict):
+            events = [events]
+        batches = divide_into_batches(events, MAX_LOG_REQUEST_SIZE)
 
         for batch in batches:
             try:
-                encoded_batch = json.dumps(batch).encode("utf-8")
                 eec_response = self._make_request(
                     self._events_url,
                     "POST",
-                    encoded_batch,
+                    batch,
                     extra_headers={"Content-Type": CONTENT_TYPE_JSON, "eec-enrichment": str(eec_enrichment).lower()},
                 ).json()
                 responses.append(eec_response)
@@ -399,24 +391,17 @@ class DebugClient(CommunicationClient):
 
     def send_metrics(self, mint_lines: list[str]) -> list[MintResponse]:
         total_lines = len(mint_lines)
-        lines_sent = 0
-
         self.logger.info(f"Start sending {total_lines} metrics to the EEC")
 
         responses = []
 
-        chunks = divide_into_chunks(mint_lines, MAX_MINT_LINES_PER_REQUEST)
-        for chunk in chunks:
-            lines_in_chunk = len(chunk)
-            lines_sent += lines_in_chunk
-            self.logger.debug(f"Sending chunk with {lines_in_chunk} metric lines. ({lines_sent}/{total_lines})")
-
+        batches = divide_into_batches(mint_lines, MAX_METRIC_REQUEST_SIZE)
+        for batch in batches:
             if self.local_ingest:
-                mint_data = "\n".join(chunk).encode("utf-8")
                 response = request(
                     "POST",
                     f"http://localhost:{self.local_ingest_port}/metrics/ingest",
-                    body=mint_data,
+                    body=batch,
                     headers={"Content-Type": CONTENT_TYPE_PLAIN},
                 ).json()
                 mint_response = MintResponse.from_json(response)
@@ -426,15 +411,13 @@ class DebugClient(CommunicationClient):
                     for line in mint_lines:
                         self.logger.info(f"send_metric: {line}")
 
-                response = MintResponse(lines_invalid=0, lines_ok=len(chunk), error=None, warnings=None)
-                responses.append(response)
         return responses
 
     def send_events(self, events: dict | list[dict], eec_enrichment: bool = True) -> list[dict | None]:
         self.logger.info(f"send_events (enrichment = {eec_enrichment}): {len(events)} events")
         if self.print_metrics:
             for event in events:
-                self.logger.info(f"sendf_event: {event}")
+                self.logger.info(f"send_event: {event}")
         return []
 
     def send_sfm_metrics(self, mint_lines: list[str]) -> MintResponse:
@@ -446,60 +429,34 @@ class DebugClient(CommunicationClient):
         return 0
 
 
-def divide_into_chunks(iterable: Iterable, chunk_size: int) -> Iterable:
+def divide_into_batches(items: Sequence[dict | str], max_size_bytes: int, join_with: str | None = None) -> Generator[bytes, None, None]:
     """
-    Yield successive n-sized chunks from iterable.
-    Example: _chunk([1, 2, 3, 4, 5, 6, 7, 8, 9], 3) -> [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
+    Yield successive batches from a list, according to sizing limitations
 
-    :param iterable: The iterable to chunk
-    :param chunk_size: The size of the chunks
-    """
-    iterator = iter(iterable)
-    while True:
-        subset = list(islice(iterator, chunk_size))
-        if not subset:
-            return
-        yield subset
-
-def divide_logs_into_batches(logs: list[dict]):
+    :param items: The list items to divide, they myst be encodable to bytes
+    :param max_size_bytes: The maximum size of the payload in bytes
+    :param join_with: A string to join the items with before encoding
+    :return: A generator of batches of log events already encoded
     """
-    Yield successive batches from a list of log events, according to sizing limitations
-    imposed by the EEC: 5 MB payload, 50,000 events
 
-    :param logs: The list of log events
-    """
-    events_left = len(logs)
-    events = deque(logs)
-
-    batch = []
-    batch_size = 0
-    batch_items = 0
-
-    while events_left > 0:
-        if batch_items == MAX_LOG_EVENTS_PER_REQUEST:
-            yield batch
-            batch = []
-            batch_size = 0
-            batch_items = 0
-            continue
-
-        event = events.popleft()
-        events_left -= 1
-
-        if event is not None:
-            event_size = len(event)
-
-            if batch_size + event_size >= MAX_LOG_REQUEST_SIZE:
-                yield batch
-                batch = [event]
-                batch_size = event_size
-                batch_items = 1
-            else:
-                batch.append(event)
-                batch_size += event_size
-                batch_items += 1
-    else:
-        yield batch
+    if not items:
+        return
+
+    if join_with is not None:
+        items = join_with.join(items)
+    encoded = f"{items}".encode(errors="replace")
+    size = len(encoded)
+    if size <= max_size_bytes:
+        yield encoded
+        return
+
+    # if we get here, the payload is too large, split it in half until we have chunks that are small enough
+    half = len(items) // 2
+    first_half = items[:half]
+    second_half = items[half:]
+    yield from divide_into_batches(first_half, max_size_bytes)
+    yield from divide_into_batches(second_half, max_size_bytes)
+
 
 @dataclass
 class MintResponse: