sentry-arroyo 2.31.1__tar.gz → 2.31.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {sentry_arroyo-2.31.1/sentry_arroyo.egg-info → sentry_arroyo-2.31.2}/PKG-INFO +1 -1
  2. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/kafka/consumer.py +36 -11
  3. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/processor.py +3 -3
  4. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2/sentry_arroyo.egg-info}/PKG-INFO +1 -1
  5. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/setup.py +1 -1
  6. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/backends/test_confluent_producer.py +3 -0
  7. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/LICENSE +0 -0
  8. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/MANIFEST.in +0 -0
  9. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/README.md +0 -0
  10. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/__init__.py +0 -0
  11. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/__init__.py +0 -0
  12. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/abstract.py +0 -0
  13. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/kafka/__init__.py +0 -0
  14. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/kafka/commit.py +0 -0
  15. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/kafka/configuration.py +0 -0
  16. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/local/__init__.py +0 -0
  17. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/local/backend.py +0 -0
  18. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/local/storages/__init__.py +0 -0
  19. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/local/storages/abstract.py +0 -0
  20. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/backends/local/storages/memory.py +0 -0
  21. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/commit.py +0 -0
  22. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/dlq.py +0 -0
  23. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/errors.py +0 -0
  24. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/__init__.py +0 -0
  25. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/__init__.py +0 -0
  26. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/abstract.py +0 -0
  27. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/batching.py +0 -0
  28. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/buffer.py +0 -0
  29. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/commit.py +0 -0
  30. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/filter.py +0 -0
  31. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/guard.py +0 -0
  32. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/healthcheck.py +0 -0
  33. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/noop.py +0 -0
  34. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/produce.py +0 -0
  35. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/reduce.py +0 -0
  36. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/run_task.py +0 -0
  37. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/run_task_in_threads.py +0 -0
  38. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/run_task_with_multiprocessing.py +0 -0
  39. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/processing/strategies/unfold.py +0 -0
  40. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/py.typed +0 -0
  41. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/types.py +0 -0
  42. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/__init__.py +0 -0
  43. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/clock.py +0 -0
  44. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/codecs.py +0 -0
  45. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/concurrent.py +0 -0
  46. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/logging.py +0 -0
  47. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/metricDefs.json +0 -0
  48. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/metric_defs.py +0 -0
  49. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/metrics.py +0 -0
  50. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/profiler.py +0 -0
  51. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/arroyo/utils/retries.py +0 -0
  52. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/examples/transform_and_produce/__init__.py +0 -0
  53. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/examples/transform_and_produce/batched.py +0 -0
  54. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/examples/transform_and_produce/script.py +0 -0
  55. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/examples/transform_and_produce/simple.py +0 -0
  56. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/requirements.txt +0 -0
  57. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/sentry_arroyo.egg-info/SOURCES.txt +0 -0
  58. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/sentry_arroyo.egg-info/dependency_links.txt +0 -0
  59. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/sentry_arroyo.egg-info/not-zip-safe +0 -0
  60. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/sentry_arroyo.egg-info/requires.txt +0 -0
  61. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/sentry_arroyo.egg-info/top_level.txt +0 -0
  62. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/setup.cfg +0 -0
  63. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/backends/__init__.py +0 -0
  64. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/backends/mixins.py +0 -0
  65. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/backends/test_commit.py +0 -0
  66. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/backends/test_kafka.py +0 -0
  67. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/backends/test_kafka_producer.py +0 -0
  68. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/backends/test_local.py +0 -0
  69. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/__init__.py +0 -0
  70. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/__init__.py +0 -0
  71. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_all.py +0 -0
  72. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_batching.py +0 -0
  73. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_buffer.py +0 -0
  74. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_commit.py +0 -0
  75. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_filter.py +0 -0
  76. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_guard.py +0 -0
  77. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_noop.py +0 -0
  78. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_produce.py +0 -0
  79. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_reduce.py +0 -0
  80. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_run_task.py +0 -0
  81. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_run_task_in_threads.py +0 -0
  82. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_run_task_with_multiprocessing.py +0 -0
  83. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/strategies/test_unfold.py +0 -0
  84. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/processing/test_processor.py +0 -0
  85. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/test_commit.py +0 -0
  86. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/test_dlq.py +0 -0
  87. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/test_kip848_e2e.py +0 -0
  88. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/test_types.py +0 -0
  89. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/utils/__init__.py +0 -0
  90. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/utils/test_concurrent.py +0 -0
  91. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/utils/test_metrics.py +0 -0
  92. {sentry_arroyo-2.31.1 → sentry_arroyo-2.31.2}/tests/utils/test_retries.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sentry-arroyo
3
- Version: 2.31.1
3
+ Version: 2.31.2
4
4
  Summary: Arroyo is a Python library for working with streaming data.
5
5
  Home-page: https://github.com/getsentry/arroyo
6
6
  Author: Sentry
@@ -1,6 +1,8 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import logging
4
+ import time
5
+ from collections import defaultdict
4
6
  from concurrent.futures import Future
5
7
  from datetime import datetime
6
8
  from enum import Enum
@@ -21,6 +23,7 @@ from typing import (
21
23
  Tuple,
22
24
  Type,
23
25
  Union,
26
+ cast,
24
27
  )
25
28
 
26
29
  from confluent_kafka import (
@@ -748,6 +751,9 @@ class KafkaProducer(Producer[KafkaPayload]):
748
751
  # Type alias for the delivery callback function
749
752
  DeliveryCallback = Callable[[Optional[KafkaError], ConfluentMessage], None]
750
753
 
754
+ # Interval between metric flushes (in seconds)
755
+ METRICS_FREQUENCY_SEC = 1.0
756
+
751
757
 
752
758
  class ConfluentProducer(ConfluentKafkaProducer): # type: ignore[misc]
753
759
  """
@@ -756,8 +762,10 @@ class ConfluentProducer(ConfluentKafkaProducer): # type: ignore[misc]
756
762
 
757
763
  def __init__(self, configuration: Mapping[str, Any]) -> None:
758
764
  super().__init__(configuration)
759
- self.__metrics = get_metrics()
760
765
  self.producer_name = configuration.get("client.id") or None
766
+ self.__metrics = get_metrics()
767
+ self.__produce_counters: MutableMapping[str, int] = defaultdict(int)
768
+ self.__reset_metrics()
761
769
 
762
770
  def __metrics_delivery_callback(
763
771
  self,
@@ -768,16 +776,8 @@ class ConfluentProducer(ConfluentKafkaProducer): # type: ignore[misc]
768
776
  status = "error"
769
777
  else:
770
778
  status = "success"
771
-
772
- tags = {"status": status}
773
- if self.producer_name:
774
- tags["producer_name"] = self.producer_name
775
-
776
- self.__metrics.increment(
777
- "arroyo.producer.produce_status",
778
- 1,
779
- tags=tags,
780
- )
779
+ self.__produce_counters[status] += 1
780
+ self.__throttled_record()
781
781
 
782
782
  def __delivery_callback(
783
783
  self,
@@ -797,3 +797,28 @@ class ConfluentProducer(ConfluentKafkaProducer): # type: ignore[misc]
797
797
  user_callback = callback or on_delivery
798
798
  wrapped_callback = self.__delivery_callback(user_callback)
799
799
  super().produce(*args, on_delivery=wrapped_callback, **kwargs)
800
+
801
+ def __flush_metrics(self) -> None:
802
+ for status, count in self.__produce_counters.items():
803
+ tags = {"status": status}
804
+ if self.producer_name:
805
+ tags["producer_name"] = self.producer_name
806
+ self.__metrics.increment(
807
+ name="arroyo.producer.produce_status",
808
+ value=count,
809
+ tags=tags,
810
+ )
811
+ self.__reset_metrics()
812
+
813
+ def flush(self, timeout: float = -1) -> int:
814
+ # Kafka producer flush should flush metrics too
815
+ self.__flush_metrics()
816
+ return cast(int, super().flush(timeout))
817
+
818
+ def __reset_metrics(self) -> None:
819
+ self.__produce_counters.clear()
820
+ self.__last_record_time = time.time()
821
+
822
+ def __throttled_record(self) -> None:
823
+ if time.time() - self.__last_record_time > METRICS_FREQUENCY_SEC:
824
+ self.__flush_metrics()
@@ -146,9 +146,9 @@ class StreamProcessor(Generic[TStrategyPayload]):
146
146
  self.__processor_factory = processor_factory
147
147
  self.__metrics_buffer = MetricsBuffer()
148
148
 
149
- self.__processing_strategy: Optional[
150
- ProcessingStrategy[TStrategyPayload]
151
- ] = None
149
+ self.__processing_strategy: Optional[ProcessingStrategy[TStrategyPayload]] = (
150
+ None
151
+ )
152
152
 
153
153
  self.__message: Optional[BrokerValue[TStrategyPayload]] = None
154
154
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sentry-arroyo
3
- Version: 2.31.1
3
+ Version: 2.31.2
4
4
  Summary: Arroyo is a Python library for working with streaming data.
5
5
  Home-page: https://github.com/getsentry/arroyo
6
6
  Author: Sentry
@@ -10,7 +10,7 @@ def get_requirements() -> Sequence[str]:
10
10
 
11
11
  setup(
12
12
  name="sentry-arroyo",
13
- version="2.31.1",
13
+ version="2.31.2",
14
14
  author="Sentry",
15
15
  author_email="oss@sentry.io",
16
16
  license="Apache-2.0",
@@ -29,6 +29,7 @@ class TestConfluentProducer:
29
29
  )
30
30
  mock_message = mock.Mock(spec=ConfluentMessage)
31
31
  producer._ConfluentProducer__metrics_delivery_callback(None, mock_message)
32
+ producer.flush() # Flush buffered metrics
32
33
  assert (
33
34
  Increment(
34
35
  "arroyo.producer.produce_status",
@@ -44,6 +45,7 @@ class TestConfluentProducer:
44
45
  mock_error = mock.Mock(spec=KafkaError)
45
46
  mock_message = mock.Mock(spec=ConfluentMessage)
46
47
  producer._ConfluentProducer__metrics_delivery_callback(mock_error, mock_message)
48
+ producer.flush() # Flush buffered metrics
47
49
  assert (
48
50
  Increment("arroyo.producer.produce_status", 1, {"status": "error"})
49
51
  in TestingMetricsBackend.calls
@@ -64,6 +66,7 @@ class TestConfluentProducer:
64
66
  wrapped = producer._ConfluentProducer__delivery_callback(user_callback)
65
67
  mock_message = mock.Mock(spec=ConfluentMessage)
66
68
  wrapped(None, mock_message)
69
+ producer.flush() # Flush buffered metrics
67
70
  assert (
68
71
  Increment(
69
72
  "arroyo.producer.produce_status",
File without changes
File without changes
File without changes