deltacat 1.1.2__py3-none-any.whl → 1.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deltacat/__init__.py +1 -1
- deltacat/compute/compactor_v2/compaction_session.py +0 -10
- deltacat/tests/utils/test_metrics.py +26 -99
- deltacat/utils/metrics.py +35 -18
- {deltacat-1.1.2.dist-info → deltacat-1.1.3.dist-info}/METADATA +1 -1
- {deltacat-1.1.2.dist-info → deltacat-1.1.3.dist-info}/RECORD +9 -9
- {deltacat-1.1.2.dist-info → deltacat-1.1.3.dist-info}/LICENSE +0 -0
- {deltacat-1.1.2.dist-info → deltacat-1.1.3.dist-info}/WHEEL +0 -0
- {deltacat-1.1.2.dist-info → deltacat-1.1.3.dist-info}/top_level.txt +0 -0
deltacat/__init__.py
CHANGED
@@ -65,7 +65,6 @@ from deltacat.compute.compactor_v2.utils.task_options import (
|
|
65
65
|
local_merge_resource_options_provider,
|
66
66
|
)
|
67
67
|
from deltacat.compute.compactor.model.compactor_version import CompactorVersion
|
68
|
-
from deltacat.utils.metrics import MetricsActor, METRICS_CONFIG_ACTOR_NAME
|
69
68
|
|
70
69
|
if importlib.util.find_spec("memray"):
|
71
70
|
import memray
|
@@ -119,15 +118,6 @@ def _execute_compaction(
|
|
119
118
|
params: CompactPartitionParams, **kwargs
|
120
119
|
) -> Tuple[Optional[Partition], Optional[RoundCompletionInfo], Optional[str]]:
|
121
120
|
|
122
|
-
if params.metrics_config:
|
123
|
-
logger.info(
|
124
|
-
f"Setting metrics config with target: {params.metrics_config.metrics_target}"
|
125
|
-
)
|
126
|
-
metrics_actor = MetricsActor.options(
|
127
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
128
|
-
).remote()
|
129
|
-
ray.get(metrics_actor.set_metrics_config.remote(params.metrics_config))
|
130
|
-
|
131
121
|
rcf_source_partition_locator = (
|
132
122
|
params.rebase_source_partition_locator or params.source_partition_locator
|
133
123
|
)
|
@@ -1,13 +1,11 @@
|
|
1
1
|
import unittest
|
2
2
|
from unittest.mock import MagicMock, ANY, call
|
3
|
-
import ray
|
4
3
|
from deltacat.utils.metrics import (
|
5
4
|
metrics,
|
6
5
|
success_metric,
|
7
6
|
failure_metric,
|
8
7
|
latency_metric,
|
9
|
-
|
10
|
-
METRICS_CONFIG_ACTOR_NAME,
|
8
|
+
MetricsConfigCache,
|
11
9
|
MetricsConfig,
|
12
10
|
MetricsTarget,
|
13
11
|
METRICS_TARGET_TO_EMITTER_DICT,
|
@@ -43,11 +41,8 @@ class TestMetricsAnnotation(unittest.TestCase):
|
|
43
41
|
def test_metrics_annotation_sanity(self):
|
44
42
|
mock, mock_target = MagicMock(), MagicMock()
|
45
43
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
46
|
-
metrics_actor = MetricsActor.options(
|
47
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
48
|
-
).remote()
|
49
44
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
50
|
-
|
45
|
+
MetricsConfigCache.metrics_config = config
|
51
46
|
|
52
47
|
# action
|
53
48
|
metrics_annotated_method(mock)
|
@@ -73,11 +68,8 @@ class TestMetricsAnnotation(unittest.TestCase):
|
|
73
68
|
def test_metrics_annotation_when_error(self):
|
74
69
|
mock, mock_target = MagicMock(), MagicMock()
|
75
70
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
76
|
-
metrics_actor = MetricsActor.options(
|
77
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
78
|
-
).remote()
|
79
71
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
80
|
-
|
72
|
+
MetricsConfigCache.metrics_config = config
|
81
73
|
|
82
74
|
# action
|
83
75
|
self.assertRaises(ValueError, lambda: metrics_annotated_method_error(mock))
|
@@ -108,11 +100,8 @@ class TestMetricsAnnotation(unittest.TestCase):
|
|
108
100
|
def test_metrics_with_prefix_annotation_sanity(self):
|
109
101
|
mock, mock_target = MagicMock(), MagicMock()
|
110
102
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
111
|
-
metrics_actor = MetricsActor.options(
|
112
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
113
|
-
).remote()
|
114
103
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
115
|
-
|
104
|
+
MetricsConfigCache.metrics_config = config
|
116
105
|
|
117
106
|
# action
|
118
107
|
metrics_with_prefix_annotated_method(mock)
|
@@ -134,11 +123,8 @@ class TestMetricsAnnotation(unittest.TestCase):
|
|
134
123
|
def test_metrics_annotation_performance(self):
|
135
124
|
mock, mock_target = MagicMock(), MagicMock()
|
136
125
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
137
|
-
metrics_actor = MetricsActor.options(
|
138
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
139
|
-
).remote()
|
140
126
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
141
|
-
|
127
|
+
MetricsConfigCache.metrics_config = config
|
142
128
|
|
143
129
|
# action with annotation
|
144
130
|
_, actual_latency = timed_invocation(metrics_annotated_method, mock)
|
@@ -155,11 +141,8 @@ class TestMetricsAnnotation(unittest.TestCase):
|
|
155
141
|
def test_metrics_with_prefix_annotation_when_error(self):
|
156
142
|
mock, mock_target = MagicMock(), MagicMock()
|
157
143
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
158
|
-
metrics_actor = MetricsActor.options(
|
159
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
160
|
-
).remote()
|
161
144
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
162
|
-
|
145
|
+
MetricsConfigCache.metrics_config = config
|
163
146
|
|
164
147
|
# action
|
165
148
|
self.assertRaises(
|
@@ -185,15 +168,10 @@ class TestMetricsAnnotation(unittest.TestCase):
|
|
185
168
|
any_order=True,
|
186
169
|
)
|
187
170
|
|
188
|
-
def
|
171
|
+
def test_metrics_with_prefix_annotation_without_config(self):
|
189
172
|
mock, mock_target = MagicMock(), MagicMock()
|
190
173
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
191
|
-
|
192
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
193
|
-
).remote()
|
194
|
-
|
195
|
-
# explicitly making sure actor is killed
|
196
|
-
ray.kill(metrics_actor)
|
174
|
+
MetricsConfigCache.metrics_config = None
|
197
175
|
|
198
176
|
# action
|
199
177
|
self.assertRaises(
|
@@ -228,11 +206,8 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
|
|
228
206
|
def test_annotation_sanity(self):
|
229
207
|
mock, mock_target = MagicMock(), MagicMock()
|
230
208
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
231
|
-
metrics_actor = MetricsActor.options(
|
232
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
233
|
-
).remote()
|
234
209
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
235
|
-
|
210
|
+
MetricsConfigCache.metrics_config = config
|
236
211
|
|
237
212
|
# action
|
238
213
|
latency_metric_annotated_method(mock)
|
@@ -253,11 +228,8 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
|
|
253
228
|
def test_annotation_when_error(self):
|
254
229
|
mock, mock_target = MagicMock(), MagicMock()
|
255
230
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
256
|
-
metrics_actor = MetricsActor.options(
|
257
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
258
|
-
).remote()
|
259
231
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
260
|
-
|
232
|
+
MetricsConfigCache.metrics_config = config
|
261
233
|
|
262
234
|
# action
|
263
235
|
self.assertRaises(
|
@@ -280,11 +252,8 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
|
|
280
252
|
def test_annotation_with_args_sanity(self):
|
281
253
|
mock, mock_target = MagicMock(), MagicMock()
|
282
254
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
283
|
-
metrics_actor = MetricsActor.options(
|
284
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
285
|
-
).remote()
|
286
255
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
287
|
-
|
256
|
+
MetricsConfigCache.metrics_config = config
|
288
257
|
|
289
258
|
# action
|
290
259
|
latency_metric_with_name_annotated_method(mock)
|
@@ -298,11 +267,8 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
|
|
298
267
|
def test_annotation_with_args_when_error(self):
|
299
268
|
mock, mock_target = MagicMock(), MagicMock()
|
300
269
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
301
|
-
metrics_actor = MetricsActor.options(
|
302
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
303
|
-
).remote()
|
304
270
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
305
|
-
|
271
|
+
MetricsConfigCache.metrics_config = config
|
306
272
|
|
307
273
|
# action
|
308
274
|
self.assertRaises(
|
@@ -315,15 +281,10 @@ class TestLatencyMetricAnnotation(unittest.TestCase):
|
|
315
281
|
[call(metrics_name="test", metrics_config=ANY, value=ANY)], any_order=True
|
316
282
|
)
|
317
283
|
|
318
|
-
def
|
284
|
+
def test_annotation_without_config(self):
|
319
285
|
mock, mock_target = MagicMock(), MagicMock()
|
320
286
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
321
|
-
|
322
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
323
|
-
).remote()
|
324
|
-
|
325
|
-
# explicitly making sure actor is killed
|
326
|
-
ray.kill(metrics_actor)
|
287
|
+
MetricsConfigCache.metrics_config = None
|
327
288
|
|
328
289
|
# action
|
329
290
|
self.assertRaises(
|
@@ -358,11 +319,8 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
|
|
358
319
|
def test_annotation_sanity(self):
|
359
320
|
mock, mock_target = MagicMock(), MagicMock()
|
360
321
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
361
|
-
metrics_actor = MetricsActor.options(
|
362
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
363
|
-
).remote()
|
364
322
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
365
|
-
|
323
|
+
MetricsConfigCache.metrics_config = config
|
366
324
|
|
367
325
|
# action
|
368
326
|
success_metric_annotated_method(mock)
|
@@ -383,11 +341,8 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
|
|
383
341
|
def test_annotation_when_error(self):
|
384
342
|
mock, mock_target = MagicMock(), MagicMock()
|
385
343
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
386
|
-
metrics_actor = MetricsActor.options(
|
387
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
388
|
-
).remote()
|
389
344
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
390
|
-
|
345
|
+
MetricsConfigCache.metrics_config = config
|
391
346
|
|
392
347
|
# action
|
393
348
|
self.assertRaises(
|
@@ -400,11 +355,8 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
|
|
400
355
|
def test_annotation_with_args_sanity(self):
|
401
356
|
mock, mock_target = MagicMock(), MagicMock()
|
402
357
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
403
|
-
metrics_actor = MetricsActor.options(
|
404
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
405
|
-
).remote()
|
406
358
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
407
|
-
|
359
|
+
MetricsConfigCache.metrics_config = config
|
408
360
|
|
409
361
|
# action
|
410
362
|
success_metric_with_name_annotated_method(mock)
|
@@ -418,11 +370,8 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
|
|
418
370
|
def test_annotation_with_args_when_error(self):
|
419
371
|
mock, mock_target = MagicMock(), MagicMock()
|
420
372
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
421
|
-
metrics_actor = MetricsActor.options(
|
422
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
423
|
-
).remote()
|
424
373
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
425
|
-
|
374
|
+
MetricsConfigCache.metrics_config = config
|
426
375
|
|
427
376
|
# action
|
428
377
|
self.assertRaises(
|
@@ -432,15 +381,10 @@ class TestSuccessMetricAnnotation(unittest.TestCase):
|
|
432
381
|
mock.assert_not_called()
|
433
382
|
mock_target.assert_not_called()
|
434
383
|
|
435
|
-
def
|
384
|
+
def test_annotation_without_config(self):
|
436
385
|
mock, mock_target = MagicMock(), MagicMock()
|
437
386
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
438
|
-
|
439
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
440
|
-
).remote()
|
441
|
-
|
442
|
-
# explicitly making sure actor is killed
|
443
|
-
ray.kill(metrics_actor)
|
387
|
+
MetricsConfigCache.metrics_config = None
|
444
388
|
|
445
389
|
# action
|
446
390
|
success_metric_annotated_method(mock)
|
@@ -473,11 +417,8 @@ class TestFailureMetricAnnotation(unittest.TestCase):
|
|
473
417
|
def test_annotation_sanity(self):
|
474
418
|
mock, mock_target = MagicMock(), MagicMock()
|
475
419
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
476
|
-
metrics_actor = MetricsActor.options(
|
477
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
478
|
-
).remote()
|
479
420
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
480
|
-
|
421
|
+
MetricsConfigCache.metrics_config = config
|
481
422
|
|
482
423
|
# action
|
483
424
|
failure_metric_annotated_method(mock)
|
@@ -488,11 +429,8 @@ class TestFailureMetricAnnotation(unittest.TestCase):
|
|
488
429
|
def test_annotation_when_error(self):
|
489
430
|
mock, mock_target = MagicMock(), MagicMock()
|
490
431
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
491
|
-
metrics_actor = MetricsActor.options(
|
492
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
493
|
-
).remote()
|
494
432
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
495
|
-
|
433
|
+
MetricsConfigCache.metrics_config = config
|
496
434
|
|
497
435
|
# action
|
498
436
|
self.assertRaises(
|
@@ -520,11 +458,8 @@ class TestFailureMetricAnnotation(unittest.TestCase):
|
|
520
458
|
def test_annotation_with_args_sanity(self):
|
521
459
|
mock, mock_target = MagicMock(), MagicMock()
|
522
460
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
523
|
-
metrics_actor = MetricsActor.options(
|
524
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
525
|
-
).remote()
|
526
461
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
527
|
-
|
462
|
+
MetricsConfigCache.metrics_config = config
|
528
463
|
|
529
464
|
# action
|
530
465
|
failure_metric_with_name_annotated_method(mock)
|
@@ -535,11 +470,8 @@ class TestFailureMetricAnnotation(unittest.TestCase):
|
|
535
470
|
def test_annotation_with_args_when_error(self):
|
536
471
|
mock, mock_target = MagicMock(), MagicMock()
|
537
472
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
538
|
-
metrics_actor = MetricsActor.options(
|
539
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
540
|
-
).remote()
|
541
473
|
config = MetricsConfig("us-east-1", MetricsTarget.NOOP)
|
542
|
-
|
474
|
+
MetricsConfigCache.metrics_config = config
|
543
475
|
|
544
476
|
# action
|
545
477
|
self.assertRaises(
|
@@ -556,15 +488,10 @@ class TestFailureMetricAnnotation(unittest.TestCase):
|
|
556
488
|
any_order=True,
|
557
489
|
)
|
558
490
|
|
559
|
-
def
|
491
|
+
def test_annotation_without_config(self):
|
560
492
|
mock, mock_target = MagicMock(), MagicMock()
|
561
493
|
METRICS_TARGET_TO_EMITTER_DICT[MetricsTarget.NOOP] = mock_target
|
562
|
-
|
563
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
564
|
-
).remote()
|
565
|
-
|
566
|
-
# explicitly making sure actor is killed
|
567
|
-
ray.kill(metrics_actor)
|
494
|
+
MetricsConfigCache.metrics_config = None
|
568
495
|
|
569
496
|
# action
|
570
497
|
self.assertRaises(
|
deltacat/utils/metrics.py
CHANGED
@@ -2,7 +2,6 @@ from dataclasses import dataclass
|
|
2
2
|
from typing import Optional
|
3
3
|
|
4
4
|
import logging
|
5
|
-
import ray
|
6
5
|
import time
|
7
6
|
import functools
|
8
7
|
|
@@ -22,7 +21,6 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
|
|
22
21
|
DEFAULT_DELTACAT_METRICS_NAMESPACE = "ray-deltacat-metrics"
|
23
22
|
DEFAULT_DELTACAT_LOG_GROUP_NAME = "ray-deltacat-metrics-EMF-logs"
|
24
23
|
DEFAULT_DELTACAT_LOG_STREAM_CALLABLE = get_node_ip_address
|
25
|
-
METRICS_CONFIG_ACTOR_NAME = "metrics-config-actor"
|
26
24
|
|
27
25
|
|
28
26
|
class MetricsTarget(str, Enum):
|
@@ -187,23 +185,13 @@ METRICS_TARGET_TO_EMITTER_DICT: Dict[str, Callable] = {
|
|
187
185
|
}
|
188
186
|
|
189
187
|
|
190
|
-
|
191
|
-
class MetricsActor:
|
188
|
+
class MetricsConfigCache:
|
192
189
|
metrics_config: MetricsConfig = None
|
193
190
|
|
194
|
-
def set_metrics_config(self, config: MetricsConfig) -> None:
|
195
|
-
self.metrics_config = config
|
196
|
-
|
197
|
-
def get_metrics_config(self) -> MetricsConfig:
|
198
|
-
return self.metrics_config
|
199
|
-
|
200
191
|
|
201
192
|
def _emit_ignore_exceptions(name: Optional[str], value: Any):
|
202
193
|
try:
|
203
|
-
|
204
|
-
name=METRICS_CONFIG_ACTOR_NAME, get_if_exists=True
|
205
|
-
).remote()
|
206
|
-
config = ray.get(config_cache.get_metrics_config.remote())
|
194
|
+
config = MetricsConfigCache.metrics_config
|
207
195
|
if config:
|
208
196
|
_emit_metrics(metrics_name=name, value=value, metrics_config=config)
|
209
197
|
except BaseException as ex:
|
@@ -222,7 +210,14 @@ def emit_timer_metrics(metrics_name, value, metrics_config, **kwargs):
|
|
222
210
|
def latency_metric(original_func=None, name: Optional[str] = None):
|
223
211
|
"""
|
224
212
|
A decorator that emits latency metrics of a function
|
225
|
-
based on configured metrics config.
|
213
|
+
based on configured metrics config. Hence, make sure to set it in all worker processes:
|
214
|
+
|
215
|
+
def setup_cache():
|
216
|
+
from deltacat.utils.metrics import MetricsConfigCache
|
217
|
+
MetricsConfigCache.metrics_config = metrics_config
|
218
|
+
|
219
|
+
setup_cache();
|
220
|
+
ray.init(address="auto", runtime_env={"worker_process_setup_hook": setup_cache})
|
226
221
|
"""
|
227
222
|
|
228
223
|
def _decorate(func):
|
@@ -248,7 +243,14 @@ def latency_metric(original_func=None, name: Optional[str] = None):
|
|
248
243
|
def success_metric(original_func=None, name: Optional[str] = None):
|
249
244
|
"""
|
250
245
|
A decorator that emits success metrics of a function
|
251
|
-
based on configured metrics config.
|
246
|
+
based on configured metrics config. Hence, make sure to set it in all worker processes:
|
247
|
+
|
248
|
+
def setup_cache():
|
249
|
+
from deltacat.utils.metrics import MetricsConfigCache
|
250
|
+
MetricsConfigCache.metrics_config = metrics_config
|
251
|
+
|
252
|
+
setup_cache();
|
253
|
+
ray.init(address="auto", runtime_env={"worker_process_setup_hook": setup_cache})
|
252
254
|
"""
|
253
255
|
|
254
256
|
def _decorate(func):
|
@@ -271,7 +273,14 @@ def success_metric(original_func=None, name: Optional[str] = None):
|
|
271
273
|
def failure_metric(original_func=None, name: Optional[str] = None):
|
272
274
|
"""
|
273
275
|
A decorator that emits failure metrics of a function
|
274
|
-
based on configured metrics config.
|
276
|
+
based on configured metrics config. Hence, make sure to set it in all worker processes:
|
277
|
+
|
278
|
+
def setup_cache():
|
279
|
+
from deltacat.utils.metrics import MetricsConfigCache
|
280
|
+
MetricsConfigCache.metrics_config = metrics_config
|
281
|
+
|
282
|
+
setup_cache();
|
283
|
+
ray.init(address="auto", runtime_env={"worker_process_setup_hook": setup_cache})
|
275
284
|
"""
|
276
285
|
|
277
286
|
def _decorate(func):
|
@@ -301,7 +310,15 @@ def failure_metric(original_func=None, name: Optional[str] = None):
|
|
301
310
|
|
302
311
|
def metrics(original_func=None, prefix: Optional[str] = None):
|
303
312
|
"""
|
304
|
-
A decorator that emits all metrics for a function.
|
313
|
+
A decorator that emits all metrics for a function. This decorator depends
|
314
|
+
on a metrics config. Hence, make sure to set it in all worker processes:
|
315
|
+
|
316
|
+
def setup_cache():
|
317
|
+
from deltacat.utils.metrics import MetricsConfigCache
|
318
|
+
MetricsConfigCache.metrics_config = metrics_config
|
319
|
+
|
320
|
+
setup_cache();
|
321
|
+
ray.init(address="auto", runtime_env={"worker_process_setup_hook": setup_cache})
|
305
322
|
"""
|
306
323
|
|
307
324
|
def _decorate(func):
|
@@ -1,4 +1,4 @@
|
|
1
|
-
deltacat/__init__.py,sha256=
|
1
|
+
deltacat/__init__.py,sha256=eO5VKENAHF-beW-5X9eJHbQk9EvjaQ09PiyxhQNUuxo,1777
|
2
2
|
deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
|
3
3
|
deltacat/exceptions.py,sha256=xqZf8CwysNYP2d39pf27OnXGStPREgBgIM-e2Tts-TI,199
|
4
4
|
deltacat/logs.py,sha256=YT26oj-oKZReyBEhiBQU5jc246aSu_d4B2bZcUVBHmE,6550
|
@@ -50,7 +50,7 @@ deltacat/compute/compactor/utils/round_completion_file.py,sha256=DmZfHeAXlQn0DDd
|
|
50
50
|
deltacat/compute/compactor/utils/sort_key.py,sha256=oK6otg-CSsma6zlGPaKg-KNEvcZRG2NqBlCw1X3_FBc,2397
|
51
51
|
deltacat/compute/compactor/utils/system_columns.py,sha256=CNIgAGos0xAGEpdaQIH7KfbSRrGZgjRbItXMararqXQ,9399
|
52
52
|
deltacat/compute/compactor_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
53
|
-
deltacat/compute/compactor_v2/compaction_session.py,sha256=
|
53
|
+
deltacat/compute/compactor_v2/compaction_session.py,sha256=xAxiTOxHffRvPqkv4ObGM-NWdpxRRMyrS8WWZWrgTFQ,25141
|
54
54
|
deltacat/compute/compactor_v2/constants.py,sha256=R5qvsaX-rDrSmSqUl0yLp6A3hVrAD4whYkV2NBkDR9Y,2199
|
55
55
|
deltacat/compute/compactor_v2/deletes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
56
56
|
deltacat/compute/compactor_v2/deletes/delete_file_envelope.py,sha256=AeuH9JRMwp6mvQf6P2cqL92hUEtResQq6qUTS0kIKac,3111
|
@@ -174,7 +174,7 @@ deltacat/tests/test_utils/utils.py,sha256=a32qEwcSSd1lvRi0aJJ4ZLnc1ZyXmoQF_K95za
|
|
174
174
|
deltacat/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
175
175
|
deltacat/tests/utils/test_cloudpickle.py,sha256=J0pnBY3-PxlUh6MamZAN1PuquKQPr2iyzjiJ7-Rcl0o,1506
|
176
176
|
deltacat/tests/utils/test_daft.py,sha256=Xal84zR42rXsWQI3lImdDYWOzewomKmhmiUQ59m67V0,6488
|
177
|
-
deltacat/tests/utils/test_metrics.py,sha256=
|
177
|
+
deltacat/tests/utils/test_metrics.py,sha256=Ym9nOz1EtB180pLmvugihj1sDTNDMb5opIjjr5Nmcls,16339
|
178
178
|
deltacat/tests/utils/test_pyarrow.py,sha256=eZAuYp9MUf8lmpIilH57JkURuNsTGZ3IAGC4Gm5hdrM,17307
|
179
179
|
deltacat/tests/utils/test_record_batch_tables.py,sha256=AkG1WyljQmjnl-AxhbFWyo5LnMIKRyLScfgC2B_ES-s,11321
|
180
180
|
deltacat/tests/utils/test_resources.py,sha256=HtpvDrfPZQNtGDXUlsIzc_yd7Vf1cDscZ3YbN0oTvO8,2560
|
@@ -188,7 +188,7 @@ deltacat/utils/arguments.py,sha256=5y1Xz4HSAD8M8Jt83i6gOEKoYjy_fMQe1V43IhIE4hY,1
|
|
188
188
|
deltacat/utils/cloudpickle.py,sha256=XE7YDmQe56ksfl3NdYZkzOAhbHSuhNcBZGOehQpgZr0,1187
|
189
189
|
deltacat/utils/common.py,sha256=RG_-enXNpLKaYrqyx1ne2lL10lxN9vK7F631oJP6SE8,1375
|
190
190
|
deltacat/utils/daft.py,sha256=heg6J8NHnTI1UbcjXvsjGBbFKU6f7mvAio-KZuTSuFI,5506
|
191
|
-
deltacat/utils/metrics.py,sha256=
|
191
|
+
deltacat/utils/metrics.py,sha256=HYKyZSrtVLu8gXezg_TMNUKJp4h1WWI0VEzn0Xlzf-I,10778
|
192
192
|
deltacat/utils/numpy.py,sha256=ZiGREobTVT6IZXgPxkSUpLJFN2Hn8KEZcrqybLDXCIA,2027
|
193
193
|
deltacat/utils/pandas.py,sha256=GfwjYb8FUSEeoBdXZI1_NJkdjxPMbCCUhlyRfGbDkn8,9562
|
194
194
|
deltacat/utils/performance.py,sha256=7ZLaMkS1ehPSIhT5uOQVBHvjC70iKHzoFquFo-KL0PI,645
|
@@ -203,8 +203,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
|
|
203
203
|
deltacat/utils/ray_utils/dataset.py,sha256=SIljK3UkSqQ6Ntit_iSiYt9yYjN_gGrCTX6_72XdQ3w,3244
|
204
204
|
deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
|
205
205
|
deltacat/utils/ray_utils/runtime.py,sha256=5eaBWTDm0IXVoc5Y6aacoVB-f0Mnv-K2ewyTSjHKHwM,5009
|
206
|
-
deltacat-1.1.
|
207
|
-
deltacat-1.1.
|
208
|
-
deltacat-1.1.
|
209
|
-
deltacat-1.1.
|
210
|
-
deltacat-1.1.
|
206
|
+
deltacat-1.1.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
207
|
+
deltacat-1.1.3.dist-info/METADATA,sha256=uj1xTijXxJlOZQy38MbY6Axj7cMjsT75xlKjWZPJyzw,1780
|
208
|
+
deltacat-1.1.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
209
|
+
deltacat-1.1.3.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
|
210
|
+
deltacat-1.1.3.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|