ga-api-client 0.3.3__tar.gz → 0.3.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ga_api_client-0.3.3/ga_api_client.egg-info → ga_api_client-0.3.5}/PKG-INFO +1 -1
- ga_api_client-0.3.5/ga_api/__init__.py +8 -0
- ga_api_client-0.3.5/ga_api/average.py +38 -0
- ga_api_client-0.3.5/ga_api/capture.py +144 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/ga_api/ga_api_client.py +66 -25
- ga_api_client-0.3.5/ga_api/rate.py +50 -0
- ga_api_client-0.3.5/ga_api/std.py +47 -0
- ga_api_client-0.3.5/ga_api/tdigest.py +189 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5/ga_api_client.egg-info}/PKG-INFO +1 -1
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/ga_api_client.egg-info/SOURCES.txt +5 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/setup.cfg +1 -1
- ga_api_client-0.3.3/ga_api/__init__.py +0 -3
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/LICENSE +0 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/README.md +0 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/ga_api/field_tree.py +0 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/ga_api/hyper_log_log.py +0 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/ga_api_client.egg-info/dependency_links.txt +0 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/ga_api_client.egg-info/requires.txt +0 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/ga_api_client.egg-info/top_level.txt +0 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/pyproject.toml +0 -0
- {ga_api_client-0.3.3 → ga_api_client-0.3.5}/setup.py +0 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
from .ga_api_client import System, Tenant, Repository, pprint
|
|
2
|
+
from .hyper_log_log import HyperLogLog
|
|
3
|
+
from .rate import Rate
|
|
4
|
+
from .capture import CaptureMin, CaptureMax, CaptureAnyMin, CaptureAnyMax
|
|
5
|
+
from .std import Std
|
|
6
|
+
from .average import Average
|
|
7
|
+
from .tdigest import TDigest
|
|
8
|
+
from .field_tree import FieldTable, field_table
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import math
|
|
2
|
+
|
|
3
|
+
class Average:
|
|
4
|
+
def __init__(self, arg=[0, 0]):
|
|
5
|
+
self._sum = arg[0]
|
|
6
|
+
self._count = arg[1]
|
|
7
|
+
def __add__(self, other):
|
|
8
|
+
return Average([self._sum + other._sum, self._count + other._count])
|
|
9
|
+
# return HyperLogLog(bytearray(max(a,b) for a, b in zip(self._rmem, other._rmem)))
|
|
10
|
+
|
|
11
|
+
def __radd__(self, other):
|
|
12
|
+
return Average([self._sum + other._sum, self._count + other._count])
|
|
13
|
+
# return HyperLogLog(bytearray(max(a,b) for a, b in zip(self._rmem, other._rmem)))
|
|
14
|
+
|
|
15
|
+
def __iadd__(self, other):
|
|
16
|
+
self._sum += other._sum
|
|
17
|
+
self._count += other._count
|
|
18
|
+
return self
|
|
19
|
+
|
|
20
|
+
def __str__(self):
|
|
21
|
+
return f'{self.value()}'
|
|
22
|
+
|
|
23
|
+
def __repr__(self):
|
|
24
|
+
return f'$avg(sum={self._sum}, count={self._count})'
|
|
25
|
+
|
|
26
|
+
def value(self):
|
|
27
|
+
if self._count == 0:
|
|
28
|
+
return math.nan
|
|
29
|
+
return self._sum / self._count
|
|
30
|
+
|
|
31
|
+
if __name__ == '__main__':
|
|
32
|
+
x = Average([10, 2])
|
|
33
|
+
y = Average([10, 5])
|
|
34
|
+
print('x = ', x)
|
|
35
|
+
print('y = ', y)
|
|
36
|
+
print('x + y = ', x + y)
|
|
37
|
+
print('repr(x) = ', repr(x))
|
|
38
|
+
print('repr(y) = ', repr(y))
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
from datetime import datetime, timezone
|
|
2
|
+
class CaptureBase:
|
|
3
|
+
def __init__(self, value=None, time=None):
|
|
4
|
+
# 如果 value 是列表,則分配 _v 和 _t
|
|
5
|
+
if isinstance(value, list):
|
|
6
|
+
time = value[0]
|
|
7
|
+
self._v = value[1]
|
|
8
|
+
else:
|
|
9
|
+
self._v = value
|
|
10
|
+
|
|
11
|
+
# convert _v to datetime object
|
|
12
|
+
self._v = self._parse_datetime(self._v)
|
|
13
|
+
|
|
14
|
+
# convert _t to datetime object
|
|
15
|
+
self._t = self._parse_datetime(time)
|
|
16
|
+
|
|
17
|
+
if self._t is None:
|
|
18
|
+
self._t = datetime.fromtimestamp(0, tz=timezone.utc)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _parse_datetime(self, date_str):
|
|
22
|
+
if isinstance(date_str, str) and date_str.startswith('d:'):
|
|
23
|
+
try:
|
|
24
|
+
return datetime.fromisoformat(date_str[2:])
|
|
25
|
+
except ValueError as e:
|
|
26
|
+
raise ValueError(f"Invalid datetime format: {date_str}") from e
|
|
27
|
+
return date_str
|
|
28
|
+
|
|
29
|
+
def compare(self, other):
|
|
30
|
+
raise NotImplementedError("Subclasses must implement `compare` method.")
|
|
31
|
+
|
|
32
|
+
def __add__(self, other):
|
|
33
|
+
if self._t is None:
|
|
34
|
+
return self.__class__([other._v, other._t])
|
|
35
|
+
if other._t is None:
|
|
36
|
+
return self.__class__([self._v, self._t])
|
|
37
|
+
return self.compare(other)
|
|
38
|
+
|
|
39
|
+
def __radd__(self, other):
|
|
40
|
+
return self + other
|
|
41
|
+
|
|
42
|
+
def __iadd__(self, other):
|
|
43
|
+
if self._t is None:
|
|
44
|
+
self._v = other._v
|
|
45
|
+
self._t = other._t
|
|
46
|
+
elif other._t is not None:
|
|
47
|
+
result = self.compare(other)
|
|
48
|
+
self._v = result._v
|
|
49
|
+
self._t = result._t
|
|
50
|
+
return self
|
|
51
|
+
|
|
52
|
+
def __str__(self):
|
|
53
|
+
return f'{self._v}'
|
|
54
|
+
|
|
55
|
+
def __repr__(self):
|
|
56
|
+
return f'{self.__class__.__name__}(v={repr(self._v)}, t={repr(self._t)})'
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class CaptureAnyMin(CaptureBase):
|
|
60
|
+
def compare(self, other):
|
|
61
|
+
print(repr(self), repr(other))
|
|
62
|
+
if self._t is None:
|
|
63
|
+
return self.__class__([other._t, other._v])
|
|
64
|
+
elif other._t is None:
|
|
65
|
+
return self.__class__([self._t, self._v])
|
|
66
|
+
elif self._t < other._t:
|
|
67
|
+
return self.__class__([self._t, self._v])
|
|
68
|
+
return self.__class__([other._t, other._v])
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class CaptureAnyMax(CaptureBase):
|
|
72
|
+
def compare(self, other):
|
|
73
|
+
print(repr(self), repr(other))
|
|
74
|
+
if self._t is None:
|
|
75
|
+
return self.__class__([other._t, other._v])
|
|
76
|
+
elif other._t is None:
|
|
77
|
+
return self.__class__([self._t, self._v])
|
|
78
|
+
elif self._t > other._t:
|
|
79
|
+
return self.__class__([self._t, self._v])
|
|
80
|
+
return self.__class__([other._t, other._v])
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class CaptureMin(CaptureBase):
|
|
84
|
+
def compare(self, other):
|
|
85
|
+
if self._v is None:
|
|
86
|
+
return self.__class__([other._t, other._v ])
|
|
87
|
+
elif other._v is None:
|
|
88
|
+
return self.__class__([self._t, self._v])
|
|
89
|
+
return self.__class__([self._t, min(self._v, other._v)])
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class CaptureMax(CaptureBase):
|
|
93
|
+
def compare(self, other):
|
|
94
|
+
if self._v is None:
|
|
95
|
+
return self.__class__([other._t, other._v ])
|
|
96
|
+
elif other._v is None:
|
|
97
|
+
return self.__class__([self._t, self._v])
|
|
98
|
+
return self.__class__([self._t, max(self._v, other._v)])
|
|
99
|
+
|
|
100
|
+
if __name__ == '__main__':
|
|
101
|
+
# Test CaptureAnyMin
|
|
102
|
+
print('CaptureAnyMin')
|
|
103
|
+
a = CaptureAnyMin([5, 'd:2024-12-01T00:00:00'])
|
|
104
|
+
b = CaptureAnyMin([10, 'd:2024-12-02T00:00:00'])
|
|
105
|
+
|
|
106
|
+
print(a + b) # CaptureAnyMin(v=5, t=datetime.datetime(2024, 12, 1, 0, 0))
|
|
107
|
+
|
|
108
|
+
a += b
|
|
109
|
+
print(a) # CaptureAnyMin(v=5, t=datetime.datetime(2024, 12, 1, 0, 0))
|
|
110
|
+
print('CaptureAnyMax')
|
|
111
|
+
# Test CaptureAnyMax
|
|
112
|
+
c = CaptureAnyMax([5, 'd:2024-12-01T00:00:00'])
|
|
113
|
+
d = CaptureAnyMax([10, 'd:2024-12-02T00:00:00'])
|
|
114
|
+
|
|
115
|
+
print(c + d) # CaptureAnyMax(v=10, t=datetime.datetime(2024, 12, 2, 0, 0))
|
|
116
|
+
|
|
117
|
+
c += d
|
|
118
|
+
print(c) # CaptureAnyMax(v=10, t=datetime.datetime(2024, 12, 2, 0, 0))
|
|
119
|
+
print('CaptureMin')
|
|
120
|
+
# Test CaptureMin
|
|
121
|
+
e = CaptureMin(10)
|
|
122
|
+
f = CaptureMin(5)
|
|
123
|
+
print(e + f) # CaptureMin(v=5, t=None)
|
|
124
|
+
e += f
|
|
125
|
+
print(e) # CaptureMin(v=5, t=None)
|
|
126
|
+
|
|
127
|
+
e = CaptureMin('d:2024-12-01T00:00:00')
|
|
128
|
+
f = CaptureMin('d:2024-12-02T00:00:00')
|
|
129
|
+
print(e + f) # CaptureMin(v=5, t=None)
|
|
130
|
+
e += f
|
|
131
|
+
print(e) # CaptureMin(v=5, t=None)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
e = CaptureMax(5)
|
|
135
|
+
f = CaptureMax(10)
|
|
136
|
+
print(e + f) # CaptureMin(v=5, t=None)
|
|
137
|
+
e += f
|
|
138
|
+
print(e) # CaptureMin(v=5, t=None)
|
|
139
|
+
|
|
140
|
+
e = CaptureMax('d:2024-12-01T00:00:00')
|
|
141
|
+
f = CaptureMax('d:2024-12-02T00:00:00')
|
|
142
|
+
print(e + f) # CaptureMin(v=5, t=None)
|
|
143
|
+
e += f
|
|
144
|
+
print(e) # CaptureMin(v=5, t=None)
|
|
@@ -13,6 +13,12 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes # 3
|
|
|
13
13
|
import aiohttp # 3.8.1
|
|
14
14
|
import pandas as pd # 1.4.2
|
|
15
15
|
from .hyper_log_log import HyperLogLog
|
|
16
|
+
from .average import Average
|
|
17
|
+
from .tdigest import TDigest
|
|
18
|
+
from .rate import Rate
|
|
19
|
+
from .std import Std
|
|
20
|
+
from .capture import CaptureMin, CaptureMax, CaptureAnyMin, CaptureAnyMax
|
|
21
|
+
|
|
16
22
|
|
|
17
23
|
def _logtime(ts):
|
|
18
24
|
return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(ts))
|
|
@@ -532,9 +538,24 @@ class Tenant(Client):
|
|
|
532
538
|
logging.info(f'status={status} dsid={dsid} plid={plid} ts={_logtime(ts)}')
|
|
533
539
|
return rlt if isinstance(rlt, list) else []
|
|
534
540
|
|
|
541
|
+
MOPER_MAP = {
|
|
542
|
+
"$distinct": HyperLogLog,
|
|
543
|
+
"$count": float,
|
|
544
|
+
"$sum": float,
|
|
545
|
+
"$capMin": CaptureMin,
|
|
546
|
+
"$capMax": CaptureMax,
|
|
547
|
+
"$capAnyMin": CaptureAnyMin,
|
|
548
|
+
"$capAnyMax": CaptureAnyMax,
|
|
549
|
+
"$std": Std,
|
|
550
|
+
"$avg": Average,
|
|
551
|
+
"$tDigest": TDigest,
|
|
552
|
+
"$rate": Rate,
|
|
553
|
+
}
|
|
554
|
+
|
|
535
555
|
def _compute_kcol_mcol(pipeline):
|
|
536
556
|
kcol = 0 # key columns
|
|
537
557
|
mcol = 0 # mtr columns
|
|
558
|
+
moper = []
|
|
538
559
|
if 'bucket' in pipeline:
|
|
539
560
|
for bkt in pipeline['bucket']:
|
|
540
561
|
if bkt[0] in ['$distinctTuple','enumTuple']:
|
|
@@ -546,10 +567,25 @@ def _compute_kcol_mcol(pipeline):
|
|
|
546
567
|
if 'metric' in pipeline:
|
|
547
568
|
# each metric introduce two columns
|
|
548
569
|
mcol += len(pipeline['metric']) * 2
|
|
549
|
-
|
|
570
|
+
for m in pipeline['metric']:
|
|
571
|
+
if m[0] in MOPER_MAP:
|
|
572
|
+
moper.append(m[0])
|
|
573
|
+
elif m[0] in ['$capNum', '$capDate']:
|
|
574
|
+
if not 'method' in m[1] or m[1]['method'] == '%max':
|
|
575
|
+
moper.append('$capMax')
|
|
576
|
+
else:
|
|
577
|
+
moper.append('$capMin')
|
|
578
|
+
elif m[0] == '$capAny':
|
|
579
|
+
if not 'method' in m[1] or m[1]['method'] == '%max':
|
|
580
|
+
moper.append('$capAnyMax')
|
|
581
|
+
else:
|
|
582
|
+
moper.append('$capAnyMin')
|
|
583
|
+
else:
|
|
584
|
+
raise Exception(f"unknown metric operation {m[0]}")
|
|
585
|
+
return kcol, mcol, moper
|
|
550
586
|
|
|
551
587
|
class _translate_table:
|
|
552
|
-
def __init__(self, table, ts, kcol, mcol):
|
|
588
|
+
def __init__(self, table, ts, kcol, mcol, moper):
|
|
553
589
|
if isinstance(table, list) and len(table) > 0:
|
|
554
590
|
self.table = table
|
|
555
591
|
else:
|
|
@@ -557,29 +593,34 @@ class _translate_table:
|
|
|
557
593
|
mtr_cols = list(itertools.repeat(math.nan, mcol))
|
|
558
594
|
self.table = [key_cols+mtr_cols]
|
|
559
595
|
self.ts = ts
|
|
596
|
+
self.kcol = kcol
|
|
597
|
+
self.moper = moper
|
|
560
598
|
|
|
561
599
|
def __iter__(self):
|
|
562
600
|
for input_row in self.table:
|
|
563
601
|
output_row = [] if self.ts is None else [self.ts]
|
|
564
|
-
for elem in input_row:
|
|
565
|
-
if
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
602
|
+
for idx, elem in enumerate(input_row):
|
|
603
|
+
if idx < self.kcol:
|
|
604
|
+
if type(elem) == list:
|
|
605
|
+
output_row.append(tuple(elem))
|
|
606
|
+
else:
|
|
607
|
+
output_row.append(elem)
|
|
569
608
|
else:
|
|
570
|
-
|
|
609
|
+
moper_idx = self.moper[int((idx-self.kcol)/2)]
|
|
610
|
+
output_row.append(MOPER_MAP[moper_idx](elem))
|
|
571
611
|
yield output_row
|
|
572
612
|
|
|
573
613
|
class _translate_rlts:
|
|
574
|
-
def __init__(self, tsidx, rlts, kcol, mcol):
|
|
614
|
+
def __init__(self, tsidx, rlts, kcol, mcol, moper):
|
|
575
615
|
self.tsidx = tsidx
|
|
576
616
|
self.rlts = rlts
|
|
577
617
|
self.kcol = kcol
|
|
578
618
|
self.mcol = mcol
|
|
619
|
+
self.moper = moper
|
|
579
620
|
|
|
580
621
|
def __iter__(self):
|
|
581
622
|
for ts, rlt in zip(self.tsidx, self.rlts):
|
|
582
|
-
yield _translate_table(rlt, ts, self.kcol, self.mcol)
|
|
623
|
+
yield _translate_table(rlt, ts, self.kcol, self.mcol, self.moper)
|
|
583
624
|
|
|
584
625
|
TZINFO = datetime.datetime.now().astimezone().tzinfo
|
|
585
626
|
def _get_utc_timestamp(ts):
|
|
@@ -610,20 +651,20 @@ class Adhoc:
|
|
|
610
651
|
async def _read_point(self, ts, series, columns=None, refresh=True):
|
|
611
652
|
if refresh:
|
|
612
653
|
await self._tenant.create_adhoc(self.pipeline)
|
|
613
|
-
freq_str = f"{self.freq}
|
|
654
|
+
freq_str = f"{self.freq}s"
|
|
614
655
|
delta = pd.Timedelta(freq_str)
|
|
615
656
|
start = pd.Timestamp(ts).floor(freq_str)
|
|
616
657
|
end = start + delta
|
|
617
658
|
table = await self._tenant.execute_adhoc(self.pid, _get_utc_timestamp(start), _get_utc_timestamp(end), series)
|
|
618
|
-
kcol, mcol = _compute_kcol_mcol(self.pipeline)
|
|
619
|
-
return pd.DataFrame(_translate_table(table, None, kcol, mcol), columns=columns)
|
|
659
|
+
kcol, mcol, moper = _compute_kcol_mcol(self.pipeline)
|
|
660
|
+
return pd.DataFrame(_translate_table(table, None, kcol, mcol, moper), columns=columns)
|
|
620
661
|
|
|
621
662
|
async def _read_range(self, dts, series, columns=None, refresh=True):
|
|
622
663
|
if refresh:
|
|
623
664
|
await self._tenant.create_adhoc(self.pipeline)
|
|
624
665
|
tasks = []
|
|
625
666
|
tsidx = []
|
|
626
|
-
freq_str = f"{self.freq}
|
|
667
|
+
freq_str = f"{self.freq}s"
|
|
627
668
|
rlts = None
|
|
628
669
|
delta = pd.Timedelta(freq_str)
|
|
629
670
|
for ts in dts:
|
|
@@ -639,8 +680,8 @@ class Adhoc:
|
|
|
639
680
|
for task in tasks:
|
|
640
681
|
task.cancel()
|
|
641
682
|
raise e
|
|
642
|
-
kcol, mcol = _compute_kcol_mcol(self.pipeline)
|
|
643
|
-
iter = itertools.chain.from_iterable(_translate_rlts(tsidx, rlts, kcol, mcol))
|
|
683
|
+
kcol, mcol, moper = _compute_kcol_mcol(self.pipeline)
|
|
684
|
+
iter = itertools.chain.from_iterable(_translate_rlts(tsidx, rlts, kcol, mcol, moper))
|
|
644
685
|
return pd.DataFrame(iter, columns=columns)
|
|
645
686
|
|
|
646
687
|
async def read_data(self, dts, *args, **kwargs):
|
|
@@ -680,16 +721,16 @@ class Pipeline:
|
|
|
680
721
|
self.conf = conf
|
|
681
722
|
|
|
682
723
|
async def _read_point(self, ts, columns=None):
|
|
683
|
-
freq_str = f"{self._dset.conf['freq']}
|
|
724
|
+
freq_str = f"{self._dset.conf['freq']}s"
|
|
684
725
|
start = ts.floor(freq_str)
|
|
685
726
|
table = await self._tenant.query_pipeline_data(self._dset.dsid, self.plid, _get_utc_timestamp(start))
|
|
686
|
-
kcol, mcol = _compute_kcol_mcol(self.conf['pipeline'])
|
|
687
|
-
return pd.DataFrame(_translate_table(table, None, kcol, mcol), columns=columns)
|
|
727
|
+
kcol, mcol, moper = _compute_kcol_mcol(self.conf['pipeline'])
|
|
728
|
+
return pd.DataFrame(_translate_table(table, None, kcol, mcol, moper), columns=columns)
|
|
688
729
|
|
|
689
730
|
async def _read_range(self, dts, columns=None):
|
|
690
731
|
tasks = []
|
|
691
732
|
tsidx = []
|
|
692
|
-
freq_str = f"{self._dset.conf['freq']}
|
|
733
|
+
freq_str = f"{self._dset.conf['freq']}s"
|
|
693
734
|
rlts = None
|
|
694
735
|
for ts in dts:
|
|
695
736
|
start = ts.floor(freq_str)
|
|
@@ -703,8 +744,8 @@ class Pipeline:
|
|
|
703
744
|
for task in tasks:
|
|
704
745
|
task.cancel()
|
|
705
746
|
raise e
|
|
706
|
-
kcol, mcol = _compute_kcol_mcol(self.conf['pipeline'])
|
|
707
|
-
iter = itertools.chain.from_iterable(_translate_rlts(tsidx, rlts, kcol, mcol))
|
|
747
|
+
kcol, mcol, moper = _compute_kcol_mcol(self.conf['pipeline'])
|
|
748
|
+
iter = itertools.chain.from_iterable(_translate_rlts(tsidx, rlts, kcol, mcol, moper))
|
|
708
749
|
return pd.DataFrame(iter, columns=columns)
|
|
709
750
|
|
|
710
751
|
async def read_data(self, dts, *args, **kwargs):
|
|
@@ -767,7 +808,7 @@ class Dataset:
|
|
|
767
808
|
Returns:
|
|
768
809
|
Patch result in frame.
|
|
769
810
|
"""
|
|
770
|
-
freq_str = f"{self.conf['freq']}
|
|
811
|
+
freq_str = f"{self.conf['freq']}s"
|
|
771
812
|
tasks = []
|
|
772
813
|
tsidx = []
|
|
773
814
|
rlts = None
|
|
@@ -795,7 +836,7 @@ class Dataset:
|
|
|
795
836
|
Returns:
|
|
796
837
|
Patch result in frame.
|
|
797
838
|
"""
|
|
798
|
-
freq_str = f"{self.conf['freq']}
|
|
839
|
+
freq_str = f"{self.conf['freq']}s"
|
|
799
840
|
tasks = []
|
|
800
841
|
tsidx = []
|
|
801
842
|
rlts = None
|
|
@@ -824,7 +865,7 @@ class Dataset:
|
|
|
824
865
|
Returns:
|
|
825
866
|
The created task
|
|
826
867
|
"""
|
|
827
|
-
freq_str = f"{self.conf['freq']}
|
|
868
|
+
freq_str = f"{self.conf['freq']}s"
|
|
828
869
|
next_ts = pd.Timestamp(ts).floor(freq_str)
|
|
829
870
|
delta = pd.Timedelta(freq_str)
|
|
830
871
|
return asyncio.create_task(self._monitor_loop(next_ts, delta, coro, *args))
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import math
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
|
+
class Rate:
|
|
4
|
+
def __init__(self, arg=[0, math.inf, 0]):
|
|
5
|
+
self._sum = arg[0]
|
|
6
|
+
self._start_ms = arg[1]
|
|
7
|
+
self._end_ms = arg[2]
|
|
8
|
+
if (isinstance(self._start_ms, str) and self._start_ms.startswith('d:')):
|
|
9
|
+
self._start_ms = datetime.fromisoformat(self._start_ms[2:]).timestamp() * 1000
|
|
10
|
+
if (isinstance(self._end_ms, str) and self._end_ms.startswith('d:')):
|
|
11
|
+
self._end_ms = datetime.fromisoformat(self._end_ms[2:]).timestamp() * 1000
|
|
12
|
+
|
|
13
|
+
def __add__(self, other):
|
|
14
|
+
first = self._start_ms if self._start_ms < other._start_ms else other._start_ms
|
|
15
|
+
last = self._end_ms if self._end_ms > other._end_ms else other._end_ms
|
|
16
|
+
return Rate([self._sum + other._sum, first, last])
|
|
17
|
+
|
|
18
|
+
def __radd__(self, other):
|
|
19
|
+
first = self._start_ms if self._start_ms < other._start_ms else other._start_ms
|
|
20
|
+
last = self._end_ms if self._end_ms > other._end_ms else other._end_ms
|
|
21
|
+
return Rate([self._sum + other._sum, first, last])
|
|
22
|
+
|
|
23
|
+
def __iadd__(self, other):
|
|
24
|
+
self._sum += other._sum
|
|
25
|
+
self._start_ms = self._start_ms if self._start_ms < other._start_ms else other._start_ms
|
|
26
|
+
self._end_ms = self._end_ms if self._end_ms > other._end_ms else other._end_ms
|
|
27
|
+
return self
|
|
28
|
+
|
|
29
|
+
def __str__(self):
|
|
30
|
+
return f'{self.value()}'
|
|
31
|
+
|
|
32
|
+
def __repr__(self):
|
|
33
|
+
if self._end_ms <= self._start_ms:
|
|
34
|
+
return f'$rate (empty)'
|
|
35
|
+
return f'$rate(sum={self._sum}, start={datetime.fromtimestamp(self._start_ms/1000, tz=timezone.utc)}, end={datetime.fromtimestamp(self._end_ms/1000, tz=timezone.utc)}'
|
|
36
|
+
def value(self):
|
|
37
|
+
if self._end_ms <= self._start_ms:
|
|
38
|
+
return math.nan
|
|
39
|
+
return self._sum / ((self._end_ms - self._start_ms)/1000)
|
|
40
|
+
|
|
41
|
+
if __name__ == '__main__':
|
|
42
|
+
x = Rate([10, 1734592909000, 1734592910000])
|
|
43
|
+
y = Rate([20, 1734592909000, 1734592910000])
|
|
44
|
+
z = Rate([30, "d:2020-12-20T10:00:00Z", "d:2020-12-20T10:00:05Z"])
|
|
45
|
+
print('x = ', x)
|
|
46
|
+
print('y = ', y)
|
|
47
|
+
print('x + y = ', x + y)
|
|
48
|
+
print('repr(x) = ', repr(x))
|
|
49
|
+
print('repr(y) = ', repr(y))
|
|
50
|
+
print('z = ', z)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import math
|
|
2
|
+
class Std:
|
|
3
|
+
def __init__(self, arg=[0, 0, 0]):
|
|
4
|
+
self._mean = arg[0]
|
|
5
|
+
self._count = arg[1]
|
|
6
|
+
self._variance = arg[2]
|
|
7
|
+
def __add__(self, other):
|
|
8
|
+
new_count = self._count + other._count
|
|
9
|
+
new_mean = (self._mean * self._count + other._mean * other._count) / new_count
|
|
10
|
+
new_variance = self._variance + other._variance + (self._mean - other._mean)**2 * self._count * other._count / new_count
|
|
11
|
+
return Std([new_mean, new_count, new_variance])
|
|
12
|
+
|
|
13
|
+
def __radd__(self, other):
|
|
14
|
+
new_count = self._count + other._count
|
|
15
|
+
new_mean = (self._mean * self._count + other._mean * other._count) / new_count
|
|
16
|
+
new_variance = self._variance + other._variance + (self._mean - other._mean)**2 * self._count * other._count / new_count
|
|
17
|
+
return Std([new_mean, new_count, new_variance])
|
|
18
|
+
|
|
19
|
+
def __iadd__(self, other):
|
|
20
|
+
new_count = self._count + other._count
|
|
21
|
+
new_mean = (self._mean * self._count + other._mean * other._count) / new_count
|
|
22
|
+
new_variance = self._variance + other._variance + (self._mean - other._mean)**2 * self._count * other._count / new_count
|
|
23
|
+
self._mean = new_mean
|
|
24
|
+
self._count = new_count
|
|
25
|
+
self._variance = new_variance
|
|
26
|
+
return self
|
|
27
|
+
|
|
28
|
+
def __str__(self):
|
|
29
|
+
return f'{self.value()}'
|
|
30
|
+
|
|
31
|
+
def __repr__(self):
|
|
32
|
+
return f'$std(mean={self._mean}, count={self._count}, variance={self._variance})'
|
|
33
|
+
|
|
34
|
+
def value(self):
|
|
35
|
+
if self._count == 0:
|
|
36
|
+
return math.nan
|
|
37
|
+
return math.sqrt(self._variance / self._count)
|
|
38
|
+
|
|
39
|
+
if __name__ == '__main__':
|
|
40
|
+
pass
|
|
41
|
+
# x = Std([10, 1734592909000, 1734592910000])
|
|
42
|
+
# y = Std([20, 1734592909000, 1734592910000])
|
|
43
|
+
# print('x = ', x)
|
|
44
|
+
# print('y = ', y)
|
|
45
|
+
# print('x + y = ', x + y)
|
|
46
|
+
# print('repr(x) = ', repr(x))
|
|
47
|
+
# print('repr(y) = ', repr(y))
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import math
|
|
2
|
+
# To use in pandas df['PR90'] = df['tDigest'].apply(lambda x: x.quantile(0.9))
|
|
3
|
+
# print tDigest would output the median
|
|
4
|
+
# Supported Method: min, max, mean, sum, empty, quantile
|
|
5
|
+
|
|
6
|
+
class Centroid:
|
|
7
|
+
def __init__(self, mean, weight):
|
|
8
|
+
self.mean = mean
|
|
9
|
+
self.weight = weight
|
|
10
|
+
|
|
11
|
+
def update(self, sum_, weight):
|
|
12
|
+
sum_ += self.mean * self.weight
|
|
13
|
+
self.weight += weight
|
|
14
|
+
self.mean = sum_ / self.weight
|
|
15
|
+
return sum_
|
|
16
|
+
|
|
17
|
+
class TDigest:
|
|
18
|
+
def __init__(self, arg=[[math.inf, 0, 64]]):
|
|
19
|
+
centroids = arg[1:]
|
|
20
|
+
self.centroids = []
|
|
21
|
+
self.min = arg[0][0]
|
|
22
|
+
self.max = arg[0][1]
|
|
23
|
+
self.count = 0
|
|
24
|
+
self.sum = 0
|
|
25
|
+
self.max_size = arg[0][2]
|
|
26
|
+
|
|
27
|
+
for c in centroids:
|
|
28
|
+
# Assume each centroid is a tuple (mean, weight)
|
|
29
|
+
self.sum += c[0] * c[1]
|
|
30
|
+
self.count += c[1]
|
|
31
|
+
self.centroids.append(Centroid(c[0], c[1]))
|
|
32
|
+
|
|
33
|
+
def __add__(self, other):
|
|
34
|
+
new_one = TDigest([[math.inf, 0, max(self.max_size, other.max_size)]])
|
|
35
|
+
new_one.update(self)
|
|
36
|
+
new_one.update(other)
|
|
37
|
+
return new_one
|
|
38
|
+
|
|
39
|
+
def __radd__(self, other):
|
|
40
|
+
new_one = TDigest([[math.inf, 0, max(self.max_size, other.max_size)]])
|
|
41
|
+
new_one.update(self)
|
|
42
|
+
new_one.update(other)
|
|
43
|
+
return new_one
|
|
44
|
+
|
|
45
|
+
def __iadd__(self, other):
|
|
46
|
+
self.update(other)
|
|
47
|
+
return self
|
|
48
|
+
|
|
49
|
+
def __str__(self):
|
|
50
|
+
return f'{self.quantile(0.5)}'
|
|
51
|
+
|
|
52
|
+
def __repr__(self):
|
|
53
|
+
return f'$tDigest({self.output()})'
|
|
54
|
+
|
|
55
|
+
def output(self):
|
|
56
|
+
centroids = [[c.mean, c.weight] for c in self.centroids]
|
|
57
|
+
centroids.insert(0, [self.min, self.max, self.max_size])
|
|
58
|
+
return centroids
|
|
59
|
+
|
|
60
|
+
def __k_to_q(self, k, d):
|
|
61
|
+
k_div_d = k / d
|
|
62
|
+
if k_div_d >= 0.5:
|
|
63
|
+
base = 1 - k_div_d
|
|
64
|
+
return 1 - 2 * base * base
|
|
65
|
+
else:
|
|
66
|
+
return 2 * k_div_d * k_div_d
|
|
67
|
+
|
|
68
|
+
def update(self, another):
|
|
69
|
+
if not isinstance(another, TDigest):
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
new_centroids = []
|
|
73
|
+
self.centroids.extend(another.centroids)
|
|
74
|
+
self.centroids.sort(key=lambda c: c.mean)
|
|
75
|
+
self.min = min(self.min, another.min)
|
|
76
|
+
self.max = max(self.max, another.max)
|
|
77
|
+
self.count += another.count
|
|
78
|
+
|
|
79
|
+
new_sum = 0
|
|
80
|
+
k_limit = 1
|
|
81
|
+
q_limit_times_count = self.__k_to_q(k_limit, self.max_size) * self.count
|
|
82
|
+
k_limit += 1
|
|
83
|
+
|
|
84
|
+
centroid_idx = 0
|
|
85
|
+
cur_centroid = self.centroids[centroid_idx]
|
|
86
|
+
centroid_idx += 1
|
|
87
|
+
|
|
88
|
+
weight_so_far = cur_centroid.weight
|
|
89
|
+
sums_to_merge = 0
|
|
90
|
+
weight_to_merge = 0
|
|
91
|
+
|
|
92
|
+
for _ in range(1, len(self.centroids)):
|
|
93
|
+
next_centroid = self.centroids[centroid_idx]
|
|
94
|
+
centroid_idx += 1
|
|
95
|
+
|
|
96
|
+
next_sum = next_centroid.mean * next_centroid.weight
|
|
97
|
+
weight_so_far += next_centroid.weight
|
|
98
|
+
|
|
99
|
+
if weight_so_far <= q_limit_times_count:
|
|
100
|
+
sums_to_merge += next_sum
|
|
101
|
+
weight_to_merge += next_centroid.weight
|
|
102
|
+
else:
|
|
103
|
+
new_sum += cur_centroid.update(sums_to_merge, weight_to_merge)
|
|
104
|
+
sums_to_merge = 0
|
|
105
|
+
weight_to_merge = 0
|
|
106
|
+
new_centroids.append(cur_centroid)
|
|
107
|
+
|
|
108
|
+
q_limit_times_count = self.__k_to_q(k_limit, self.max_size) * self.count
|
|
109
|
+
k_limit += 1
|
|
110
|
+
cur_centroid = next_centroid
|
|
111
|
+
|
|
112
|
+
new_sum += cur_centroid.update(sums_to_merge, weight_to_merge)
|
|
113
|
+
new_centroids.append(cur_centroid)
|
|
114
|
+
|
|
115
|
+
self.centroids = new_centroids
|
|
116
|
+
self.sum = new_sum
|
|
117
|
+
|
|
118
|
+
def quantile(self, q):
|
|
119
|
+
if q <= 0:
|
|
120
|
+
return self.min
|
|
121
|
+
elif q >= 1:
|
|
122
|
+
return self.max
|
|
123
|
+
elif self.count == 0:
|
|
124
|
+
return math.nan
|
|
125
|
+
|
|
126
|
+
rank = q * self.count
|
|
127
|
+
pos = None
|
|
128
|
+
weight_so_far = None
|
|
129
|
+
|
|
130
|
+
if q > 0.5:
|
|
131
|
+
pos = 0
|
|
132
|
+
weight_so_far = self.count
|
|
133
|
+
for i in range(len(self.centroids) - 1, -1, -1):
|
|
134
|
+
weight_so_far -= self.centroids[i].weight
|
|
135
|
+
if rank >= weight_so_far:
|
|
136
|
+
pos = i
|
|
137
|
+
break
|
|
138
|
+
else:
|
|
139
|
+
pos = len(self.centroids) - 1
|
|
140
|
+
weight_so_far = 0
|
|
141
|
+
for i, centroid in enumerate(self.centroids):
|
|
142
|
+
if rank < weight_so_far + centroid.weight:
|
|
143
|
+
pos = i
|
|
144
|
+
break
|
|
145
|
+
weight_so_far += centroid.weight
|
|
146
|
+
|
|
147
|
+
delta = 0
|
|
148
|
+
min_ = self.min
|
|
149
|
+
max_ = self.max
|
|
150
|
+
|
|
151
|
+
if len(self.centroids) > 1:
|
|
152
|
+
if pos == 0:
|
|
153
|
+
delta = self.centroids[pos + 1].mean - self.centroids[pos].mean
|
|
154
|
+
max_ = self.centroids[pos + 1].mean
|
|
155
|
+
elif pos == len(self.centroids) - 1:
|
|
156
|
+
delta = self.centroids[pos].mean - self.centroids[pos - 1].mean
|
|
157
|
+
min_ = self.centroids[pos - 1].mean
|
|
158
|
+
else:
|
|
159
|
+
delta = (self.centroids[pos + 1].mean - self.centroids[pos - 1].mean) / 2
|
|
160
|
+
min_ = self.centroids[pos - 1].mean
|
|
161
|
+
max_ = self.centroids[pos + 1].mean
|
|
162
|
+
|
|
163
|
+
value = (self.centroids[pos].mean +
|
|
164
|
+
((rank - weight_so_far) / self.centroids[pos].weight - 0.5) * delta)
|
|
165
|
+
|
|
166
|
+
if value < min_:
|
|
167
|
+
value = min_
|
|
168
|
+
elif value > max_:
|
|
169
|
+
value = max_
|
|
170
|
+
|
|
171
|
+
return value
|
|
172
|
+
|
|
173
|
+
def mean(self):
|
|
174
|
+
return self.sum / self.count if self.count > 0 else math.nan
|
|
175
|
+
|
|
176
|
+
def sum(self):
|
|
177
|
+
return self.sum
|
|
178
|
+
|
|
179
|
+
def min(self):
|
|
180
|
+
return self.min
|
|
181
|
+
|
|
182
|
+
def max(self):
|
|
183
|
+
return self.max
|
|
184
|
+
|
|
185
|
+
def empty(self):
|
|
186
|
+
return self.count == 0
|
|
187
|
+
|
|
188
|
+
def max_size(self):
|
|
189
|
+
return self.max_size
|
|
@@ -4,9 +4,14 @@ pyproject.toml
|
|
|
4
4
|
setup.cfg
|
|
5
5
|
setup.py
|
|
6
6
|
ga_api/__init__.py
|
|
7
|
+
ga_api/average.py
|
|
8
|
+
ga_api/capture.py
|
|
7
9
|
ga_api/field_tree.py
|
|
8
10
|
ga_api/ga_api_client.py
|
|
9
11
|
ga_api/hyper_log_log.py
|
|
12
|
+
ga_api/rate.py
|
|
13
|
+
ga_api/std.py
|
|
14
|
+
ga_api/tdigest.py
|
|
10
15
|
ga_api_client.egg-info/PKG-INFO
|
|
11
16
|
ga_api_client.egg-info/SOURCES.txt
|
|
12
17
|
ga_api_client.egg-info/dependency_links.txt
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|