ga-api-client 0.4.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. {ga_api_client-0.4.0/ga_api_client.egg-info → ga_api_client-0.5.0}/PKG-INFO +3 -2
  2. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api/ga_api_client.py +91 -9
  3. {ga_api_client-0.4.0 → ga_api_client-0.5.0/ga_api_client.egg-info}/PKG-INFO +3 -2
  4. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api_client.egg-info/requires.txt +1 -0
  5. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/setup.cfg +2 -1
  6. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/LICENSE +0 -0
  7. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/README.md +0 -0
  8. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api/__init__.py +0 -0
  9. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api/average.py +0 -0
  10. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api/capture.py +0 -0
  11. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api/field_tree.py +0 -0
  12. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api/hyper_log_log.py +0 -0
  13. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api/rate.py +0 -0
  14. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api/std.py +0 -0
  15. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api/tdigest.py +0 -0
  16. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api_client.egg-info/SOURCES.txt +0 -0
  17. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api_client.egg-info/dependency_links.txt +0 -0
  18. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/ga_api_client.egg-info/top_level.txt +0 -0
  19. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/pyproject.toml +0 -0
  20. {ga_api_client-0.4.0 → ga_api_client-0.5.0}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: ga-api-client
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Classifier: Programming Language :: Python :: 3
5
5
  Classifier: License :: OSI Approved :: MIT License
6
6
  Classifier: Operating System :: OS Independent
@@ -10,6 +10,7 @@ License-File: LICENSE
10
10
  Requires-Dist: pandas>=1.4
11
11
  Requires-Dist: aiohttp>=3.8
12
12
  Requires-Dist: cryptography>=36.0
13
+ Requires-Dist: async-timeout>=5.0
13
14
 
14
15
  # GenieAnalytics Python Client
15
16
 
@@ -23,6 +23,13 @@ from .capture import CaptureMin, CaptureMax, CaptureAnyMin, CaptureAnyMax
23
23
  def _logtime(ts):
24
24
  return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(ts))
25
25
 
26
+ TZINFO = datetime.datetime.now().astimezone().tzinfo
27
+ def _get_utc_timestamp(ts):
28
+ if ts.tz is None:
29
+ ts = ts.tz_localize(tz=TZINFO)
30
+ return ts.timestamp()
31
+
32
+
26
33
  def pprint(value, indent=4, depth=0):
27
34
  if isinstance(value, dict):
28
35
  # empty dict, printed as simple
@@ -95,6 +102,7 @@ class Client:
95
102
  self.password = base64.b64encode(password.encode('utf-8')).decode() # base64 encoded password
96
103
  self.ssl = ssl # False to disable certificate verification
97
104
  self.retry = retry
105
+ self.burst = burst
98
106
  self._sem = asyncio.Semaphore(burst) # semaphone to throttle concurrent burst
99
107
  self._headers = {
100
108
  'Content-Type': 'application/json'
@@ -177,7 +185,7 @@ class Client:
177
185
  self._last_auth = time.time()
178
186
  logging.info(f'authenticate done')
179
187
 
180
- async def _request(self, method, path, code, data=None):
188
+ async def _request(self, method, path, code, data=None, test_fn = None):
181
189
  # authenticate if required
182
190
  if time.time() - self._last_auth >= ACCESS_TOKEN_TIMEOUT:
183
191
  retry_count = 0
@@ -195,6 +203,9 @@ class Client:
195
203
  # send the request
196
204
  retry_count = 0
197
205
  async with self._sem:
206
+ # test_fn is a function to check if the request should be sent
207
+ if test_fn is not None and test_fn() is False:
208
+ return None
198
209
  ts_req = datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S')
199
210
  while True:
200
211
  try:
@@ -386,6 +397,68 @@ def desc_dataframe(desc_list):
386
397
  desc_dict[dset['_id']] = item
387
398
  return pd.DataFrame.from_dict(desc_dict, orient='index')
388
399
 
400
+ class Reader:
401
+ def __init__(self, client, pipeline):
402
+ self._client = client
403
+ self.pipeline = pipeline
404
+ self.count = 0
405
+ self.size = 0
406
+ def test_fn(self):
407
+ return self.count < self.size
408
+
409
+ async def _exec(self, start, end):
410
+ start = _get_utc_timestamp(start)
411
+ end = _get_utc_timestamp(end)
412
+ if 'config' not in self.pipeline:
413
+ self.pipeline['config'] = {}
414
+ self.pipeline['config']['start'] = start
415
+ self.pipeline['config']['end'] = end
416
+ self.pipeline['config']['size'] = self.size
417
+ self.pipeline['config']['series'] = self.series
418
+ if self.fields is not None:
419
+ self.pipeline['config']['fields'] = self.fields
420
+ status, rlt = await self._client._request('POST', f'/record', [200], self.pipeline, self.test_fn)
421
+ logging.info(f'/record status={status} start={_logtime(start)} end={_logtime(end)} series={self.series}')
422
+ if rlt is not None and len(rlt) > 0:
423
+ self.count += len(rlt) - 1 # skip first row, which is header
424
+ # print(rlt)
425
+ return pd.DataFrame(rlt[1:], columns=rlt[0])
426
+ return pd.DataFrame() # empty dataframe is returned if no data
427
+
428
+
429
+ async def read_data(self, dts, series='full', size=100, fields=None):
430
+ """Execute adhoc request and read its data
431
+ Args:
432
+ dts: pd.DatetimeIndex, using pd.date_range to generate.
433
+ series: 'full', 'hour', 'day' or 'log
434
+ size: number of rows to read
435
+ fields: list of field names
436
+ Returns:
437
+ The record of dataframe.
438
+ """
439
+ self.size = size
440
+ self.count = 0
441
+ self.series = f"%{series}" # present in literal
442
+ self.fields = fields
443
+ tasks = []
444
+ for i in range(0, len(dts)):
445
+ start_time = dts[i]
446
+ end_time = dts[i] + pd.Timedelta(dts.freq)
447
+ tasks.append(asyncio.create_task(self._exec(start_time, end_time)))
448
+ try:
449
+ rlts = await asyncio.gather(*tasks)
450
+ df = pd.DataFrame()
451
+ for rlt in rlts:
452
+ df = pd.concat([df, rlt], ignore_index=True, copy=False)
453
+ if len(df) >= self.size:
454
+ break
455
+ return df.head(self.size)[fields]
456
+ except Exception as e:
457
+ logging.error(f'execute record fail {e}')
458
+ for task in tasks:
459
+ task.cancel()
460
+ raise e
461
+
389
462
  class Tenant(Client):
390
463
  def __init__(self, url, user=None, tenant=None, password=None, ssl=True, burst=1, retry=0, conn_timeout=30, read_timeout=300, strategy="custom", controller=None):
391
464
  if user is None:
@@ -415,7 +488,10 @@ class Tenant(Client):
415
488
  status, rlt = await self._request('GET', f'/pipeline/{pid}?start={start}&end={end}&series={series}', [200])
416
489
  logging.info(f'status={status} pid={pid} start={_logtime(start)} end={_logtime(end)} series={series}')
417
490
  return rlt if isinstance(rlt, list) else []
418
-
491
+
492
+ async def create_reader(self, pipeline):
493
+ return Reader(self, pipeline)
494
+
419
495
  async def get_all_datasets(self):
420
496
  # 200 - ok, return array of dataset descriptor
421
497
  status, rlt = await self._request('GET', '/cq/dataset', [200])
@@ -643,12 +719,6 @@ class _translate_rlts:
643
719
  for ts, rlt in zip(self.tsidx, self.rlts):
644
720
  yield _translate_table(rlt, ts, self.kcol, self.mcol, self.moper)
645
721
 
646
- TZINFO = datetime.datetime.now().astimezone().tzinfo
647
- def _get_utc_timestamp(ts):
648
- if ts.tz is None:
649
- ts = ts.tz_localize(tz=TZINFO)
650
- return ts.timestamp()
651
-
652
722
  class Adhoc:
653
723
  @staticmethod
654
724
  async def _create(tenant, freq, pipeline):
@@ -784,8 +854,10 @@ class Pipeline:
784
854
  except:
785
855
  ts = None
786
856
  if ts is None:
857
+ print('range')
787
858
  return await self._read_range(dts, *args, **kwargs)
788
859
  else:
860
+ print('point')
789
861
  return await self._read_point(ts, *args, **kwargs)
790
862
 
791
863
  class Dataset:
@@ -1102,7 +1174,7 @@ class Controller:
1102
1174
  rlt = await self._tenant.get_mgr_config(req)
1103
1175
  return self.desc_resource_group(rlt)
1104
1176
  class Repository:
1105
- def __init__(self, url, user=None, tenant=None, password=None, ssl=True, burst=1, retry=0, conn_timeout=30, read_timeout=300, strategy=None, controller=None):
1177
+ def __init__(self, url, user=None, tenant=None, password=None, ssl=True, burst=1, retry=0, conn_timeout=30, read_timeout=300, strategy='custom', controller=None):
1106
1178
  """Construct repository object to send tenant requests to API server
1107
1179
 
1108
1180
  Args:
@@ -1181,6 +1253,16 @@ class Repository:
1181
1253
  """
1182
1254
  return await Adhoc._create(self._tenant, freq, pipeline)
1183
1255
 
1256
+ async def reader(self, pipeline):
1257
+ """Execute an Record operation.
1258
+
1259
+ Args:
1260
+ pipeline: the pipeline descriptor
1261
+ Returns:
1262
+ The record data frame.
1263
+ """
1264
+ return Reader(self._tenant, pipeline)
1265
+
1184
1266
  async def authenticate(self):
1185
1267
  """Execute authentication.
1186
1268
 
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: ga-api-client
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Classifier: Programming Language :: Python :: 3
5
5
  Classifier: License :: OSI Approved :: MIT License
6
6
  Classifier: Operating System :: OS Independent
@@ -10,6 +10,7 @@ License-File: LICENSE
10
10
  Requires-Dist: pandas>=1.4
11
11
  Requires-Dist: aiohttp>=3.8
12
12
  Requires-Dist: cryptography>=36.0
13
+ Requires-Dist: async-timeout>=5.0
13
14
 
14
15
  # GenieAnalytics Python Client
15
16
 
@@ -1,3 +1,4 @@
1
1
  pandas>=1.4
2
2
  aiohttp>=3.8
3
3
  cryptography>=36.0
4
+ async-timeout>=5.0
@@ -1,6 +1,6 @@
1
1
  [metadata]
2
2
  name = ga-api-client
3
- version = 0.4.0
3
+ version = 0.5.0
4
4
  long_description = file: README.md
5
5
  long_description_content_type = text/markdown
6
6
  classifiers =
@@ -14,6 +14,7 @@ install_requires =
14
14
  pandas >= 1.4
15
15
  aiohttp >= 3.8
16
16
  cryptography >= 36.0
17
+ async-timeout >= 5.0
17
18
  packages = ga_api
18
19
 
19
20
  [options.packages.find]
File without changes
File without changes
File without changes