meerschaum 2.8.3__py3-none-any.whl → 2.9.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. meerschaum/_internal/arguments/_parser.py +5 -0
  2. meerschaum/actions/drop.py +1 -1
  3. meerschaum/actions/start.py +14 -6
  4. meerschaum/actions/sync.py +9 -0
  5. meerschaum/api/__init__.py +9 -3
  6. meerschaum/api/_chunks.py +67 -0
  7. meerschaum/api/dash/callbacks/custom.py +23 -2
  8. meerschaum/api/dash/callbacks/dashboard.py +41 -3
  9. meerschaum/api/dash/components.py +27 -19
  10. meerschaum/api/dash/pages/dashboard.py +11 -9
  11. meerschaum/api/dash/pages/plugins.py +31 -27
  12. meerschaum/api/dash/webterm.py +6 -3
  13. meerschaum/api/resources/static/css/dash.css +1 -1
  14. meerschaum/api/resources/templates/termpage.html +4 -0
  15. meerschaum/api/routes/_pipes.py +193 -81
  16. meerschaum/config/_default.py +3 -0
  17. meerschaum/config/_version.py +1 -1
  18. meerschaum/connectors/api/_APIConnector.py +12 -1
  19. meerschaum/connectors/api/_pipes.py +27 -15
  20. meerschaum/connectors/api/_plugins.py +51 -45
  21. meerschaum/connectors/api/_request.py +1 -1
  22. meerschaum/connectors/parse.py +1 -2
  23. meerschaum/connectors/sql/_SQLConnector.py +1 -1
  24. meerschaum/core/Pipe/_data.py +1 -2
  25. meerschaum/core/Pipe/_verify.py +23 -8
  26. meerschaum/jobs/systemd.py +1 -1
  27. meerschaum/plugins/_Plugin.py +21 -5
  28. meerschaum/plugins/__init__.py +6 -4
  29. meerschaum/utils/formatting/_shell.py +1 -4
  30. meerschaum/utils/packages/_packages.py +2 -1
  31. meerschaum/utils/process.py +27 -3
  32. meerschaum/utils/schedule.py +3 -3
  33. meerschaum/utils/sql.py +1 -1
  34. meerschaum/utils/venv/__init__.py +6 -1
  35. {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/METADATA +4 -1
  36. {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/RECORD +42 -41
  37. {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/WHEEL +1 -1
  38. {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/LICENSE +0 -0
  39. {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/NOTICE +0 -0
  40. {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/entry_points.txt +0 -0
  41. {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/top_level.txt +0 -0
  42. {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/zip-safe +0 -0
@@ -27,8 +27,10 @@ from meerschaum.api import (
27
27
  debug,
28
28
  no_auth,
29
29
  )
30
+ from meerschaum.api._chunks import generate_chunks_cursor_token
30
31
  from meerschaum.utils.packages import attempt_import
31
32
  from meerschaum.utils.dataframe import to_json
33
+ from meerschaum.utils.dtypes import are_dtypes_equal, json_serialize_value
32
34
  from meerschaum.utils.misc import (
33
35
  is_pipe_registered,
34
36
  is_int,
@@ -38,14 +40,16 @@ from meerschaum.connectors.sql.tables import get_tables
38
40
 
39
41
  fastapi_responses = attempt_import('fastapi.responses', lazy=False)
40
42
  StreamingResponse = fastapi_responses.StreamingResponse
41
- dateutil_parser = attempt_import('dateutil.parser', lazy=False)
42
43
  pipes_endpoint = endpoints['pipes']
43
44
  pd = attempt_import('pandas', lazy=False)
44
45
 
45
46
  MAX_RESPONSE_ROW_LIMIT: int = mrsm.get_config('system', 'api', 'data', 'max_response_row_limit')
46
47
 
47
48
 
48
- @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register', tags=['Pipes'])
49
+ @app.post(
50
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register',
51
+ tags=['Pipes: Attributes'],
52
+ )
49
53
  def register_pipe(
50
54
  connector_keys: str,
51
55
  metric_key: str,
@@ -80,18 +84,23 @@ def register_pipe(
80
84
  return results
81
85
 
82
86
 
83
- @app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete', tags=['Pipes'])
84
- def delete_pipe(
87
+ @app.patch(
88
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit',
89
+ tags=['Pipes: Attributes'],
90
+ )
91
+ def edit_pipe(
85
92
  connector_keys: str,
86
93
  metric_key: str,
87
94
  location_key: str,
95
+ parameters: dict,
88
96
  instance_keys: Optional[str] = None,
97
+ patch: bool = False,
89
98
  curr_user = (
90
99
  fastapi.Depends(manager) if not no_auth else None
91
100
  ),
92
101
  ):
93
102
  """
94
- Delete a Pipe (without dropping its table).
103
+ Edit an existing pipe's parameters.
95
104
  """
96
105
  allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
97
106
  if not allow_actions:
@@ -107,54 +116,27 @@ def delete_pipe(
107
116
  raise fastapi.HTTPException(
108
117
  status_code=409, detail=f"{pipe} is not registered."
109
118
  )
110
- results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
111
- pipes(instance_keys, refresh=True)
112
- return results
113
-
114
-
115
- @app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop', tags=['Pipes'])
116
- def drop_pipe(
117
- connector_keys: str,
118
- metric_key: str,
119
- location_key: str,
120
- instance_keys: Optional[str] = None,
121
- curr_user = (
122
- fastapi.Depends(manager) if not no_auth else None
123
- ),
124
- ):
125
- """
126
- Drop a pipes' underlying target table.
127
- """
128
- allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
129
- if not allow_actions:
130
- return False, (
131
- "The administrator for this server has not allowed actions.\n\n"
132
- "Please contact the system administrator, or if you are running this server, "
133
- "open the configuration file with `edit config system` and search for 'permissions'."
134
- " Under the keys `api:permissions:actions`, " +
135
- "you can toggle non-admin actions."
136
- )
137
- pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
138
- results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
119
+ pipe.parameters = parameters
120
+ results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
139
121
  pipes(instance_keys, refresh=True)
140
122
  return results
141
123
 
142
124
 
143
-
144
- @app.patch(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit', tags=['Pipes'])
145
- def edit_pipe(
125
+ @app.delete(
126
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete',
127
+ tags=['Pipes: Attributes'],
128
+ )
129
+ def delete_pipe(
146
130
  connector_keys: str,
147
131
  metric_key: str,
148
132
  location_key: str,
149
- parameters: dict,
150
133
  instance_keys: Optional[str] = None,
151
- patch: bool = False,
152
134
  curr_user = (
153
135
  fastapi.Depends(manager) if not no_auth else None
154
136
  ),
155
137
  ):
156
138
  """
157
- Edit an existing pipe's parameters.
139
+ Delete a Pipe (without dropping its table).
158
140
  """
159
141
  allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
160
142
  if not allow_actions:
@@ -170,13 +152,12 @@ def edit_pipe(
170
152
  raise fastapi.HTTPException(
171
153
  status_code=409, detail=f"{pipe} is not registered."
172
154
  )
173
- pipe.parameters = parameters
174
- results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
155
+ results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
175
156
  pipes(instance_keys, refresh=True)
176
157
  return results
177
158
 
178
159
 
179
- @app.get(pipes_endpoint + '/keys', tags=['Pipes'])
160
+ @app.get(pipes_endpoint + '/keys', tags=['Pipes: Attributes'])
180
161
  async def fetch_pipes_keys(
181
162
  connector_keys: str = "[]",
182
163
  metric_keys: str = "[]",
@@ -201,7 +182,7 @@ async def fetch_pipes_keys(
201
182
  return keys
202
183
 
203
184
 
204
- @app.get(pipes_endpoint, tags=['Pipes'])
185
+ @app.get(pipes_endpoint, tags=['Pipes: Attributes'])
205
186
  async def get_pipes(
206
187
  connector_keys: str = "",
207
188
  metric_keys: str = "",
@@ -225,7 +206,7 @@ async def get_pipes(
225
206
  return replace_pipes_in_dict(_get_pipes(**kw), lambda p: p.attributes)
226
207
 
227
208
 
228
- @app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes'])
209
+ @app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes: Attributes'])
229
210
  async def get_pipes_by_connector(
230
211
  connector_keys: str,
231
212
  instance_keys: Optional[str] = None,
@@ -243,7 +224,7 @@ async def get_pipes_by_connector(
243
224
  return replace_pipes_in_dict(pipes(instance_keys)[connector_keys], lambda p: p.attributes)
244
225
 
245
226
 
246
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes'])
227
+ @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes: Attributes'])
247
228
  async def get_pipes_by_connector_and_metric(
248
229
  connector_keys: str,
249
230
  metric_key: str,
@@ -271,8 +252,11 @@ async def get_pipes_by_connector_and_metric(
271
252
  )
272
253
 
273
254
 
274
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}', tags=['Pipes'])
275
- async def get_pipes_by_connector_and_metric_and_location(
255
+ @app.get(
256
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}',
257
+ tags=['Pipes: Attributes'],
258
+ )
259
+ async def get_pipe_by_connector_and_metric_and_location(
276
260
  connector_keys: str,
277
261
  metric_key: str,
278
262
  location_key: str,
@@ -299,10 +283,13 @@ async def get_pipes_by_connector_and_metric_and_location(
299
283
  detail=f"location_key '{location_key}' not found."
300
284
  )
301
285
 
302
- return str(pipes(instance_keys)[connector_keys][metric_key][location_key])
286
+ return pipes(instance_keys)[connector_keys][metric_key][location_key].attributes
303
287
 
304
288
 
305
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time', tags=['Pipes'])
289
+ @app.get(
290
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time',
291
+ tags=['Pipes: Data'],
292
+ )
306
293
  def get_sync_time(
307
294
  connector_keys: str,
308
295
  metric_key: str,
@@ -333,7 +320,10 @@ def get_sync_time(
333
320
  return sync_time
334
321
 
335
322
 
336
- @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
323
+ @app.post(
324
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
325
+ tags=['Pipes: Data'],
326
+ )
337
327
  def sync_pipe(
338
328
  connector_keys: str,
339
329
  metric_key: str,
@@ -377,7 +367,10 @@ def sync_pipe(
377
367
  return list((success, msg))
378
368
 
379
369
 
380
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
370
+ @app.get(
371
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
372
+ tags=['Pipes: Data'],
373
+ )
381
374
  def get_pipe_data(
382
375
  connector_keys: str,
383
376
  metric_key: str,
@@ -389,6 +382,9 @@ def get_pipe_data(
389
382
  end: Union[str, int, None] = None,
390
383
  params: Optional[str] = None,
391
384
  limit: int = MAX_RESPONSE_ROW_LIMIT,
385
+ order: str = 'asc',
386
+ as_chunks: bool = False,
387
+ chunk_interval: Optional[int] = None,
392
388
  curr_user = (
393
389
  fastapi.Depends(manager) if not no_auth else None
394
390
  ),
@@ -398,11 +394,16 @@ def get_pipe_data(
398
394
  See [`Pipe.get_data()`](https://docs.meerschaum.io/meerschaum.html#Pipe.get_data).
399
395
 
400
396
  Note that `select_columns`, `omit_columns`, and `params` are JSON-encoded strings.
397
+
398
+ Parameters
399
+ ----------
400
+ instance_keys: Optional[str], default None
401
+ The connector key to the instance on which the pipe is registered.
402
+ Defaults to the configured value for `meerschaum:api_instance`.
403
+
404
+ as_chunks: bool, default False
405
+ If `True`, return a chunk token to be consumed by the `/chunks` endpoint.
401
406
  """
402
- if is_int(begin):
403
- begin = int(begin)
404
- if is_int(end):
405
- end = int(end)
406
407
  if limit > MAX_RESPONSE_ROW_LIMIT:
407
408
  raise fastapi.HTTPException(
408
409
  status_code=413,
@@ -455,18 +456,38 @@ def get_pipe_data(
455
456
  )
456
457
 
457
458
  pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
459
+ begin, end = pipe.parse_date_bounds(begin, end)
458
460
  if not is_pipe_registered(pipe, pipes(instance_keys, refresh=True)):
459
461
  raise fastapi.HTTPException(
460
462
  status_code=409,
461
463
  detail="Pipe must be registered with the datetime column specified."
462
464
  )
463
465
 
464
- if pipe.target in ('users', 'plugins', 'pipes'):
466
+ if pipe.target in ('mrsm_users', 'mrsm_plugins', 'mrsm_pipes'):
465
467
  raise fastapi.HTTPException(
466
468
  status_code=409,
467
469
  detail=f"Cannot retrieve data from protected table '{pipe.target}'.",
468
470
  )
469
471
 
472
+ if as_chunks:
473
+ chunks_cursor_token = generate_chunks_cursor_token(
474
+ pipe,
475
+ select_columns=_select_columns,
476
+ omit_columns=_omit_columns,
477
+ begin=begin,
478
+ end=end,
479
+ params=_params,
480
+ limit=limit,
481
+ order=order,
482
+ debug=debug,
483
+ )
484
+ return fastapi.Response(
485
+ json.dumps({
486
+ 'chunks_cursor': chunks_cursor,
487
+ }),
488
+ media_type='application/json',
489
+ )
490
+
470
491
  df = pipe.get_data(
471
492
  select_columns=_select_columns,
472
493
  omit_columns=_omit_columns,
@@ -474,6 +495,7 @@ def get_pipe_data(
474
495
  end=end,
475
496
  params=_params,
476
497
  limit=min(limit, MAX_RESPONSE_ROW_LIMIT),
498
+ order=order,
477
499
  debug=debug,
478
500
  )
479
501
  if df is None:
@@ -489,7 +511,100 @@ def get_pipe_data(
489
511
  )
490
512
 
491
513
 
492
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv', tags=['Pipes'])
514
+ @app.get(
515
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk_bounds',
516
+ tags=['Pipes: Data'],
517
+ )
518
+ def get_pipe_chunk_bounds(
519
+ connector_keys: str,
520
+ metric_key: str,
521
+ location_key: str,
522
+ instance_keys: Optional[str] = None,
523
+ begin: Union[str, int, None] = None,
524
+ end: Union[str, int, None] = None,
525
+ bounded: bool = True,
526
+ chunk_interval_minutes: Union[int, None] = None,
527
+ ) -> List[List[Union[str, int, None]]]:
528
+ """
529
+ Return a list of request boundaries between `begin` and `end` (or the pipe's sync times).
530
+ Optionally specify the interval between chunk bounds
531
+ (defaults to the pipe's configured chunk interval).
532
+ """
533
+ pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
534
+ begin, end = pipe.parse_date_bounds(begin, end)
535
+ dt_col = pipe.columns.get('datetime', None)
536
+ dt_typ = pipe.dtypes.get(dt_col, 'datetime')
537
+ chunk_interval = None if chunk_interval_minutes is None else (
538
+ chunk_interval_minutes
539
+ if are_dtypes_equal(dt_typ, 'int')
540
+ else timedelta(minutes=chunk_interval_minutes)
541
+ )
542
+
543
+ chunk_bounds = pipe.get_chunk_bounds(
544
+ begin=begin,
545
+ end=end,
546
+ bounded=bounded,
547
+ chunk_interval=chunk_interval,
548
+ debug=debug,
549
+ )
550
+
551
+ return fastapi.Response(
552
+ json.dumps(chunk_bounds, default=json_serialize_value),
553
+ media_type='application/json',
554
+ )
555
+
556
+
557
+ @app.get(
558
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk/{chunk_token}',
559
+ tags=['Pipes: Data'],
560
+ )
561
+ def get_pipe_chunk(
562
+ connector_keys: str,
563
+ metric_key: str,
564
+ location_key: str,
565
+ chunk_token: str
566
+ ) -> Dict[str, Any]:
567
+ """
568
+ Consume a chunk token, returning the dataframe.
569
+ """
570
+
571
+
572
+ @app.delete(
573
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop',
574
+ tags=['Pipes: Data'],
575
+ )
576
+ def drop_pipe(
577
+ connector_keys: str,
578
+ metric_key: str,
579
+ location_key: str,
580
+ instance_keys: Optional[str] = None,
581
+ curr_user = (
582
+ fastapi.Depends(manager) if not no_auth else None
583
+ ),
584
+ ):
585
+ """
586
+ Drop a pipes' underlying target table.
587
+ """
588
+ allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
589
+ if not allow_actions:
590
+ return False, (
591
+ "The administrator for this server has not allowed actions.\n\n"
592
+ "Please contact the system administrator, or if you are running this server, "
593
+ "open the configuration file with `edit config system` and search for 'permissions'."
594
+ " Under the keys `api:permissions:actions`, " +
595
+ "you can toggle non-admin actions."
596
+ )
597
+ pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
598
+ results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
599
+ pipes(instance_keys, refresh=True)
600
+ return results
601
+
602
+
603
+
604
+ @app.get(
605
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv',
606
+ tags=['Pipes: Data'],
607
+ )
493
608
  def get_pipe_csv(
494
609
  connector_keys: str,
495
610
  metric_key: str,
@@ -505,24 +620,11 @@ def get_pipe_csv(
505
620
  """
506
621
  Get a pipe's data as a CSV file. Optionally set query boundaries.
507
622
  """
508
- if begin is not None:
509
- begin = (
510
- int(begin)
511
- if is_int(begin)
512
- else dateutil_parser.parse(begin)
513
- )
514
- if end is not None:
515
- end = (
516
- int(end)
517
- if is_int(end)
518
- else dateutil_parser.parse(end)
519
- )
520
623
 
521
624
  _params = {}
522
625
  if params == 'null':
523
626
  params = None
524
627
  if params is not None:
525
- import json
526
628
  try:
527
629
  _params = json.loads(params)
528
630
  except Exception:
@@ -541,6 +643,7 @@ def get_pipe_csv(
541
643
  detail="Pipe must be registered."
542
644
  )
543
645
 
646
+ begin, end = pipe.parse_date_bounds(begin, end)
544
647
  dt_col = pipe.columns.get('datetime', None)
545
648
  if dt_col:
546
649
  if begin is None:
@@ -568,7 +671,10 @@ def get_pipe_csv(
568
671
  return response
569
672
 
570
673
 
571
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id', tags=['Pipes'])
674
+ @app.get(
675
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id',
676
+ tags=['Pipes: Attributes'],
677
+ )
572
678
  def get_pipe_id(
573
679
  connector_keys: str,
574
680
  metric_key: str,
@@ -589,7 +695,7 @@ def get_pipe_id(
589
695
 
590
696
  @app.get(
591
697
  pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/attributes',
592
- tags=['Pipes']
698
+ tags=['Pipes: Attributes'],
593
699
  )
594
700
  def get_pipe_attributes(
595
701
  connector_keys: str,
@@ -610,7 +716,10 @@ def get_pipe_attributes(
610
716
  ).attributes
611
717
 
612
718
 
613
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists', tags=['Pipes'])
719
+ @app.get(
720
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists',
721
+ tags=['Pipes: Data'],
722
+ )
614
723
  def get_pipe_exists(
615
724
  connector_keys: str,
616
725
  metric_key: str,
@@ -624,7 +733,7 @@ def get_pipe_exists(
624
733
  return get_pipe(connector_keys, metric_key, location_key, instance_keys).exists(debug=debug)
625
734
 
626
735
 
627
- @app.post(endpoints['metadata'], tags=['Pipes'])
736
+ @app.post(endpoints['metadata'], tags=['Misc'])
628
737
  def create_metadata(
629
738
  instance_keys: Optional[str] = None,
630
739
  curr_user = (
@@ -642,7 +751,10 @@ def create_metadata(
642
751
  return True
643
752
 
644
753
 
645
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount', tags=['Pipes'])
754
+ @app.get(
755
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount',
756
+ tags=['Pipes: Data'],
757
+ )
646
758
  def get_pipe_rowcount(
647
759
  connector_keys: str,
648
760
  metric_key: str,
@@ -693,7 +805,7 @@ def get_pipe_rowcount(
693
805
 
694
806
  @app.get(
695
807
  pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/types',
696
- tags=['Pipes']
808
+ tags=['Pipes: Data'],
697
809
  )
698
810
  def get_pipe_columns_types(
699
811
  connector_keys: str,
@@ -721,7 +833,7 @@ def get_pipe_columns_types(
721
833
 
722
834
  @app.get(
723
835
  pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/indices',
724
- tags=['Pipes']
836
+ tags=['Pipes: Data'],
725
837
  )
726
838
  def get_pipe_columns_indices(
727
839
  connector_keys: str,
@@ -775,7 +887,7 @@ def get_pipe_columns_indices(
775
887
 
776
888
  @app.get(
777
889
  pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/indices/names',
778
- tags=['Pipes']
890
+ tags=['Pipes: Data']
779
891
  )
780
892
  def get_pipe_index_names(
781
893
  connector_keys: str,
@@ -785,7 +897,7 @@ def get_pipe_index_names(
785
897
  curr_user=(
786
898
  fastapi.Depends(manager) if not no_auth else None
787
899
  ),
788
- ) -> Dict[str, List[Dict[str, str]]]:
900
+ ) -> Dict[str, Union[str, Dict[str, str], List[Dict[str, str]]]]:
789
901
  """
790
902
  Return a dictionary of index keys and index names.
791
903
 
@@ -796,4 +908,4 @@ def get_pipe_index_names(
796
908
  metric_key,
797
909
  location_key,
798
910
  instance_keys,
799
- ).get_indices(debug=debug)
911
+ ).get_indices()
@@ -113,6 +113,9 @@ default_system_config = {
113
113
  },
114
114
  'data': {
115
115
  'max_response_row_limit': 100_000,
116
+ 'chunks': {
117
+ 'ttl_seconds': 1800,
118
+ },
116
119
  },
117
120
  'endpoints': {
118
121
  'docs_in_production': True,
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.8.3"
5
+ __version__ = "2.9.0.dev1"
@@ -6,8 +6,10 @@
6
6
  Interact with Meerschaum APIs. May be chained together (see 'meerschaum:api_instance' in your config.yaml).
7
7
  """
8
8
 
9
+ from __future__ import annotations
10
+
9
11
  from datetime import datetime, timedelta, timezone
10
- from meerschaum.utils.typing import Optional, List
12
+ from meerschaum.utils.typing import Optional, List, Union
11
13
  from meerschaum.connectors import Connector
12
14
  from meerschaum.utils.warnings import warn, error
13
15
  from meerschaum.utils.packages import attempt_import
@@ -43,6 +45,7 @@ class APIConnector(Connector):
43
45
  )
44
46
  from ._misc import get_mrsm_version, get_chaining_status
45
47
  from ._pipes import (
48
+ get_pipe_instance_keys,
46
49
  register_pipe,
47
50
  fetch_pipes_keys,
48
51
  edit_pipe,
@@ -143,6 +146,7 @@ class APIConnector(Connector):
143
146
  self._token = None
144
147
  self._expires = None
145
148
  self._session = None
149
+ self._instance_keys = self.__dict__.get('instance_keys', None)
146
150
 
147
151
 
148
152
  @property
@@ -195,3 +199,10 @@ class APIConnector(Connector):
195
199
  warn(msg, stack=False)
196
200
  self._emitted_warning = True
197
201
  return self._token
202
+
203
+ @property
204
+ def instance_keys(self) -> Union[str, None]:
205
+ """
206
+ Return the instance keys to be sent alongside pipe requests.
207
+ """
208
+ return self._instance_keys
@@ -31,6 +31,14 @@ def pipe_r_url(
31
31
  )
32
32
 
33
33
 
34
+ def get_pipe_instance_keys(self, pipe: mrsm.Pipe) -> Union[str, None]:
35
+ """
36
+ Return the configured instance keys for a pipe if set,
37
+ else fall back to the default `instance_keys` for this `APIConnector`.
38
+ """
39
+ return pipe.parameters.get('instance_keys', self.instance_keys)
40
+
41
+
34
42
  def register_pipe(
35
43
  self,
36
44
  pipe: mrsm.Pipe,
@@ -40,13 +48,12 @@ def register_pipe(
40
48
  Returns a tuple of (success_bool, response_dict).
41
49
  """
42
50
  from meerschaum.utils.debug import dprint
43
- ### NOTE: if `parameters` is supplied in the Pipe constructor,
44
- ### then `pipe.parameters` will exist and not be fetched from the database.
45
51
  r_url = pipe_r_url(pipe)
46
52
  response = self.post(
47
53
  r_url + '/register',
48
- json = pipe.parameters,
49
- debug = debug,
54
+ json=pipe._attributes.get('parameters', {}),
55
+ params={'instance_keys': self.get_pipe_instance_keys(pipe)},
56
+ debug=debug,
50
57
  )
51
58
  if debug:
52
59
  dprint(response.text)
@@ -79,9 +86,9 @@ def edit_pipe(
79
86
  r_url = pipe_r_url(pipe)
80
87
  response = self.patch(
81
88
  r_url + '/edit',
82
- params = {'patch': patch,},
83
- json = pipe.parameters,
84
- debug = debug,
89
+ params={'patch': patch, 'instance_keys': self.get_pipe_instance_keys(pipe)},
90
+ json=pipe.parameters,
91
+ debug=debug,
85
92
  )
86
93
  if debug:
87
94
  dprint(response.text)
@@ -149,12 +156,13 @@ def fetch_pipes_keys(
149
156
  try:
150
157
  j = self.get(
151
158
  r_url,
152
- params = {
159
+ params={
153
160
  'connector_keys': json.dumps(connector_keys),
154
161
  'metric_keys': json.dumps(metric_keys),
155
162
  'location_keys': json.dumps(location_keys),
156
163
  'tags': json.dumps(tags),
157
164
  'params': json.dumps(params),
165
+ 'instance_keys': self.instance_keys,
158
166
  },
159
167
  debug=debug
160
168
  ).json()
@@ -250,8 +258,10 @@ def sync_pipe(
250
258
  chunks = (df[i] for i in more_itertools.chunked(df, _chunksize))
251
259
 
252
260
  ### Send columns in case the user has defined them locally.
261
+ request_params = kw.copy()
253
262
  if pipe.columns:
254
- kw['columns'] = json.dumps(pipe.columns)
263
+ request_params['columns'] = json.dumps(pipe.columns)
264
+ request_params['instance_keys'] = self.get_pipe_instance_keys(pipe)
255
265
  r_url = pipe_r_url(pipe) + '/data'
256
266
 
257
267
  rowcount = 0
@@ -268,10 +278,9 @@ def sync_pipe(
268
278
  try:
269
279
  response = self.post(
270
280
  r_url,
271
- ### handles check_existing
272
- params = kw,
273
- data = json_str,
274
- debug = debug
281
+ params=request_params,
282
+ data=json_str,
283
+ debug=debug,
275
284
  )
276
285
  except Exception as e:
277
286
  msg = f"Failed to post a chunk to {pipe}:\n{e}"
@@ -323,7 +332,8 @@ def delete_pipe(
323
332
  r_url = pipe_r_url(pipe)
324
333
  response = self.delete(
325
334
  r_url + '/delete',
326
- debug = debug,
335
+ params={'instance_keys': self.get_pipe_instance_keys(pipe)},
336
+ debug=debug,
327
337
  )
328
338
  if debug:
329
339
  dprint(response.text)
@@ -361,7 +371,9 @@ def get_pipe_data(
361
371
  'omit_columns': json.dumps(omit_columns),
362
372
  'begin': begin,
363
373
  'end': end,
364
- 'params': json.dumps(params, default=str)
374
+ 'params': json.dumps(params, default=str),
375
+ 'instance': self.get_pipe_instance_keys(pipe),
376
+ 'as_chunks': as_chunks,
365
377
  },
366
378
  debug=debug
367
379
  )