meerschaum 2.8.4__py3-none-any.whl → 2.9.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. meerschaum/api/_chunks.py +67 -0
  2. meerschaum/api/dash/callbacks/custom.py +23 -2
  3. meerschaum/api/dash/callbacks/dashboard.py +41 -3
  4. meerschaum/api/dash/components.py +27 -19
  5. meerschaum/api/dash/pages/dashboard.py +11 -9
  6. meerschaum/api/dash/pages/plugins.py +31 -27
  7. meerschaum/api/dash/webterm.py +6 -3
  8. meerschaum/api/resources/static/css/dash.css +1 -1
  9. meerschaum/api/resources/templates/termpage.html +4 -0
  10. meerschaum/api/routes/_pipes.py +191 -78
  11. meerschaum/config/_default.py +3 -0
  12. meerschaum/config/_version.py +1 -1
  13. meerschaum/connectors/api/_APIConnector.py +12 -1
  14. meerschaum/connectors/api/_pipes.py +27 -15
  15. meerschaum/connectors/api/_plugins.py +51 -45
  16. meerschaum/connectors/api/_request.py +1 -1
  17. meerschaum/connectors/parse.py +1 -2
  18. meerschaum/core/Pipe/_data.py +1 -2
  19. meerschaum/plugins/_Plugin.py +21 -5
  20. meerschaum/plugins/__init__.py +6 -4
  21. meerschaum/utils/formatting/_shell.py +1 -4
  22. meerschaum/utils/packages/_packages.py +1 -0
  23. meerschaum/utils/venv/__init__.py +2 -0
  24. {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/METADATA +4 -1
  25. {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/RECORD +31 -30
  26. {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/WHEEL +1 -1
  27. {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/LICENSE +0 -0
  28. {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/NOTICE +0 -0
  29. {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/entry_points.txt +0 -0
  30. {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/top_level.txt +0 -0
  31. {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/zip-safe +0 -0
@@ -27,8 +27,10 @@ from meerschaum.api import (
27
27
  debug,
28
28
  no_auth,
29
29
  )
30
+ from meerschaum.api._chunks import generate_chunks_cursor_token
30
31
  from meerschaum.utils.packages import attempt_import
31
32
  from meerschaum.utils.dataframe import to_json
33
+ from meerschaum.utils.dtypes import are_dtypes_equal, json_serialize_value
32
34
  from meerschaum.utils.misc import (
33
35
  is_pipe_registered,
34
36
  is_int,
@@ -38,14 +40,16 @@ from meerschaum.connectors.sql.tables import get_tables
38
40
 
39
41
  fastapi_responses = attempt_import('fastapi.responses', lazy=False)
40
42
  StreamingResponse = fastapi_responses.StreamingResponse
41
- dateutil_parser = attempt_import('dateutil.parser', lazy=False)
42
43
  pipes_endpoint = endpoints['pipes']
43
44
  pd = attempt_import('pandas', lazy=False)
44
45
 
45
46
  MAX_RESPONSE_ROW_LIMIT: int = mrsm.get_config('system', 'api', 'data', 'max_response_row_limit')
46
47
 
47
48
 
48
- @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register', tags=['Pipes'])
49
+ @app.post(
50
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register',
51
+ tags=['Pipes: Attributes'],
52
+ )
49
53
  def register_pipe(
50
54
  connector_keys: str,
51
55
  metric_key: str,
@@ -80,18 +84,23 @@ def register_pipe(
80
84
  return results
81
85
 
82
86
 
83
- @app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete', tags=['Pipes'])
84
- def delete_pipe(
87
+ @app.patch(
88
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit',
89
+ tags=['Pipes: Attributes'],
90
+ )
91
+ def edit_pipe(
85
92
  connector_keys: str,
86
93
  metric_key: str,
87
94
  location_key: str,
95
+ parameters: dict,
88
96
  instance_keys: Optional[str] = None,
97
+ patch: bool = False,
89
98
  curr_user = (
90
99
  fastapi.Depends(manager) if not no_auth else None
91
100
  ),
92
101
  ):
93
102
  """
94
- Delete a Pipe (without dropping its table).
103
+ Edit an existing pipe's parameters.
95
104
  """
96
105
  allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
97
106
  if not allow_actions:
@@ -107,54 +116,27 @@ def delete_pipe(
107
116
  raise fastapi.HTTPException(
108
117
  status_code=409, detail=f"{pipe} is not registered."
109
118
  )
110
- results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
111
- pipes(instance_keys, refresh=True)
112
- return results
113
-
114
-
115
- @app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop', tags=['Pipes'])
116
- def drop_pipe(
117
- connector_keys: str,
118
- metric_key: str,
119
- location_key: str,
120
- instance_keys: Optional[str] = None,
121
- curr_user = (
122
- fastapi.Depends(manager) if not no_auth else None
123
- ),
124
- ):
125
- """
126
- Drop a pipes' underlying target table.
127
- """
128
- allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
129
- if not allow_actions:
130
- return False, (
131
- "The administrator for this server has not allowed actions.\n\n"
132
- "Please contact the system administrator, or if you are running this server, "
133
- "open the configuration file with `edit config system` and search for 'permissions'."
134
- " Under the keys `api:permissions:actions`, " +
135
- "you can toggle non-admin actions."
136
- )
137
- pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
138
- results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
119
+ pipe.parameters = parameters
120
+ results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
139
121
  pipes(instance_keys, refresh=True)
140
122
  return results
141
123
 
142
124
 
143
-
144
- @app.patch(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit', tags=['Pipes'])
145
- def edit_pipe(
125
+ @app.delete(
126
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete',
127
+ tags=['Pipes: Attributes'],
128
+ )
129
+ def delete_pipe(
146
130
  connector_keys: str,
147
131
  metric_key: str,
148
132
  location_key: str,
149
- parameters: dict,
150
133
  instance_keys: Optional[str] = None,
151
- patch: bool = False,
152
134
  curr_user = (
153
135
  fastapi.Depends(manager) if not no_auth else None
154
136
  ),
155
137
  ):
156
138
  """
157
- Edit an existing pipe's parameters.
139
+ Delete a Pipe (without dropping its table).
158
140
  """
159
141
  allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
160
142
  if not allow_actions:
@@ -170,13 +152,12 @@ def edit_pipe(
170
152
  raise fastapi.HTTPException(
171
153
  status_code=409, detail=f"{pipe} is not registered."
172
154
  )
173
- pipe.parameters = parameters
174
- results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
155
+ results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
175
156
  pipes(instance_keys, refresh=True)
176
157
  return results
177
158
 
178
159
 
179
- @app.get(pipes_endpoint + '/keys', tags=['Pipes'])
160
+ @app.get(pipes_endpoint + '/keys', tags=['Pipes: Attributes'])
180
161
  async def fetch_pipes_keys(
181
162
  connector_keys: str = "[]",
182
163
  metric_keys: str = "[]",
@@ -201,7 +182,7 @@ async def fetch_pipes_keys(
201
182
  return keys
202
183
 
203
184
 
204
- @app.get(pipes_endpoint, tags=['Pipes'])
185
+ @app.get(pipes_endpoint, tags=['Pipes: Attributes'])
205
186
  async def get_pipes(
206
187
  connector_keys: str = "",
207
188
  metric_keys: str = "",
@@ -225,7 +206,7 @@ async def get_pipes(
225
206
  return replace_pipes_in_dict(_get_pipes(**kw), lambda p: p.attributes)
226
207
 
227
208
 
228
- @app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes'])
209
+ @app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes: Attributes'])
229
210
  async def get_pipes_by_connector(
230
211
  connector_keys: str,
231
212
  instance_keys: Optional[str] = None,
@@ -243,7 +224,7 @@ async def get_pipes_by_connector(
243
224
  return replace_pipes_in_dict(pipes(instance_keys)[connector_keys], lambda p: p.attributes)
244
225
 
245
226
 
246
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes'])
227
+ @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes: Attributes'])
247
228
  async def get_pipes_by_connector_and_metric(
248
229
  connector_keys: str,
249
230
  metric_key: str,
@@ -271,7 +252,10 @@ async def get_pipes_by_connector_and_metric(
271
252
  )
272
253
 
273
254
 
274
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}', tags=['Pipes'])
255
+ @app.get(
256
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}',
257
+ tags=['Pipes: Attributes'],
258
+ )
275
259
  async def get_pipe_by_connector_and_metric_and_location(
276
260
  connector_keys: str,
277
261
  metric_key: str,
@@ -302,7 +286,10 @@ async def get_pipe_by_connector_and_metric_and_location(
302
286
  return pipes(instance_keys)[connector_keys][metric_key][location_key].attributes
303
287
 
304
288
 
305
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time', tags=['Pipes'])
289
+ @app.get(
290
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time',
291
+ tags=['Pipes: Data'],
292
+ )
306
293
  def get_sync_time(
307
294
  connector_keys: str,
308
295
  metric_key: str,
@@ -333,7 +320,10 @@ def get_sync_time(
333
320
  return sync_time
334
321
 
335
322
 
336
- @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
323
+ @app.post(
324
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
325
+ tags=['Pipes: Data'],
326
+ )
337
327
  def sync_pipe(
338
328
  connector_keys: str,
339
329
  metric_key: str,
@@ -377,7 +367,10 @@ def sync_pipe(
377
367
  return list((success, msg))
378
368
 
379
369
 
380
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
370
+ @app.get(
371
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
372
+ tags=['Pipes: Data'],
373
+ )
381
374
  def get_pipe_data(
382
375
  connector_keys: str,
383
376
  metric_key: str,
@@ -389,6 +382,9 @@ def get_pipe_data(
389
382
  end: Union[str, int, None] = None,
390
383
  params: Optional[str] = None,
391
384
  limit: int = MAX_RESPONSE_ROW_LIMIT,
385
+ order: str = 'asc',
386
+ as_chunks: bool = False,
387
+ chunk_interval: Optional[int] = None,
392
388
  curr_user = (
393
389
  fastapi.Depends(manager) if not no_auth else None
394
390
  ),
@@ -398,11 +394,16 @@ def get_pipe_data(
398
394
  See [`Pipe.get_data()`](https://docs.meerschaum.io/meerschaum.html#Pipe.get_data).
399
395
 
400
396
  Note that `select_columns`, `omit_columns`, and `params` are JSON-encoded strings.
397
+
398
+ Parameters
399
+ ----------
400
+ instance_keys: Optional[str], default None
401
+ The connector key to the instance on which the pipe is registered.
402
+ Defaults to the configured value for `meerschaum:api_instance`.
403
+
404
+ as_chunks: bool, default False
405
+ If `True`, return a chunk token to be consumed by the `/chunks` endpoint.
401
406
  """
402
- if is_int(begin):
403
- begin = int(begin)
404
- if is_int(end):
405
- end = int(end)
406
407
  if limit > MAX_RESPONSE_ROW_LIMIT:
407
408
  raise fastapi.HTTPException(
408
409
  status_code=413,
@@ -455,18 +456,38 @@ def get_pipe_data(
455
456
  )
456
457
 
457
458
  pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
459
+ begin, end = pipe.parse_date_bounds(begin, end)
458
460
  if not is_pipe_registered(pipe, pipes(instance_keys, refresh=True)):
459
461
  raise fastapi.HTTPException(
460
462
  status_code=409,
461
463
  detail="Pipe must be registered with the datetime column specified."
462
464
  )
463
465
 
464
- if pipe.target in ('users', 'plugins', 'pipes'):
466
+ if pipe.target in ('mrsm_users', 'mrsm_plugins', 'mrsm_pipes'):
465
467
  raise fastapi.HTTPException(
466
468
  status_code=409,
467
469
  detail=f"Cannot retrieve data from protected table '{pipe.target}'.",
468
470
  )
469
471
 
472
+ if as_chunks:
473
+ chunks_cursor_token = generate_chunks_cursor_token(
474
+ pipe,
475
+ select_columns=_select_columns,
476
+ omit_columns=_omit_columns,
477
+ begin=begin,
478
+ end=end,
479
+ params=_params,
480
+ limit=limit,
481
+ order=order,
482
+ debug=debug,
483
+ )
484
+ return fastapi.Response(
485
+ json.dumps({
486
+ 'chunks_cursor': chunks_cursor,
487
+ }),
488
+ media_type='application/json',
489
+ )
490
+
470
491
  df = pipe.get_data(
471
492
  select_columns=_select_columns,
472
493
  omit_columns=_omit_columns,
@@ -474,6 +495,7 @@ def get_pipe_data(
474
495
  end=end,
475
496
  params=_params,
476
497
  limit=min(limit, MAX_RESPONSE_ROW_LIMIT),
498
+ order=order,
477
499
  debug=debug,
478
500
  )
479
501
  if df is None:
@@ -489,7 +511,100 @@ def get_pipe_data(
489
511
  )
490
512
 
491
513
 
492
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv', tags=['Pipes'])
514
+ @app.get(
515
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk_bounds',
516
+ tags=['Pipes: Data'],
517
+ )
518
+ def get_pipe_chunk_bounds(
519
+ connector_keys: str,
520
+ metric_key: str,
521
+ location_key: str,
522
+ instance_keys: Optional[str] = None,
523
+ begin: Union[str, int, None] = None,
524
+ end: Union[str, int, None] = None,
525
+ bounded: bool = True,
526
+ chunk_interval_minutes: Union[int, None] = None,
527
+ ) -> List[List[Union[str, int, None]]]:
528
+ """
529
+ Return a list of request boundaries between `begin` and `end` (or the pipe's sync times).
530
+ Optionally specify the interval between chunk bounds
531
+ (defaults to the pipe's configured chunk interval).
532
+ """
533
+ pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
534
+ begin, end = pipe.parse_date_bounds(begin, end)
535
+ dt_col = pipe.columns.get('datetime', None)
536
+ dt_typ = pipe.dtypes.get(dt_col, 'datetime')
537
+ chunk_interval = None if chunk_interval_minutes is None else (
538
+ chunk_interval_minutes
539
+ if are_dtypes_equal(dt_typ, 'int')
540
+ else timedelta(minutes=chunk_interval_minutes)
541
+ )
542
+
543
+ chunk_bounds = pipe.get_chunk_bounds(
544
+ begin=begin,
545
+ end=end,
546
+ bounded=bounded,
547
+ chunk_interval=chunk_interval,
548
+ debug=debug,
549
+ )
550
+
551
+ return fastapi.Response(
552
+ json.dumps(chunk_bounds, default=json_serialize_value),
553
+ media_type='application/json',
554
+ )
555
+
556
+
557
+ @app.get(
558
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk/{chunk_token}',
559
+ tags=['Pipes: Data'],
560
+ )
561
+ def get_pipe_chunk(
562
+ connector_keys: str,
563
+ metric_key: str,
564
+ location_key: str,
565
+ chunk_token: str
566
+ ) -> Dict[str, Any]:
567
+ """
568
+ Consume a chunk token, returning the dataframe.
569
+ """
570
+
571
+
572
+ @app.delete(
573
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop',
574
+ tags=['Pipes: Data'],
575
+ )
576
+ def drop_pipe(
577
+ connector_keys: str,
578
+ metric_key: str,
579
+ location_key: str,
580
+ instance_keys: Optional[str] = None,
581
+ curr_user = (
582
+ fastapi.Depends(manager) if not no_auth else None
583
+ ),
584
+ ):
585
+ """
586
+ Drop a pipes' underlying target table.
587
+ """
588
+ allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
589
+ if not allow_actions:
590
+ return False, (
591
+ "The administrator for this server has not allowed actions.\n\n"
592
+ "Please contact the system administrator, or if you are running this server, "
593
+ "open the configuration file with `edit config system` and search for 'permissions'."
594
+ " Under the keys `api:permissions:actions`, " +
595
+ "you can toggle non-admin actions."
596
+ )
597
+ pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
598
+ results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
599
+ pipes(instance_keys, refresh=True)
600
+ return results
601
+
602
+
603
+
604
+ @app.get(
605
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv',
606
+ tags=['Pipes: Data'],
607
+ )
493
608
  def get_pipe_csv(
494
609
  connector_keys: str,
495
610
  metric_key: str,
@@ -505,18 +620,6 @@ def get_pipe_csv(
505
620
  """
506
621
  Get a pipe's data as a CSV file. Optionally set query boundaries.
507
622
  """
508
- if begin is not None:
509
- begin = (
510
- int(begin)
511
- if is_int(begin)
512
- else dateutil_parser.parse(begin)
513
- )
514
- if end is not None:
515
- end = (
516
- int(end)
517
- if is_int(end)
518
- else dateutil_parser.parse(end)
519
- )
520
623
 
521
624
  _params = {}
522
625
  if params == 'null':
@@ -540,6 +643,7 @@ def get_pipe_csv(
540
643
  detail="Pipe must be registered."
541
644
  )
542
645
 
646
+ begin, end = pipe.parse_date_bounds(begin, end)
543
647
  dt_col = pipe.columns.get('datetime', None)
544
648
  if dt_col:
545
649
  if begin is None:
@@ -567,7 +671,10 @@ def get_pipe_csv(
567
671
  return response
568
672
 
569
673
 
570
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id', tags=['Pipes'])
674
+ @app.get(
675
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id',
676
+ tags=['Pipes: Attributes'],
677
+ )
571
678
  def get_pipe_id(
572
679
  connector_keys: str,
573
680
  metric_key: str,
@@ -588,7 +695,7 @@ def get_pipe_id(
588
695
 
589
696
  @app.get(
590
697
  pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/attributes',
591
- tags=['Pipes']
698
+ tags=['Pipes: Attributes'],
592
699
  )
593
700
  def get_pipe_attributes(
594
701
  connector_keys: str,
@@ -609,7 +716,10 @@ def get_pipe_attributes(
609
716
  ).attributes
610
717
 
611
718
 
612
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists', tags=['Pipes'])
719
+ @app.get(
720
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists',
721
+ tags=['Pipes: Data'],
722
+ )
613
723
  def get_pipe_exists(
614
724
  connector_keys: str,
615
725
  metric_key: str,
@@ -623,7 +733,7 @@ def get_pipe_exists(
623
733
  return get_pipe(connector_keys, metric_key, location_key, instance_keys).exists(debug=debug)
624
734
 
625
735
 
626
- @app.post(endpoints['metadata'], tags=['Pipes'])
736
+ @app.post(endpoints['metadata'], tags=['Misc'])
627
737
  def create_metadata(
628
738
  instance_keys: Optional[str] = None,
629
739
  curr_user = (
@@ -641,7 +751,10 @@ def create_metadata(
641
751
  return True
642
752
 
643
753
 
644
- @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount', tags=['Pipes'])
754
+ @app.get(
755
+ pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount',
756
+ tags=['Pipes: Data'],
757
+ )
645
758
  def get_pipe_rowcount(
646
759
  connector_keys: str,
647
760
  metric_key: str,
@@ -692,7 +805,7 @@ def get_pipe_rowcount(
692
805
 
693
806
  @app.get(
694
807
  pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/types',
695
- tags=['Pipes']
808
+ tags=['Pipes: Data'],
696
809
  )
697
810
  def get_pipe_columns_types(
698
811
  connector_keys: str,
@@ -720,7 +833,7 @@ def get_pipe_columns_types(
720
833
 
721
834
  @app.get(
722
835
  pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/indices',
723
- tags=['Pipes']
836
+ tags=['Pipes: Data'],
724
837
  )
725
838
  def get_pipe_columns_indices(
726
839
  connector_keys: str,
@@ -774,7 +887,7 @@ def get_pipe_columns_indices(
774
887
 
775
888
  @app.get(
776
889
  pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/indices/names',
777
- tags=['Pipes']
890
+ tags=['Pipes: Data']
778
891
  )
779
892
  def get_pipe_index_names(
780
893
  connector_keys: str,
@@ -784,7 +897,7 @@ def get_pipe_index_names(
784
897
  curr_user=(
785
898
  fastapi.Depends(manager) if not no_auth else None
786
899
  ),
787
- ) -> Dict[str, List[Dict[str, str]]]:
900
+ ) -> Dict[str, Union[str, Dict[str, str], List[Dict[str, str]]]]:
788
901
  """
789
902
  Return a dictionary of index keys and index names.
790
903
 
@@ -795,4 +908,4 @@ def get_pipe_index_names(
795
908
  metric_key,
796
909
  location_key,
797
910
  instance_keys,
798
- ).get_indices(debug=debug)
911
+ ).get_indices()
@@ -113,6 +113,9 @@ default_system_config = {
113
113
  },
114
114
  'data': {
115
115
  'max_response_row_limit': 100_000,
116
+ 'chunks': {
117
+ 'ttl_seconds': 1800,
118
+ },
116
119
  },
117
120
  'endpoints': {
118
121
  'docs_in_production': True,
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.8.4"
5
+ __version__ = "2.9.0.dev1"
@@ -6,8 +6,10 @@
6
6
  Interact with Meerschaum APIs. May be chained together (see 'meerschaum:api_instance' in your config.yaml).
7
7
  """
8
8
 
9
+ from __future__ import annotations
10
+
9
11
  from datetime import datetime, timedelta, timezone
10
- from meerschaum.utils.typing import Optional, List
12
+ from meerschaum.utils.typing import Optional, List, Union
11
13
  from meerschaum.connectors import Connector
12
14
  from meerschaum.utils.warnings import warn, error
13
15
  from meerschaum.utils.packages import attempt_import
@@ -43,6 +45,7 @@ class APIConnector(Connector):
43
45
  )
44
46
  from ._misc import get_mrsm_version, get_chaining_status
45
47
  from ._pipes import (
48
+ get_pipe_instance_keys,
46
49
  register_pipe,
47
50
  fetch_pipes_keys,
48
51
  edit_pipe,
@@ -143,6 +146,7 @@ class APIConnector(Connector):
143
146
  self._token = None
144
147
  self._expires = None
145
148
  self._session = None
149
+ self._instance_keys = self.__dict__.get('instance_keys', None)
146
150
 
147
151
 
148
152
  @property
@@ -195,3 +199,10 @@ class APIConnector(Connector):
195
199
  warn(msg, stack=False)
196
200
  self._emitted_warning = True
197
201
  return self._token
202
+
203
+ @property
204
+ def instance_keys(self) -> Union[str, None]:
205
+ """
206
+ Return the instance keys to be sent alongside pipe requests.
207
+ """
208
+ return self._instance_keys
@@ -31,6 +31,14 @@ def pipe_r_url(
31
31
  )
32
32
 
33
33
 
34
+ def get_pipe_instance_keys(self, pipe: mrsm.Pipe) -> Union[str, None]:
35
+ """
36
+ Return the configured instance keys for a pipe if set,
37
+ else fall back to the default `instance_keys` for this `APIConnector`.
38
+ """
39
+ return pipe.parameters.get('instance_keys', self.instance_keys)
40
+
41
+
34
42
  def register_pipe(
35
43
  self,
36
44
  pipe: mrsm.Pipe,
@@ -40,13 +48,12 @@ def register_pipe(
40
48
  Returns a tuple of (success_bool, response_dict).
41
49
  """
42
50
  from meerschaum.utils.debug import dprint
43
- ### NOTE: if `parameters` is supplied in the Pipe constructor,
44
- ### then `pipe.parameters` will exist and not be fetched from the database.
45
51
  r_url = pipe_r_url(pipe)
46
52
  response = self.post(
47
53
  r_url + '/register',
48
- json = pipe.parameters,
49
- debug = debug,
54
+ json=pipe._attributes.get('parameters', {}),
55
+ params={'instance_keys': self.get_pipe_instance_keys(pipe)},
56
+ debug=debug,
50
57
  )
51
58
  if debug:
52
59
  dprint(response.text)
@@ -79,9 +86,9 @@ def edit_pipe(
79
86
  r_url = pipe_r_url(pipe)
80
87
  response = self.patch(
81
88
  r_url + '/edit',
82
- params = {'patch': patch,},
83
- json = pipe.parameters,
84
- debug = debug,
89
+ params={'patch': patch, 'instance_keys': self.get_pipe_instance_keys(pipe)},
90
+ json=pipe.parameters,
91
+ debug=debug,
85
92
  )
86
93
  if debug:
87
94
  dprint(response.text)
@@ -149,12 +156,13 @@ def fetch_pipes_keys(
149
156
  try:
150
157
  j = self.get(
151
158
  r_url,
152
- params = {
159
+ params={
153
160
  'connector_keys': json.dumps(connector_keys),
154
161
  'metric_keys': json.dumps(metric_keys),
155
162
  'location_keys': json.dumps(location_keys),
156
163
  'tags': json.dumps(tags),
157
164
  'params': json.dumps(params),
165
+ 'instance_keys': self.instance_keys,
158
166
  },
159
167
  debug=debug
160
168
  ).json()
@@ -250,8 +258,10 @@ def sync_pipe(
250
258
  chunks = (df[i] for i in more_itertools.chunked(df, _chunksize))
251
259
 
252
260
  ### Send columns in case the user has defined them locally.
261
+ request_params = kw.copy()
253
262
  if pipe.columns:
254
- kw['columns'] = json.dumps(pipe.columns)
263
+ request_params['columns'] = json.dumps(pipe.columns)
264
+ request_params['instance_keys'] = self.get_pipe_instance_keys(pipe)
255
265
  r_url = pipe_r_url(pipe) + '/data'
256
266
 
257
267
  rowcount = 0
@@ -268,10 +278,9 @@ def sync_pipe(
268
278
  try:
269
279
  response = self.post(
270
280
  r_url,
271
- ### handles check_existing
272
- params = kw,
273
- data = json_str,
274
- debug = debug
281
+ params=request_params,
282
+ data=json_str,
283
+ debug=debug,
275
284
  )
276
285
  except Exception as e:
277
286
  msg = f"Failed to post a chunk to {pipe}:\n{e}"
@@ -323,7 +332,8 @@ def delete_pipe(
323
332
  r_url = pipe_r_url(pipe)
324
333
  response = self.delete(
325
334
  r_url + '/delete',
326
- debug = debug,
335
+ params={'instance_keys': self.get_pipe_instance_keys(pipe)},
336
+ debug=debug,
327
337
  )
328
338
  if debug:
329
339
  dprint(response.text)
@@ -361,7 +371,9 @@ def get_pipe_data(
361
371
  'omit_columns': json.dumps(omit_columns),
362
372
  'begin': begin,
363
373
  'end': end,
364
- 'params': json.dumps(params, default=str)
374
+ 'params': json.dumps(params, default=str),
375
+ 'instance': self.get_pipe_instance_keys(pipe),
376
+ 'as_chunks': as_chunks,
365
377
  },
366
378
  debug=debug
367
379
  )