meerschaum 2.8.4__py3-none-any.whl → 2.9.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/api/_chunks.py +67 -0
- meerschaum/api/dash/callbacks/custom.py +23 -2
- meerschaum/api/dash/callbacks/dashboard.py +41 -3
- meerschaum/api/dash/components.py +27 -19
- meerschaum/api/dash/pages/dashboard.py +11 -9
- meerschaum/api/dash/pages/plugins.py +31 -27
- meerschaum/api/dash/webterm.py +6 -3
- meerschaum/api/resources/static/css/dash.css +1 -1
- meerschaum/api/resources/templates/termpage.html +4 -0
- meerschaum/api/routes/_pipes.py +191 -78
- meerschaum/config/_default.py +3 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_APIConnector.py +12 -1
- meerschaum/connectors/api/_pipes.py +27 -15
- meerschaum/connectors/api/_plugins.py +51 -45
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/parse.py +1 -2
- meerschaum/core/Pipe/_data.py +1 -2
- meerschaum/plugins/_Plugin.py +21 -5
- meerschaum/plugins/__init__.py +6 -4
- meerschaum/utils/formatting/_shell.py +1 -4
- meerschaum/utils/packages/_packages.py +1 -0
- meerschaum/utils/venv/__init__.py +2 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/METADATA +4 -1
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/RECORD +31 -30
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/WHEEL +1 -1
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/LICENSE +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/NOTICE +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/zip-safe +0 -0
meerschaum/api/routes/_pipes.py
CHANGED
@@ -27,8 +27,10 @@ from meerschaum.api import (
|
|
27
27
|
debug,
|
28
28
|
no_auth,
|
29
29
|
)
|
30
|
+
from meerschaum.api._chunks import generate_chunks_cursor_token
|
30
31
|
from meerschaum.utils.packages import attempt_import
|
31
32
|
from meerschaum.utils.dataframe import to_json
|
33
|
+
from meerschaum.utils.dtypes import are_dtypes_equal, json_serialize_value
|
32
34
|
from meerschaum.utils.misc import (
|
33
35
|
is_pipe_registered,
|
34
36
|
is_int,
|
@@ -38,14 +40,16 @@ from meerschaum.connectors.sql.tables import get_tables
|
|
38
40
|
|
39
41
|
fastapi_responses = attempt_import('fastapi.responses', lazy=False)
|
40
42
|
StreamingResponse = fastapi_responses.StreamingResponse
|
41
|
-
dateutil_parser = attempt_import('dateutil.parser', lazy=False)
|
42
43
|
pipes_endpoint = endpoints['pipes']
|
43
44
|
pd = attempt_import('pandas', lazy=False)
|
44
45
|
|
45
46
|
MAX_RESPONSE_ROW_LIMIT: int = mrsm.get_config('system', 'api', 'data', 'max_response_row_limit')
|
46
47
|
|
47
48
|
|
48
|
-
@app.post(
|
49
|
+
@app.post(
|
50
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register',
|
51
|
+
tags=['Pipes: Attributes'],
|
52
|
+
)
|
49
53
|
def register_pipe(
|
50
54
|
connector_keys: str,
|
51
55
|
metric_key: str,
|
@@ -80,18 +84,23 @@ def register_pipe(
|
|
80
84
|
return results
|
81
85
|
|
82
86
|
|
83
|
-
@app.
|
84
|
-
|
87
|
+
@app.patch(
|
88
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit',
|
89
|
+
tags=['Pipes: Attributes'],
|
90
|
+
)
|
91
|
+
def edit_pipe(
|
85
92
|
connector_keys: str,
|
86
93
|
metric_key: str,
|
87
94
|
location_key: str,
|
95
|
+
parameters: dict,
|
88
96
|
instance_keys: Optional[str] = None,
|
97
|
+
patch: bool = False,
|
89
98
|
curr_user = (
|
90
99
|
fastapi.Depends(manager) if not no_auth else None
|
91
100
|
),
|
92
101
|
):
|
93
102
|
"""
|
94
|
-
|
103
|
+
Edit an existing pipe's parameters.
|
95
104
|
"""
|
96
105
|
allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
|
97
106
|
if not allow_actions:
|
@@ -107,54 +116,27 @@ def delete_pipe(
|
|
107
116
|
raise fastapi.HTTPException(
|
108
117
|
status_code=409, detail=f"{pipe} is not registered."
|
109
118
|
)
|
110
|
-
|
111
|
-
|
112
|
-
return results
|
113
|
-
|
114
|
-
|
115
|
-
@app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop', tags=['Pipes'])
|
116
|
-
def drop_pipe(
|
117
|
-
connector_keys: str,
|
118
|
-
metric_key: str,
|
119
|
-
location_key: str,
|
120
|
-
instance_keys: Optional[str] = None,
|
121
|
-
curr_user = (
|
122
|
-
fastapi.Depends(manager) if not no_auth else None
|
123
|
-
),
|
124
|
-
):
|
125
|
-
"""
|
126
|
-
Drop a pipes' underlying target table.
|
127
|
-
"""
|
128
|
-
allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
|
129
|
-
if not allow_actions:
|
130
|
-
return False, (
|
131
|
-
"The administrator for this server has not allowed actions.\n\n"
|
132
|
-
"Please contact the system administrator, or if you are running this server, "
|
133
|
-
"open the configuration file with `edit config system` and search for 'permissions'."
|
134
|
-
" Under the keys `api:permissions:actions`, " +
|
135
|
-
"you can toggle non-admin actions."
|
136
|
-
)
|
137
|
-
pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
138
|
-
results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
|
119
|
+
pipe.parameters = parameters
|
120
|
+
results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
|
139
121
|
pipes(instance_keys, refresh=True)
|
140
122
|
return results
|
141
123
|
|
142
124
|
|
143
|
-
|
144
|
-
|
145
|
-
|
125
|
+
@app.delete(
|
126
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete',
|
127
|
+
tags=['Pipes: Attributes'],
|
128
|
+
)
|
129
|
+
def delete_pipe(
|
146
130
|
connector_keys: str,
|
147
131
|
metric_key: str,
|
148
132
|
location_key: str,
|
149
|
-
parameters: dict,
|
150
133
|
instance_keys: Optional[str] = None,
|
151
|
-
patch: bool = False,
|
152
134
|
curr_user = (
|
153
135
|
fastapi.Depends(manager) if not no_auth else None
|
154
136
|
),
|
155
137
|
):
|
156
138
|
"""
|
157
|
-
|
139
|
+
Delete a Pipe (without dropping its table).
|
158
140
|
"""
|
159
141
|
allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
|
160
142
|
if not allow_actions:
|
@@ -170,13 +152,12 @@ def edit_pipe(
|
|
170
152
|
raise fastapi.HTTPException(
|
171
153
|
status_code=409, detail=f"{pipe} is not registered."
|
172
154
|
)
|
173
|
-
pipe
|
174
|
-
results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
|
155
|
+
results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
|
175
156
|
pipes(instance_keys, refresh=True)
|
176
157
|
return results
|
177
158
|
|
178
159
|
|
179
|
-
@app.get(pipes_endpoint + '/keys', tags=['Pipes'])
|
160
|
+
@app.get(pipes_endpoint + '/keys', tags=['Pipes: Attributes'])
|
180
161
|
async def fetch_pipes_keys(
|
181
162
|
connector_keys: str = "[]",
|
182
163
|
metric_keys: str = "[]",
|
@@ -201,7 +182,7 @@ async def fetch_pipes_keys(
|
|
201
182
|
return keys
|
202
183
|
|
203
184
|
|
204
|
-
@app.get(pipes_endpoint, tags=['Pipes'])
|
185
|
+
@app.get(pipes_endpoint, tags=['Pipes: Attributes'])
|
205
186
|
async def get_pipes(
|
206
187
|
connector_keys: str = "",
|
207
188
|
metric_keys: str = "",
|
@@ -225,7 +206,7 @@ async def get_pipes(
|
|
225
206
|
return replace_pipes_in_dict(_get_pipes(**kw), lambda p: p.attributes)
|
226
207
|
|
227
208
|
|
228
|
-
@app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes'])
|
209
|
+
@app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes: Attributes'])
|
229
210
|
async def get_pipes_by_connector(
|
230
211
|
connector_keys: str,
|
231
212
|
instance_keys: Optional[str] = None,
|
@@ -243,7 +224,7 @@ async def get_pipes_by_connector(
|
|
243
224
|
return replace_pipes_in_dict(pipes(instance_keys)[connector_keys], lambda p: p.attributes)
|
244
225
|
|
245
226
|
|
246
|
-
@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes'])
|
227
|
+
@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes: Attributes'])
|
247
228
|
async def get_pipes_by_connector_and_metric(
|
248
229
|
connector_keys: str,
|
249
230
|
metric_key: str,
|
@@ -271,7 +252,10 @@ async def get_pipes_by_connector_and_metric(
|
|
271
252
|
)
|
272
253
|
|
273
254
|
|
274
|
-
@app.get(
|
255
|
+
@app.get(
|
256
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}',
|
257
|
+
tags=['Pipes: Attributes'],
|
258
|
+
)
|
275
259
|
async def get_pipe_by_connector_and_metric_and_location(
|
276
260
|
connector_keys: str,
|
277
261
|
metric_key: str,
|
@@ -302,7 +286,10 @@ async def get_pipe_by_connector_and_metric_and_location(
|
|
302
286
|
return pipes(instance_keys)[connector_keys][metric_key][location_key].attributes
|
303
287
|
|
304
288
|
|
305
|
-
@app.get(
|
289
|
+
@app.get(
|
290
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time',
|
291
|
+
tags=['Pipes: Data'],
|
292
|
+
)
|
306
293
|
def get_sync_time(
|
307
294
|
connector_keys: str,
|
308
295
|
metric_key: str,
|
@@ -333,7 +320,10 @@ def get_sync_time(
|
|
333
320
|
return sync_time
|
334
321
|
|
335
322
|
|
336
|
-
@app.post(
|
323
|
+
@app.post(
|
324
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
|
325
|
+
tags=['Pipes: Data'],
|
326
|
+
)
|
337
327
|
def sync_pipe(
|
338
328
|
connector_keys: str,
|
339
329
|
metric_key: str,
|
@@ -377,7 +367,10 @@ def sync_pipe(
|
|
377
367
|
return list((success, msg))
|
378
368
|
|
379
369
|
|
380
|
-
@app.get(
|
370
|
+
@app.get(
|
371
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
|
372
|
+
tags=['Pipes: Data'],
|
373
|
+
)
|
381
374
|
def get_pipe_data(
|
382
375
|
connector_keys: str,
|
383
376
|
metric_key: str,
|
@@ -389,6 +382,9 @@ def get_pipe_data(
|
|
389
382
|
end: Union[str, int, None] = None,
|
390
383
|
params: Optional[str] = None,
|
391
384
|
limit: int = MAX_RESPONSE_ROW_LIMIT,
|
385
|
+
order: str = 'asc',
|
386
|
+
as_chunks: bool = False,
|
387
|
+
chunk_interval: Optional[int] = None,
|
392
388
|
curr_user = (
|
393
389
|
fastapi.Depends(manager) if not no_auth else None
|
394
390
|
),
|
@@ -398,11 +394,16 @@ def get_pipe_data(
|
|
398
394
|
See [`Pipe.get_data()`](https://docs.meerschaum.io/meerschaum.html#Pipe.get_data).
|
399
395
|
|
400
396
|
Note that `select_columns`, `omit_columns`, and `params` are JSON-encoded strings.
|
397
|
+
|
398
|
+
Parameters
|
399
|
+
----------
|
400
|
+
instance_keys: Optional[str], default None
|
401
|
+
The connector key to the instance on which the pipe is registered.
|
402
|
+
Defaults to the configured value for `meerschaum:api_instance`.
|
403
|
+
|
404
|
+
as_chunks: bool, default False
|
405
|
+
If `True`, return a chunk token to be consumed by the `/chunks` endpoint.
|
401
406
|
"""
|
402
|
-
if is_int(begin):
|
403
|
-
begin = int(begin)
|
404
|
-
if is_int(end):
|
405
|
-
end = int(end)
|
406
407
|
if limit > MAX_RESPONSE_ROW_LIMIT:
|
407
408
|
raise fastapi.HTTPException(
|
408
409
|
status_code=413,
|
@@ -455,18 +456,38 @@ def get_pipe_data(
|
|
455
456
|
)
|
456
457
|
|
457
458
|
pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
459
|
+
begin, end = pipe.parse_date_bounds(begin, end)
|
458
460
|
if not is_pipe_registered(pipe, pipes(instance_keys, refresh=True)):
|
459
461
|
raise fastapi.HTTPException(
|
460
462
|
status_code=409,
|
461
463
|
detail="Pipe must be registered with the datetime column specified."
|
462
464
|
)
|
463
465
|
|
464
|
-
if pipe.target in ('
|
466
|
+
if pipe.target in ('mrsm_users', 'mrsm_plugins', 'mrsm_pipes'):
|
465
467
|
raise fastapi.HTTPException(
|
466
468
|
status_code=409,
|
467
469
|
detail=f"Cannot retrieve data from protected table '{pipe.target}'.",
|
468
470
|
)
|
469
471
|
|
472
|
+
if as_chunks:
|
473
|
+
chunks_cursor_token = generate_chunks_cursor_token(
|
474
|
+
pipe,
|
475
|
+
select_columns=_select_columns,
|
476
|
+
omit_columns=_omit_columns,
|
477
|
+
begin=begin,
|
478
|
+
end=end,
|
479
|
+
params=_params,
|
480
|
+
limit=limit,
|
481
|
+
order=order,
|
482
|
+
debug=debug,
|
483
|
+
)
|
484
|
+
return fastapi.Response(
|
485
|
+
json.dumps({
|
486
|
+
'chunks_cursor': chunks_cursor,
|
487
|
+
}),
|
488
|
+
media_type='application/json',
|
489
|
+
)
|
490
|
+
|
470
491
|
df = pipe.get_data(
|
471
492
|
select_columns=_select_columns,
|
472
493
|
omit_columns=_omit_columns,
|
@@ -474,6 +495,7 @@ def get_pipe_data(
|
|
474
495
|
end=end,
|
475
496
|
params=_params,
|
476
497
|
limit=min(limit, MAX_RESPONSE_ROW_LIMIT),
|
498
|
+
order=order,
|
477
499
|
debug=debug,
|
478
500
|
)
|
479
501
|
if df is None:
|
@@ -489,7 +511,100 @@ def get_pipe_data(
|
|
489
511
|
)
|
490
512
|
|
491
513
|
|
492
|
-
@app.get(
|
514
|
+
@app.get(
|
515
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk_bounds',
|
516
|
+
tags=['Pipes: Data'],
|
517
|
+
)
|
518
|
+
def get_pipe_chunk_bounds(
|
519
|
+
connector_keys: str,
|
520
|
+
metric_key: str,
|
521
|
+
location_key: str,
|
522
|
+
instance_keys: Optional[str] = None,
|
523
|
+
begin: Union[str, int, None] = None,
|
524
|
+
end: Union[str, int, None] = None,
|
525
|
+
bounded: bool = True,
|
526
|
+
chunk_interval_minutes: Union[int, None] = None,
|
527
|
+
) -> List[List[Union[str, int, None]]]:
|
528
|
+
"""
|
529
|
+
Return a list of request boundaries between `begin` and `end` (or the pipe's sync times).
|
530
|
+
Optionally specify the interval between chunk bounds
|
531
|
+
(defaults to the pipe's configured chunk interval).
|
532
|
+
"""
|
533
|
+
pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
534
|
+
begin, end = pipe.parse_date_bounds(begin, end)
|
535
|
+
dt_col = pipe.columns.get('datetime', None)
|
536
|
+
dt_typ = pipe.dtypes.get(dt_col, 'datetime')
|
537
|
+
chunk_interval = None if chunk_interval_minutes is None else (
|
538
|
+
chunk_interval_minutes
|
539
|
+
if are_dtypes_equal(dt_typ, 'int')
|
540
|
+
else timedelta(minutes=chunk_interval_minutes)
|
541
|
+
)
|
542
|
+
|
543
|
+
chunk_bounds = pipe.get_chunk_bounds(
|
544
|
+
begin=begin,
|
545
|
+
end=end,
|
546
|
+
bounded=bounded,
|
547
|
+
chunk_interval=chunk_interval,
|
548
|
+
debug=debug,
|
549
|
+
)
|
550
|
+
|
551
|
+
return fastapi.Response(
|
552
|
+
json.dumps(chunk_bounds, default=json_serialize_value),
|
553
|
+
media_type='application/json',
|
554
|
+
)
|
555
|
+
|
556
|
+
|
557
|
+
@app.get(
|
558
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk/{chunk_token}',
|
559
|
+
tags=['Pipes: Data'],
|
560
|
+
)
|
561
|
+
def get_pipe_chunk(
|
562
|
+
connector_keys: str,
|
563
|
+
metric_key: str,
|
564
|
+
location_key: str,
|
565
|
+
chunk_token: str
|
566
|
+
) -> Dict[str, Any]:
|
567
|
+
"""
|
568
|
+
Consume a chunk token, returning the dataframe.
|
569
|
+
"""
|
570
|
+
|
571
|
+
|
572
|
+
@app.delete(
|
573
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop',
|
574
|
+
tags=['Pipes: Data'],
|
575
|
+
)
|
576
|
+
def drop_pipe(
|
577
|
+
connector_keys: str,
|
578
|
+
metric_key: str,
|
579
|
+
location_key: str,
|
580
|
+
instance_keys: Optional[str] = None,
|
581
|
+
curr_user = (
|
582
|
+
fastapi.Depends(manager) if not no_auth else None
|
583
|
+
),
|
584
|
+
):
|
585
|
+
"""
|
586
|
+
Drop a pipes' underlying target table.
|
587
|
+
"""
|
588
|
+
allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
|
589
|
+
if not allow_actions:
|
590
|
+
return False, (
|
591
|
+
"The administrator for this server has not allowed actions.\n\n"
|
592
|
+
"Please contact the system administrator, or if you are running this server, "
|
593
|
+
"open the configuration file with `edit config system` and search for 'permissions'."
|
594
|
+
" Under the keys `api:permissions:actions`, " +
|
595
|
+
"you can toggle non-admin actions."
|
596
|
+
)
|
597
|
+
pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
598
|
+
results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
|
599
|
+
pipes(instance_keys, refresh=True)
|
600
|
+
return results
|
601
|
+
|
602
|
+
|
603
|
+
|
604
|
+
@app.get(
|
605
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv',
|
606
|
+
tags=['Pipes: Data'],
|
607
|
+
)
|
493
608
|
def get_pipe_csv(
|
494
609
|
connector_keys: str,
|
495
610
|
metric_key: str,
|
@@ -505,18 +620,6 @@ def get_pipe_csv(
|
|
505
620
|
"""
|
506
621
|
Get a pipe's data as a CSV file. Optionally set query boundaries.
|
507
622
|
"""
|
508
|
-
if begin is not None:
|
509
|
-
begin = (
|
510
|
-
int(begin)
|
511
|
-
if is_int(begin)
|
512
|
-
else dateutil_parser.parse(begin)
|
513
|
-
)
|
514
|
-
if end is not None:
|
515
|
-
end = (
|
516
|
-
int(end)
|
517
|
-
if is_int(end)
|
518
|
-
else dateutil_parser.parse(end)
|
519
|
-
)
|
520
623
|
|
521
624
|
_params = {}
|
522
625
|
if params == 'null':
|
@@ -540,6 +643,7 @@ def get_pipe_csv(
|
|
540
643
|
detail="Pipe must be registered."
|
541
644
|
)
|
542
645
|
|
646
|
+
begin, end = pipe.parse_date_bounds(begin, end)
|
543
647
|
dt_col = pipe.columns.get('datetime', None)
|
544
648
|
if dt_col:
|
545
649
|
if begin is None:
|
@@ -567,7 +671,10 @@ def get_pipe_csv(
|
|
567
671
|
return response
|
568
672
|
|
569
673
|
|
570
|
-
@app.get(
|
674
|
+
@app.get(
|
675
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id',
|
676
|
+
tags=['Pipes: Attributes'],
|
677
|
+
)
|
571
678
|
def get_pipe_id(
|
572
679
|
connector_keys: str,
|
573
680
|
metric_key: str,
|
@@ -588,7 +695,7 @@ def get_pipe_id(
|
|
588
695
|
|
589
696
|
@app.get(
|
590
697
|
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/attributes',
|
591
|
-
tags=['Pipes']
|
698
|
+
tags=['Pipes: Attributes'],
|
592
699
|
)
|
593
700
|
def get_pipe_attributes(
|
594
701
|
connector_keys: str,
|
@@ -609,7 +716,10 @@ def get_pipe_attributes(
|
|
609
716
|
).attributes
|
610
717
|
|
611
718
|
|
612
|
-
@app.get(
|
719
|
+
@app.get(
|
720
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists',
|
721
|
+
tags=['Pipes: Data'],
|
722
|
+
)
|
613
723
|
def get_pipe_exists(
|
614
724
|
connector_keys: str,
|
615
725
|
metric_key: str,
|
@@ -623,7 +733,7 @@ def get_pipe_exists(
|
|
623
733
|
return get_pipe(connector_keys, metric_key, location_key, instance_keys).exists(debug=debug)
|
624
734
|
|
625
735
|
|
626
|
-
@app.post(endpoints['metadata'], tags=['
|
736
|
+
@app.post(endpoints['metadata'], tags=['Misc'])
|
627
737
|
def create_metadata(
|
628
738
|
instance_keys: Optional[str] = None,
|
629
739
|
curr_user = (
|
@@ -641,7 +751,10 @@ def create_metadata(
|
|
641
751
|
return True
|
642
752
|
|
643
753
|
|
644
|
-
@app.get(
|
754
|
+
@app.get(
|
755
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount',
|
756
|
+
tags=['Pipes: Data'],
|
757
|
+
)
|
645
758
|
def get_pipe_rowcount(
|
646
759
|
connector_keys: str,
|
647
760
|
metric_key: str,
|
@@ -692,7 +805,7 @@ def get_pipe_rowcount(
|
|
692
805
|
|
693
806
|
@app.get(
|
694
807
|
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/types',
|
695
|
-
tags=['Pipes']
|
808
|
+
tags=['Pipes: Data'],
|
696
809
|
)
|
697
810
|
def get_pipe_columns_types(
|
698
811
|
connector_keys: str,
|
@@ -720,7 +833,7 @@ def get_pipe_columns_types(
|
|
720
833
|
|
721
834
|
@app.get(
|
722
835
|
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/indices',
|
723
|
-
tags=['Pipes']
|
836
|
+
tags=['Pipes: Data'],
|
724
837
|
)
|
725
838
|
def get_pipe_columns_indices(
|
726
839
|
connector_keys: str,
|
@@ -774,7 +887,7 @@ def get_pipe_columns_indices(
|
|
774
887
|
|
775
888
|
@app.get(
|
776
889
|
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/indices/names',
|
777
|
-
tags=['Pipes']
|
890
|
+
tags=['Pipes: Data']
|
778
891
|
)
|
779
892
|
def get_pipe_index_names(
|
780
893
|
connector_keys: str,
|
@@ -784,7 +897,7 @@ def get_pipe_index_names(
|
|
784
897
|
curr_user=(
|
785
898
|
fastapi.Depends(manager) if not no_auth else None
|
786
899
|
),
|
787
|
-
) -> Dict[str, List[Dict[str, str]]]:
|
900
|
+
) -> Dict[str, Union[str, Dict[str, str], List[Dict[str, str]]]]:
|
788
901
|
"""
|
789
902
|
Return a dictionary of index keys and index names.
|
790
903
|
|
@@ -795,4 +908,4 @@ def get_pipe_index_names(
|
|
795
908
|
metric_key,
|
796
909
|
location_key,
|
797
910
|
instance_keys,
|
798
|
-
).get_indices(
|
911
|
+
).get_indices()
|
meerschaum/config/_default.py
CHANGED
meerschaum/config/_version.py
CHANGED
@@ -6,8 +6,10 @@
|
|
6
6
|
Interact with Meerschaum APIs. May be chained together (see 'meerschaum:api_instance' in your config.yaml).
|
7
7
|
"""
|
8
8
|
|
9
|
+
from __future__ import annotations
|
10
|
+
|
9
11
|
from datetime import datetime, timedelta, timezone
|
10
|
-
from meerschaum.utils.typing import Optional, List
|
12
|
+
from meerschaum.utils.typing import Optional, List, Union
|
11
13
|
from meerschaum.connectors import Connector
|
12
14
|
from meerschaum.utils.warnings import warn, error
|
13
15
|
from meerschaum.utils.packages import attempt_import
|
@@ -43,6 +45,7 @@ class APIConnector(Connector):
|
|
43
45
|
)
|
44
46
|
from ._misc import get_mrsm_version, get_chaining_status
|
45
47
|
from ._pipes import (
|
48
|
+
get_pipe_instance_keys,
|
46
49
|
register_pipe,
|
47
50
|
fetch_pipes_keys,
|
48
51
|
edit_pipe,
|
@@ -143,6 +146,7 @@ class APIConnector(Connector):
|
|
143
146
|
self._token = None
|
144
147
|
self._expires = None
|
145
148
|
self._session = None
|
149
|
+
self._instance_keys = self.__dict__.get('instance_keys', None)
|
146
150
|
|
147
151
|
|
148
152
|
@property
|
@@ -195,3 +199,10 @@ class APIConnector(Connector):
|
|
195
199
|
warn(msg, stack=False)
|
196
200
|
self._emitted_warning = True
|
197
201
|
return self._token
|
202
|
+
|
203
|
+
@property
|
204
|
+
def instance_keys(self) -> Union[str, None]:
|
205
|
+
"""
|
206
|
+
Return the instance keys to be sent alongside pipe requests.
|
207
|
+
"""
|
208
|
+
return self._instance_keys
|
@@ -31,6 +31,14 @@ def pipe_r_url(
|
|
31
31
|
)
|
32
32
|
|
33
33
|
|
34
|
+
def get_pipe_instance_keys(self, pipe: mrsm.Pipe) -> Union[str, None]:
|
35
|
+
"""
|
36
|
+
Return the configured instance keys for a pipe if set,
|
37
|
+
else fall back to the default `instance_keys` for this `APIConnector`.
|
38
|
+
"""
|
39
|
+
return pipe.parameters.get('instance_keys', self.instance_keys)
|
40
|
+
|
41
|
+
|
34
42
|
def register_pipe(
|
35
43
|
self,
|
36
44
|
pipe: mrsm.Pipe,
|
@@ -40,13 +48,12 @@ def register_pipe(
|
|
40
48
|
Returns a tuple of (success_bool, response_dict).
|
41
49
|
"""
|
42
50
|
from meerschaum.utils.debug import dprint
|
43
|
-
### NOTE: if `parameters` is supplied in the Pipe constructor,
|
44
|
-
### then `pipe.parameters` will exist and not be fetched from the database.
|
45
51
|
r_url = pipe_r_url(pipe)
|
46
52
|
response = self.post(
|
47
53
|
r_url + '/register',
|
48
|
-
json
|
49
|
-
|
54
|
+
json=pipe._attributes.get('parameters', {}),
|
55
|
+
params={'instance_keys': self.get_pipe_instance_keys(pipe)},
|
56
|
+
debug=debug,
|
50
57
|
)
|
51
58
|
if debug:
|
52
59
|
dprint(response.text)
|
@@ -79,9 +86,9 @@ def edit_pipe(
|
|
79
86
|
r_url = pipe_r_url(pipe)
|
80
87
|
response = self.patch(
|
81
88
|
r_url + '/edit',
|
82
|
-
params
|
83
|
-
json
|
84
|
-
debug
|
89
|
+
params={'patch': patch, 'instance_keys': self.get_pipe_instance_keys(pipe)},
|
90
|
+
json=pipe.parameters,
|
91
|
+
debug=debug,
|
85
92
|
)
|
86
93
|
if debug:
|
87
94
|
dprint(response.text)
|
@@ -149,12 +156,13 @@ def fetch_pipes_keys(
|
|
149
156
|
try:
|
150
157
|
j = self.get(
|
151
158
|
r_url,
|
152
|
-
params
|
159
|
+
params={
|
153
160
|
'connector_keys': json.dumps(connector_keys),
|
154
161
|
'metric_keys': json.dumps(metric_keys),
|
155
162
|
'location_keys': json.dumps(location_keys),
|
156
163
|
'tags': json.dumps(tags),
|
157
164
|
'params': json.dumps(params),
|
165
|
+
'instance_keys': self.instance_keys,
|
158
166
|
},
|
159
167
|
debug=debug
|
160
168
|
).json()
|
@@ -250,8 +258,10 @@ def sync_pipe(
|
|
250
258
|
chunks = (df[i] for i in more_itertools.chunked(df, _chunksize))
|
251
259
|
|
252
260
|
### Send columns in case the user has defined them locally.
|
261
|
+
request_params = kw.copy()
|
253
262
|
if pipe.columns:
|
254
|
-
|
263
|
+
request_params['columns'] = json.dumps(pipe.columns)
|
264
|
+
request_params['instance_keys'] = self.get_pipe_instance_keys(pipe)
|
255
265
|
r_url = pipe_r_url(pipe) + '/data'
|
256
266
|
|
257
267
|
rowcount = 0
|
@@ -268,10 +278,9 @@ def sync_pipe(
|
|
268
278
|
try:
|
269
279
|
response = self.post(
|
270
280
|
r_url,
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
debug = debug
|
281
|
+
params=request_params,
|
282
|
+
data=json_str,
|
283
|
+
debug=debug,
|
275
284
|
)
|
276
285
|
except Exception as e:
|
277
286
|
msg = f"Failed to post a chunk to {pipe}:\n{e}"
|
@@ -323,7 +332,8 @@ def delete_pipe(
|
|
323
332
|
r_url = pipe_r_url(pipe)
|
324
333
|
response = self.delete(
|
325
334
|
r_url + '/delete',
|
326
|
-
|
335
|
+
params={'instance_keys': self.get_pipe_instance_keys(pipe)},
|
336
|
+
debug=debug,
|
327
337
|
)
|
328
338
|
if debug:
|
329
339
|
dprint(response.text)
|
@@ -361,7 +371,9 @@ def get_pipe_data(
|
|
361
371
|
'omit_columns': json.dumps(omit_columns),
|
362
372
|
'begin': begin,
|
363
373
|
'end': end,
|
364
|
-
'params': json.dumps(params, default=str)
|
374
|
+
'params': json.dumps(params, default=str),
|
375
|
+
'instance': self.get_pipe_instance_keys(pipe),
|
376
|
+
'as_chunks': as_chunks,
|
365
377
|
},
|
366
378
|
debug=debug
|
367
379
|
)
|