meerschaum 2.8.3__py3-none-any.whl → 2.9.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parser.py +5 -0
- meerschaum/actions/drop.py +1 -1
- meerschaum/actions/start.py +14 -6
- meerschaum/actions/sync.py +9 -0
- meerschaum/api/__init__.py +9 -3
- meerschaum/api/_chunks.py +67 -0
- meerschaum/api/dash/callbacks/custom.py +23 -2
- meerschaum/api/dash/callbacks/dashboard.py +41 -3
- meerschaum/api/dash/components.py +27 -19
- meerschaum/api/dash/pages/dashboard.py +11 -9
- meerschaum/api/dash/pages/plugins.py +31 -27
- meerschaum/api/dash/webterm.py +6 -3
- meerschaum/api/resources/static/css/dash.css +1 -1
- meerschaum/api/resources/templates/termpage.html +4 -0
- meerschaum/api/routes/_pipes.py +193 -81
- meerschaum/config/_default.py +3 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_APIConnector.py +12 -1
- meerschaum/connectors/api/_pipes.py +27 -15
- meerschaum/connectors/api/_plugins.py +51 -45
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/parse.py +1 -2
- meerschaum/connectors/sql/_SQLConnector.py +1 -1
- meerschaum/core/Pipe/_data.py +1 -2
- meerschaum/core/Pipe/_verify.py +23 -8
- meerschaum/jobs/systemd.py +1 -1
- meerschaum/plugins/_Plugin.py +21 -5
- meerschaum/plugins/__init__.py +6 -4
- meerschaum/utils/formatting/_shell.py +1 -4
- meerschaum/utils/packages/_packages.py +2 -1
- meerschaum/utils/process.py +27 -3
- meerschaum/utils/schedule.py +3 -3
- meerschaum/utils/sql.py +1 -1
- meerschaum/utils/venv/__init__.py +6 -1
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/METADATA +4 -1
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/RECORD +42 -41
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/WHEEL +1 -1
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/LICENSE +0 -0
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/NOTICE +0 -0
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/zip-safe +0 -0
meerschaum/api/routes/_pipes.py
CHANGED
@@ -27,8 +27,10 @@ from meerschaum.api import (
|
|
27
27
|
debug,
|
28
28
|
no_auth,
|
29
29
|
)
|
30
|
+
from meerschaum.api._chunks import generate_chunks_cursor_token
|
30
31
|
from meerschaum.utils.packages import attempt_import
|
31
32
|
from meerschaum.utils.dataframe import to_json
|
33
|
+
from meerschaum.utils.dtypes import are_dtypes_equal, json_serialize_value
|
32
34
|
from meerschaum.utils.misc import (
|
33
35
|
is_pipe_registered,
|
34
36
|
is_int,
|
@@ -38,14 +40,16 @@ from meerschaum.connectors.sql.tables import get_tables
|
|
38
40
|
|
39
41
|
fastapi_responses = attempt_import('fastapi.responses', lazy=False)
|
40
42
|
StreamingResponse = fastapi_responses.StreamingResponse
|
41
|
-
dateutil_parser = attempt_import('dateutil.parser', lazy=False)
|
42
43
|
pipes_endpoint = endpoints['pipes']
|
43
44
|
pd = attempt_import('pandas', lazy=False)
|
44
45
|
|
45
46
|
MAX_RESPONSE_ROW_LIMIT: int = mrsm.get_config('system', 'api', 'data', 'max_response_row_limit')
|
46
47
|
|
47
48
|
|
48
|
-
@app.post(
|
49
|
+
@app.post(
|
50
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register',
|
51
|
+
tags=['Pipes: Attributes'],
|
52
|
+
)
|
49
53
|
def register_pipe(
|
50
54
|
connector_keys: str,
|
51
55
|
metric_key: str,
|
@@ -80,18 +84,23 @@ def register_pipe(
|
|
80
84
|
return results
|
81
85
|
|
82
86
|
|
83
|
-
@app.
|
84
|
-
|
87
|
+
@app.patch(
|
88
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit',
|
89
|
+
tags=['Pipes: Attributes'],
|
90
|
+
)
|
91
|
+
def edit_pipe(
|
85
92
|
connector_keys: str,
|
86
93
|
metric_key: str,
|
87
94
|
location_key: str,
|
95
|
+
parameters: dict,
|
88
96
|
instance_keys: Optional[str] = None,
|
97
|
+
patch: bool = False,
|
89
98
|
curr_user = (
|
90
99
|
fastapi.Depends(manager) if not no_auth else None
|
91
100
|
),
|
92
101
|
):
|
93
102
|
"""
|
94
|
-
|
103
|
+
Edit an existing pipe's parameters.
|
95
104
|
"""
|
96
105
|
allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
|
97
106
|
if not allow_actions:
|
@@ -107,54 +116,27 @@ def delete_pipe(
|
|
107
116
|
raise fastapi.HTTPException(
|
108
117
|
status_code=409, detail=f"{pipe} is not registered."
|
109
118
|
)
|
110
|
-
|
111
|
-
|
112
|
-
return results
|
113
|
-
|
114
|
-
|
115
|
-
@app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop', tags=['Pipes'])
|
116
|
-
def drop_pipe(
|
117
|
-
connector_keys: str,
|
118
|
-
metric_key: str,
|
119
|
-
location_key: str,
|
120
|
-
instance_keys: Optional[str] = None,
|
121
|
-
curr_user = (
|
122
|
-
fastapi.Depends(manager) if not no_auth else None
|
123
|
-
),
|
124
|
-
):
|
125
|
-
"""
|
126
|
-
Drop a pipes' underlying target table.
|
127
|
-
"""
|
128
|
-
allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
|
129
|
-
if not allow_actions:
|
130
|
-
return False, (
|
131
|
-
"The administrator for this server has not allowed actions.\n\n"
|
132
|
-
"Please contact the system administrator, or if you are running this server, "
|
133
|
-
"open the configuration file with `edit config system` and search for 'permissions'."
|
134
|
-
" Under the keys `api:permissions:actions`, " +
|
135
|
-
"you can toggle non-admin actions."
|
136
|
-
)
|
137
|
-
pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
138
|
-
results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
|
119
|
+
pipe.parameters = parameters
|
120
|
+
results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
|
139
121
|
pipes(instance_keys, refresh=True)
|
140
122
|
return results
|
141
123
|
|
142
124
|
|
143
|
-
|
144
|
-
|
145
|
-
|
125
|
+
@app.delete(
|
126
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete',
|
127
|
+
tags=['Pipes: Attributes'],
|
128
|
+
)
|
129
|
+
def delete_pipe(
|
146
130
|
connector_keys: str,
|
147
131
|
metric_key: str,
|
148
132
|
location_key: str,
|
149
|
-
parameters: dict,
|
150
133
|
instance_keys: Optional[str] = None,
|
151
|
-
patch: bool = False,
|
152
134
|
curr_user = (
|
153
135
|
fastapi.Depends(manager) if not no_auth else None
|
154
136
|
),
|
155
137
|
):
|
156
138
|
"""
|
157
|
-
|
139
|
+
Delete a Pipe (without dropping its table).
|
158
140
|
"""
|
159
141
|
allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
|
160
142
|
if not allow_actions:
|
@@ -170,13 +152,12 @@ def edit_pipe(
|
|
170
152
|
raise fastapi.HTTPException(
|
171
153
|
status_code=409, detail=f"{pipe} is not registered."
|
172
154
|
)
|
173
|
-
pipe
|
174
|
-
results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
|
155
|
+
results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
|
175
156
|
pipes(instance_keys, refresh=True)
|
176
157
|
return results
|
177
158
|
|
178
159
|
|
179
|
-
@app.get(pipes_endpoint + '/keys', tags=['Pipes'])
|
160
|
+
@app.get(pipes_endpoint + '/keys', tags=['Pipes: Attributes'])
|
180
161
|
async def fetch_pipes_keys(
|
181
162
|
connector_keys: str = "[]",
|
182
163
|
metric_keys: str = "[]",
|
@@ -201,7 +182,7 @@ async def fetch_pipes_keys(
|
|
201
182
|
return keys
|
202
183
|
|
203
184
|
|
204
|
-
@app.get(pipes_endpoint, tags=['Pipes'])
|
185
|
+
@app.get(pipes_endpoint, tags=['Pipes: Attributes'])
|
205
186
|
async def get_pipes(
|
206
187
|
connector_keys: str = "",
|
207
188
|
metric_keys: str = "",
|
@@ -225,7 +206,7 @@ async def get_pipes(
|
|
225
206
|
return replace_pipes_in_dict(_get_pipes(**kw), lambda p: p.attributes)
|
226
207
|
|
227
208
|
|
228
|
-
@app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes'])
|
209
|
+
@app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes: Attributes'])
|
229
210
|
async def get_pipes_by_connector(
|
230
211
|
connector_keys: str,
|
231
212
|
instance_keys: Optional[str] = None,
|
@@ -243,7 +224,7 @@ async def get_pipes_by_connector(
|
|
243
224
|
return replace_pipes_in_dict(pipes(instance_keys)[connector_keys], lambda p: p.attributes)
|
244
225
|
|
245
226
|
|
246
|
-
@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes'])
|
227
|
+
@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes: Attributes'])
|
247
228
|
async def get_pipes_by_connector_and_metric(
|
248
229
|
connector_keys: str,
|
249
230
|
metric_key: str,
|
@@ -271,8 +252,11 @@ async def get_pipes_by_connector_and_metric(
|
|
271
252
|
)
|
272
253
|
|
273
254
|
|
274
|
-
@app.get(
|
275
|
-
|
255
|
+
@app.get(
|
256
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}',
|
257
|
+
tags=['Pipes: Attributes'],
|
258
|
+
)
|
259
|
+
async def get_pipe_by_connector_and_metric_and_location(
|
276
260
|
connector_keys: str,
|
277
261
|
metric_key: str,
|
278
262
|
location_key: str,
|
@@ -299,10 +283,13 @@ async def get_pipes_by_connector_and_metric_and_location(
|
|
299
283
|
detail=f"location_key '{location_key}' not found."
|
300
284
|
)
|
301
285
|
|
302
|
-
return
|
286
|
+
return pipes(instance_keys)[connector_keys][metric_key][location_key].attributes
|
303
287
|
|
304
288
|
|
305
|
-
@app.get(
|
289
|
+
@app.get(
|
290
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time',
|
291
|
+
tags=['Pipes: Data'],
|
292
|
+
)
|
306
293
|
def get_sync_time(
|
307
294
|
connector_keys: str,
|
308
295
|
metric_key: str,
|
@@ -333,7 +320,10 @@ def get_sync_time(
|
|
333
320
|
return sync_time
|
334
321
|
|
335
322
|
|
336
|
-
@app.post(
|
323
|
+
@app.post(
|
324
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
|
325
|
+
tags=['Pipes: Data'],
|
326
|
+
)
|
337
327
|
def sync_pipe(
|
338
328
|
connector_keys: str,
|
339
329
|
metric_key: str,
|
@@ -377,7 +367,10 @@ def sync_pipe(
|
|
377
367
|
return list((success, msg))
|
378
368
|
|
379
369
|
|
380
|
-
@app.get(
|
370
|
+
@app.get(
|
371
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
|
372
|
+
tags=['Pipes: Data'],
|
373
|
+
)
|
381
374
|
def get_pipe_data(
|
382
375
|
connector_keys: str,
|
383
376
|
metric_key: str,
|
@@ -389,6 +382,9 @@ def get_pipe_data(
|
|
389
382
|
end: Union[str, int, None] = None,
|
390
383
|
params: Optional[str] = None,
|
391
384
|
limit: int = MAX_RESPONSE_ROW_LIMIT,
|
385
|
+
order: str = 'asc',
|
386
|
+
as_chunks: bool = False,
|
387
|
+
chunk_interval: Optional[int] = None,
|
392
388
|
curr_user = (
|
393
389
|
fastapi.Depends(manager) if not no_auth else None
|
394
390
|
),
|
@@ -398,11 +394,16 @@ def get_pipe_data(
|
|
398
394
|
See [`Pipe.get_data()`](https://docs.meerschaum.io/meerschaum.html#Pipe.get_data).
|
399
395
|
|
400
396
|
Note that `select_columns`, `omit_columns`, and `params` are JSON-encoded strings.
|
397
|
+
|
398
|
+
Parameters
|
399
|
+
----------
|
400
|
+
instance_keys: Optional[str], default None
|
401
|
+
The connector key to the instance on which the pipe is registered.
|
402
|
+
Defaults to the configured value for `meerschaum:api_instance`.
|
403
|
+
|
404
|
+
as_chunks: bool, default False
|
405
|
+
If `True`, return a chunk token to be consumed by the `/chunks` endpoint.
|
401
406
|
"""
|
402
|
-
if is_int(begin):
|
403
|
-
begin = int(begin)
|
404
|
-
if is_int(end):
|
405
|
-
end = int(end)
|
406
407
|
if limit > MAX_RESPONSE_ROW_LIMIT:
|
407
408
|
raise fastapi.HTTPException(
|
408
409
|
status_code=413,
|
@@ -455,18 +456,38 @@ def get_pipe_data(
|
|
455
456
|
)
|
456
457
|
|
457
458
|
pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
459
|
+
begin, end = pipe.parse_date_bounds(begin, end)
|
458
460
|
if not is_pipe_registered(pipe, pipes(instance_keys, refresh=True)):
|
459
461
|
raise fastapi.HTTPException(
|
460
462
|
status_code=409,
|
461
463
|
detail="Pipe must be registered with the datetime column specified."
|
462
464
|
)
|
463
465
|
|
464
|
-
if pipe.target in ('
|
466
|
+
if pipe.target in ('mrsm_users', 'mrsm_plugins', 'mrsm_pipes'):
|
465
467
|
raise fastapi.HTTPException(
|
466
468
|
status_code=409,
|
467
469
|
detail=f"Cannot retrieve data from protected table '{pipe.target}'.",
|
468
470
|
)
|
469
471
|
|
472
|
+
if as_chunks:
|
473
|
+
chunks_cursor_token = generate_chunks_cursor_token(
|
474
|
+
pipe,
|
475
|
+
select_columns=_select_columns,
|
476
|
+
omit_columns=_omit_columns,
|
477
|
+
begin=begin,
|
478
|
+
end=end,
|
479
|
+
params=_params,
|
480
|
+
limit=limit,
|
481
|
+
order=order,
|
482
|
+
debug=debug,
|
483
|
+
)
|
484
|
+
return fastapi.Response(
|
485
|
+
json.dumps({
|
486
|
+
'chunks_cursor': chunks_cursor,
|
487
|
+
}),
|
488
|
+
media_type='application/json',
|
489
|
+
)
|
490
|
+
|
470
491
|
df = pipe.get_data(
|
471
492
|
select_columns=_select_columns,
|
472
493
|
omit_columns=_omit_columns,
|
@@ -474,6 +495,7 @@ def get_pipe_data(
|
|
474
495
|
end=end,
|
475
496
|
params=_params,
|
476
497
|
limit=min(limit, MAX_RESPONSE_ROW_LIMIT),
|
498
|
+
order=order,
|
477
499
|
debug=debug,
|
478
500
|
)
|
479
501
|
if df is None:
|
@@ -489,7 +511,100 @@ def get_pipe_data(
|
|
489
511
|
)
|
490
512
|
|
491
513
|
|
492
|
-
@app.get(
|
514
|
+
@app.get(
|
515
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk_bounds',
|
516
|
+
tags=['Pipes: Data'],
|
517
|
+
)
|
518
|
+
def get_pipe_chunk_bounds(
|
519
|
+
connector_keys: str,
|
520
|
+
metric_key: str,
|
521
|
+
location_key: str,
|
522
|
+
instance_keys: Optional[str] = None,
|
523
|
+
begin: Union[str, int, None] = None,
|
524
|
+
end: Union[str, int, None] = None,
|
525
|
+
bounded: bool = True,
|
526
|
+
chunk_interval_minutes: Union[int, None] = None,
|
527
|
+
) -> List[List[Union[str, int, None]]]:
|
528
|
+
"""
|
529
|
+
Return a list of request boundaries between `begin` and `end` (or the pipe's sync times).
|
530
|
+
Optionally specify the interval between chunk bounds
|
531
|
+
(defaults to the pipe's configured chunk interval).
|
532
|
+
"""
|
533
|
+
pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
534
|
+
begin, end = pipe.parse_date_bounds(begin, end)
|
535
|
+
dt_col = pipe.columns.get('datetime', None)
|
536
|
+
dt_typ = pipe.dtypes.get(dt_col, 'datetime')
|
537
|
+
chunk_interval = None if chunk_interval_minutes is None else (
|
538
|
+
chunk_interval_minutes
|
539
|
+
if are_dtypes_equal(dt_typ, 'int')
|
540
|
+
else timedelta(minutes=chunk_interval_minutes)
|
541
|
+
)
|
542
|
+
|
543
|
+
chunk_bounds = pipe.get_chunk_bounds(
|
544
|
+
begin=begin,
|
545
|
+
end=end,
|
546
|
+
bounded=bounded,
|
547
|
+
chunk_interval=chunk_interval,
|
548
|
+
debug=debug,
|
549
|
+
)
|
550
|
+
|
551
|
+
return fastapi.Response(
|
552
|
+
json.dumps(chunk_bounds, default=json_serialize_value),
|
553
|
+
media_type='application/json',
|
554
|
+
)
|
555
|
+
|
556
|
+
|
557
|
+
@app.get(
|
558
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk/{chunk_token}',
|
559
|
+
tags=['Pipes: Data'],
|
560
|
+
)
|
561
|
+
def get_pipe_chunk(
|
562
|
+
connector_keys: str,
|
563
|
+
metric_key: str,
|
564
|
+
location_key: str,
|
565
|
+
chunk_token: str
|
566
|
+
) -> Dict[str, Any]:
|
567
|
+
"""
|
568
|
+
Consume a chunk token, returning the dataframe.
|
569
|
+
"""
|
570
|
+
|
571
|
+
|
572
|
+
@app.delete(
|
573
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop',
|
574
|
+
tags=['Pipes: Data'],
|
575
|
+
)
|
576
|
+
def drop_pipe(
|
577
|
+
connector_keys: str,
|
578
|
+
metric_key: str,
|
579
|
+
location_key: str,
|
580
|
+
instance_keys: Optional[str] = None,
|
581
|
+
curr_user = (
|
582
|
+
fastapi.Depends(manager) if not no_auth else None
|
583
|
+
),
|
584
|
+
):
|
585
|
+
"""
|
586
|
+
Drop a pipes' underlying target table.
|
587
|
+
"""
|
588
|
+
allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
|
589
|
+
if not allow_actions:
|
590
|
+
return False, (
|
591
|
+
"The administrator for this server has not allowed actions.\n\n"
|
592
|
+
"Please contact the system administrator, or if you are running this server, "
|
593
|
+
"open the configuration file with `edit config system` and search for 'permissions'."
|
594
|
+
" Under the keys `api:permissions:actions`, " +
|
595
|
+
"you can toggle non-admin actions."
|
596
|
+
)
|
597
|
+
pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
598
|
+
results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
|
599
|
+
pipes(instance_keys, refresh=True)
|
600
|
+
return results
|
601
|
+
|
602
|
+
|
603
|
+
|
604
|
+
@app.get(
|
605
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv',
|
606
|
+
tags=['Pipes: Data'],
|
607
|
+
)
|
493
608
|
def get_pipe_csv(
|
494
609
|
connector_keys: str,
|
495
610
|
metric_key: str,
|
@@ -505,24 +620,11 @@ def get_pipe_csv(
|
|
505
620
|
"""
|
506
621
|
Get a pipe's data as a CSV file. Optionally set query boundaries.
|
507
622
|
"""
|
508
|
-
if begin is not None:
|
509
|
-
begin = (
|
510
|
-
int(begin)
|
511
|
-
if is_int(begin)
|
512
|
-
else dateutil_parser.parse(begin)
|
513
|
-
)
|
514
|
-
if end is not None:
|
515
|
-
end = (
|
516
|
-
int(end)
|
517
|
-
if is_int(end)
|
518
|
-
else dateutil_parser.parse(end)
|
519
|
-
)
|
520
623
|
|
521
624
|
_params = {}
|
522
625
|
if params == 'null':
|
523
626
|
params = None
|
524
627
|
if params is not None:
|
525
|
-
import json
|
526
628
|
try:
|
527
629
|
_params = json.loads(params)
|
528
630
|
except Exception:
|
@@ -541,6 +643,7 @@ def get_pipe_csv(
|
|
541
643
|
detail="Pipe must be registered."
|
542
644
|
)
|
543
645
|
|
646
|
+
begin, end = pipe.parse_date_bounds(begin, end)
|
544
647
|
dt_col = pipe.columns.get('datetime', None)
|
545
648
|
if dt_col:
|
546
649
|
if begin is None:
|
@@ -568,7 +671,10 @@ def get_pipe_csv(
|
|
568
671
|
return response
|
569
672
|
|
570
673
|
|
571
|
-
@app.get(
|
674
|
+
@app.get(
|
675
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id',
|
676
|
+
tags=['Pipes: Attributes'],
|
677
|
+
)
|
572
678
|
def get_pipe_id(
|
573
679
|
connector_keys: str,
|
574
680
|
metric_key: str,
|
@@ -589,7 +695,7 @@ def get_pipe_id(
|
|
589
695
|
|
590
696
|
@app.get(
|
591
697
|
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/attributes',
|
592
|
-
tags=['Pipes']
|
698
|
+
tags=['Pipes: Attributes'],
|
593
699
|
)
|
594
700
|
def get_pipe_attributes(
|
595
701
|
connector_keys: str,
|
@@ -610,7 +716,10 @@ def get_pipe_attributes(
|
|
610
716
|
).attributes
|
611
717
|
|
612
718
|
|
613
|
-
@app.get(
|
719
|
+
@app.get(
|
720
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists',
|
721
|
+
tags=['Pipes: Data'],
|
722
|
+
)
|
614
723
|
def get_pipe_exists(
|
615
724
|
connector_keys: str,
|
616
725
|
metric_key: str,
|
@@ -624,7 +733,7 @@ def get_pipe_exists(
|
|
624
733
|
return get_pipe(connector_keys, metric_key, location_key, instance_keys).exists(debug=debug)
|
625
734
|
|
626
735
|
|
627
|
-
@app.post(endpoints['metadata'], tags=['
|
736
|
+
@app.post(endpoints['metadata'], tags=['Misc'])
|
628
737
|
def create_metadata(
|
629
738
|
instance_keys: Optional[str] = None,
|
630
739
|
curr_user = (
|
@@ -642,7 +751,10 @@ def create_metadata(
|
|
642
751
|
return True
|
643
752
|
|
644
753
|
|
645
|
-
@app.get(
|
754
|
+
@app.get(
|
755
|
+
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount',
|
756
|
+
tags=['Pipes: Data'],
|
757
|
+
)
|
646
758
|
def get_pipe_rowcount(
|
647
759
|
connector_keys: str,
|
648
760
|
metric_key: str,
|
@@ -693,7 +805,7 @@ def get_pipe_rowcount(
|
|
693
805
|
|
694
806
|
@app.get(
|
695
807
|
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/types',
|
696
|
-
tags=['Pipes']
|
808
|
+
tags=['Pipes: Data'],
|
697
809
|
)
|
698
810
|
def get_pipe_columns_types(
|
699
811
|
connector_keys: str,
|
@@ -721,7 +833,7 @@ def get_pipe_columns_types(
|
|
721
833
|
|
722
834
|
@app.get(
|
723
835
|
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/indices',
|
724
|
-
tags=['Pipes']
|
836
|
+
tags=['Pipes: Data'],
|
725
837
|
)
|
726
838
|
def get_pipe_columns_indices(
|
727
839
|
connector_keys: str,
|
@@ -775,7 +887,7 @@ def get_pipe_columns_indices(
|
|
775
887
|
|
776
888
|
@app.get(
|
777
889
|
pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/indices/names',
|
778
|
-
tags=['Pipes']
|
890
|
+
tags=['Pipes: Data']
|
779
891
|
)
|
780
892
|
def get_pipe_index_names(
|
781
893
|
connector_keys: str,
|
@@ -785,7 +897,7 @@ def get_pipe_index_names(
|
|
785
897
|
curr_user=(
|
786
898
|
fastapi.Depends(manager) if not no_auth else None
|
787
899
|
),
|
788
|
-
) -> Dict[str, List[Dict[str, str]]]:
|
900
|
+
) -> Dict[str, Union[str, Dict[str, str], List[Dict[str, str]]]]:
|
789
901
|
"""
|
790
902
|
Return a dictionary of index keys and index names.
|
791
903
|
|
@@ -796,4 +908,4 @@ def get_pipe_index_names(
|
|
796
908
|
metric_key,
|
797
909
|
location_key,
|
798
910
|
instance_keys,
|
799
|
-
).get_indices(
|
911
|
+
).get_indices()
|
meerschaum/config/_default.py
CHANGED
meerschaum/config/_version.py
CHANGED
@@ -6,8 +6,10 @@
|
|
6
6
|
Interact with Meerschaum APIs. May be chained together (see 'meerschaum:api_instance' in your config.yaml).
|
7
7
|
"""
|
8
8
|
|
9
|
+
from __future__ import annotations
|
10
|
+
|
9
11
|
from datetime import datetime, timedelta, timezone
|
10
|
-
from meerschaum.utils.typing import Optional, List
|
12
|
+
from meerschaum.utils.typing import Optional, List, Union
|
11
13
|
from meerschaum.connectors import Connector
|
12
14
|
from meerschaum.utils.warnings import warn, error
|
13
15
|
from meerschaum.utils.packages import attempt_import
|
@@ -43,6 +45,7 @@ class APIConnector(Connector):
|
|
43
45
|
)
|
44
46
|
from ._misc import get_mrsm_version, get_chaining_status
|
45
47
|
from ._pipes import (
|
48
|
+
get_pipe_instance_keys,
|
46
49
|
register_pipe,
|
47
50
|
fetch_pipes_keys,
|
48
51
|
edit_pipe,
|
@@ -143,6 +146,7 @@ class APIConnector(Connector):
|
|
143
146
|
self._token = None
|
144
147
|
self._expires = None
|
145
148
|
self._session = None
|
149
|
+
self._instance_keys = self.__dict__.get('instance_keys', None)
|
146
150
|
|
147
151
|
|
148
152
|
@property
|
@@ -195,3 +199,10 @@ class APIConnector(Connector):
|
|
195
199
|
warn(msg, stack=False)
|
196
200
|
self._emitted_warning = True
|
197
201
|
return self._token
|
202
|
+
|
203
|
+
@property
|
204
|
+
def instance_keys(self) -> Union[str, None]:
|
205
|
+
"""
|
206
|
+
Return the instance keys to be sent alongside pipe requests.
|
207
|
+
"""
|
208
|
+
return self._instance_keys
|
@@ -31,6 +31,14 @@ def pipe_r_url(
|
|
31
31
|
)
|
32
32
|
|
33
33
|
|
34
|
+
def get_pipe_instance_keys(self, pipe: mrsm.Pipe) -> Union[str, None]:
|
35
|
+
"""
|
36
|
+
Return the configured instance keys for a pipe if set,
|
37
|
+
else fall back to the default `instance_keys` for this `APIConnector`.
|
38
|
+
"""
|
39
|
+
return pipe.parameters.get('instance_keys', self.instance_keys)
|
40
|
+
|
41
|
+
|
34
42
|
def register_pipe(
|
35
43
|
self,
|
36
44
|
pipe: mrsm.Pipe,
|
@@ -40,13 +48,12 @@ def register_pipe(
|
|
40
48
|
Returns a tuple of (success_bool, response_dict).
|
41
49
|
"""
|
42
50
|
from meerschaum.utils.debug import dprint
|
43
|
-
### NOTE: if `parameters` is supplied in the Pipe constructor,
|
44
|
-
### then `pipe.parameters` will exist and not be fetched from the database.
|
45
51
|
r_url = pipe_r_url(pipe)
|
46
52
|
response = self.post(
|
47
53
|
r_url + '/register',
|
48
|
-
json
|
49
|
-
|
54
|
+
json=pipe._attributes.get('parameters', {}),
|
55
|
+
params={'instance_keys': self.get_pipe_instance_keys(pipe)},
|
56
|
+
debug=debug,
|
50
57
|
)
|
51
58
|
if debug:
|
52
59
|
dprint(response.text)
|
@@ -79,9 +86,9 @@ def edit_pipe(
|
|
79
86
|
r_url = pipe_r_url(pipe)
|
80
87
|
response = self.patch(
|
81
88
|
r_url + '/edit',
|
82
|
-
params
|
83
|
-
json
|
84
|
-
debug
|
89
|
+
params={'patch': patch, 'instance_keys': self.get_pipe_instance_keys(pipe)},
|
90
|
+
json=pipe.parameters,
|
91
|
+
debug=debug,
|
85
92
|
)
|
86
93
|
if debug:
|
87
94
|
dprint(response.text)
|
@@ -149,12 +156,13 @@ def fetch_pipes_keys(
|
|
149
156
|
try:
|
150
157
|
j = self.get(
|
151
158
|
r_url,
|
152
|
-
params
|
159
|
+
params={
|
153
160
|
'connector_keys': json.dumps(connector_keys),
|
154
161
|
'metric_keys': json.dumps(metric_keys),
|
155
162
|
'location_keys': json.dumps(location_keys),
|
156
163
|
'tags': json.dumps(tags),
|
157
164
|
'params': json.dumps(params),
|
165
|
+
'instance_keys': self.instance_keys,
|
158
166
|
},
|
159
167
|
debug=debug
|
160
168
|
).json()
|
@@ -250,8 +258,10 @@ def sync_pipe(
|
|
250
258
|
chunks = (df[i] for i in more_itertools.chunked(df, _chunksize))
|
251
259
|
|
252
260
|
### Send columns in case the user has defined them locally.
|
261
|
+
request_params = kw.copy()
|
253
262
|
if pipe.columns:
|
254
|
-
|
263
|
+
request_params['columns'] = json.dumps(pipe.columns)
|
264
|
+
request_params['instance_keys'] = self.get_pipe_instance_keys(pipe)
|
255
265
|
r_url = pipe_r_url(pipe) + '/data'
|
256
266
|
|
257
267
|
rowcount = 0
|
@@ -268,10 +278,9 @@ def sync_pipe(
|
|
268
278
|
try:
|
269
279
|
response = self.post(
|
270
280
|
r_url,
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
debug = debug
|
281
|
+
params=request_params,
|
282
|
+
data=json_str,
|
283
|
+
debug=debug,
|
275
284
|
)
|
276
285
|
except Exception as e:
|
277
286
|
msg = f"Failed to post a chunk to {pipe}:\n{e}"
|
@@ -323,7 +332,8 @@ def delete_pipe(
|
|
323
332
|
r_url = pipe_r_url(pipe)
|
324
333
|
response = self.delete(
|
325
334
|
r_url + '/delete',
|
326
|
-
|
335
|
+
params={'instance_keys': self.get_pipe_instance_keys(pipe)},
|
336
|
+
debug=debug,
|
327
337
|
)
|
328
338
|
if debug:
|
329
339
|
dprint(response.text)
|
@@ -361,7 +371,9 @@ def get_pipe_data(
|
|
361
371
|
'omit_columns': json.dumps(omit_columns),
|
362
372
|
'begin': begin,
|
363
373
|
'end': end,
|
364
|
-
'params': json.dumps(params, default=str)
|
374
|
+
'params': json.dumps(params, default=str),
|
375
|
+
'instance': self.get_pipe_instance_keys(pipe),
|
376
|
+
'as_chunks': as_chunks,
|
365
377
|
},
|
366
378
|
debug=debug
|
367
379
|
)
|