meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +6 -1
- meerschaum/__main__.py +9 -9
- meerschaum/_internal/arguments/__init__.py +1 -1
- meerschaum/_internal/arguments/_parse_arguments.py +72 -6
- meerschaum/_internal/arguments/_parser.py +45 -15
- meerschaum/_internal/docs/index.py +265 -8
- meerschaum/_internal/entry.py +167 -37
- meerschaum/_internal/shell/Shell.py +290 -99
- meerschaum/_internal/shell/updates.py +175 -0
- meerschaum/actions/__init__.py +29 -17
- meerschaum/actions/api.py +12 -12
- meerschaum/actions/attach.py +113 -0
- meerschaum/actions/copy.py +68 -41
- meerschaum/actions/delete.py +112 -50
- meerschaum/actions/edit.py +3 -3
- meerschaum/actions/install.py +40 -32
- meerschaum/actions/pause.py +44 -27
- meerschaum/actions/register.py +19 -5
- meerschaum/actions/restart.py +107 -0
- meerschaum/actions/show.py +130 -159
- meerschaum/actions/start.py +161 -100
- meerschaum/actions/stop.py +78 -42
- meerschaum/actions/sync.py +3 -3
- meerschaum/actions/upgrade.py +28 -36
- meerschaum/api/_events.py +25 -1
- meerschaum/api/_oauth2.py +2 -0
- meerschaum/api/_websockets.py +2 -2
- meerschaum/api/dash/callbacks/jobs.py +36 -44
- meerschaum/api/dash/jobs.py +89 -78
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +148 -17
- meerschaum/api/routes/_jobs.py +407 -0
- meerschaum/api/routes/_pipes.py +25 -25
- meerschaum/config/_default.py +1 -0
- meerschaum/config/_formatting.py +1 -0
- meerschaum/config/_jobs.py +1 -1
- meerschaum/config/_paths.py +11 -0
- meerschaum/config/_shell.py +84 -67
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +18 -0
- meerschaum/connectors/Connector.py +13 -7
- meerschaum/connectors/__init__.py +28 -15
- meerschaum/connectors/api/APIConnector.py +27 -1
- meerschaum/connectors/api/_actions.py +71 -6
- meerschaum/connectors/api/_jobs.py +368 -0
- meerschaum/connectors/api/_misc.py +1 -1
- meerschaum/connectors/api/_pipes.py +85 -84
- meerschaum/connectors/api/_request.py +13 -9
- meerschaum/connectors/parse.py +27 -15
- meerschaum/core/Pipe/_bootstrap.py +16 -8
- meerschaum/core/Pipe/_sync.py +3 -0
- meerschaum/jobs/_Executor.py +69 -0
- meerschaum/jobs/_Job.py +899 -0
- meerschaum/jobs/__init__.py +396 -0
- meerschaum/jobs/systemd.py +694 -0
- meerschaum/plugins/__init__.py +97 -12
- meerschaum/utils/daemon/Daemon.py +352 -147
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +19 -10
- meerschaum/utils/daemon/RotatingFile.py +22 -8
- meerschaum/utils/daemon/StdinFile.py +121 -0
- meerschaum/utils/daemon/__init__.py +42 -27
- meerschaum/utils/daemon/_names.py +15 -13
- meerschaum/utils/formatting/__init__.py +83 -37
- meerschaum/utils/formatting/_jobs.py +146 -55
- meerschaum/utils/formatting/_shell.py +6 -0
- meerschaum/utils/misc.py +41 -22
- meerschaum/utils/packages/__init__.py +21 -15
- meerschaum/utils/packages/_packages.py +9 -6
- meerschaum/utils/process.py +9 -9
- meerschaum/utils/prompt.py +20 -7
- meerschaum/utils/schedule.py +21 -15
- meerschaum/utils/venv/__init__.py +2 -2
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/METADATA +22 -25
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/RECORD +80 -70
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/zip-safe +0 -0
@@ -0,0 +1,368 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# vim:fenc=utf-8
|
4
|
+
|
5
|
+
"""
|
6
|
+
Manage jobs via the Meerschaum API.
|
7
|
+
"""
|
8
|
+
|
9
|
+
import asyncio
|
10
|
+
import time
|
11
|
+
import json
|
12
|
+
from datetime import datetime
|
13
|
+
|
14
|
+
import meerschaum as mrsm
|
15
|
+
from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable
|
16
|
+
from meerschaum.jobs import Job
|
17
|
+
from meerschaum.config.static import STATIC_CONFIG
|
18
|
+
from meerschaum.utils.warnings import warn, dprint
|
19
|
+
|
20
|
+
JOBS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['jobs']
|
21
|
+
LOGS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['logs']
|
22
|
+
JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
|
23
|
+
JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
|
24
|
+
JOB_METADATA_CACHE_SECONDS: int = STATIC_CONFIG['api']['jobs']['metadata_cache_seconds']
|
25
|
+
|
26
|
+
|
27
|
+
def get_jobs(self, debug: bool = False) -> Dict[str, Job]:
|
28
|
+
"""
|
29
|
+
Return a dictionary of remote jobs.
|
30
|
+
"""
|
31
|
+
response = self.get(JOBS_ENDPOINT, debug=debug)
|
32
|
+
if not response:
|
33
|
+
warn(f"Failed to get remote jobs from {self}.")
|
34
|
+
return {}
|
35
|
+
return {
|
36
|
+
name: Job(
|
37
|
+
name,
|
38
|
+
job_meta['sysargs'],
|
39
|
+
executor_keys=str(self),
|
40
|
+
_properties=job_meta['daemon']['properties']
|
41
|
+
)
|
42
|
+
for name, job_meta in response.json().items()
|
43
|
+
}
|
44
|
+
|
45
|
+
|
46
|
+
def get_job(self, name: str, debug: bool = False) -> Job:
|
47
|
+
"""
|
48
|
+
Return a single Job object.
|
49
|
+
"""
|
50
|
+
metadata = self.get_job_metadata(name, debug=debug)
|
51
|
+
if not metadata:
|
52
|
+
raise ValueError(f"Job '{name}' does not exist.")
|
53
|
+
|
54
|
+
return Job(
|
55
|
+
name,
|
56
|
+
metadata['sysargs'],
|
57
|
+
executor_keys=str(self),
|
58
|
+
_properties=metadata['daemon']['properties'],
|
59
|
+
)
|
60
|
+
|
61
|
+
|
62
|
+
def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
|
63
|
+
"""
|
64
|
+
Return the metadata for a single job.
|
65
|
+
"""
|
66
|
+
now = time.perf_counter()
|
67
|
+
_job_metadata_cache = self.__dict__.get('_job_metadata_cache', None)
|
68
|
+
_job_metadata_timestamp = (
|
69
|
+
_job_metadata_cache.get(name, {}).get('timestamp', None)
|
70
|
+
) if _job_metadata_cache is not None else None
|
71
|
+
|
72
|
+
if (
|
73
|
+
_job_metadata_timestamp is not None
|
74
|
+
and (now - _job_metadata_timestamp) < JOB_METADATA_CACHE_SECONDS
|
75
|
+
):
|
76
|
+
if debug:
|
77
|
+
dprint(f"Returning cached metadata for job '{name}'.")
|
78
|
+
return _job_metadata_cache[name]['metadata']
|
79
|
+
|
80
|
+
response = self.get(JOBS_ENDPOINT + f"/{name}", debug=debug)
|
81
|
+
if not response:
|
82
|
+
if debug:
|
83
|
+
msg = (
|
84
|
+
response.json()['detail']
|
85
|
+
if 'detail' in response.text
|
86
|
+
else response.text
|
87
|
+
)
|
88
|
+
warn(f"Failed to get metadata for job '{name}':\n{msg}")
|
89
|
+
return {}
|
90
|
+
|
91
|
+
metadata = response.json()
|
92
|
+
if _job_metadata_cache is None:
|
93
|
+
self._job_metadata_cache = {}
|
94
|
+
|
95
|
+
self._job_metadata_cache[name] = {
|
96
|
+
'timestamp': now,
|
97
|
+
'metadata': metadata,
|
98
|
+
}
|
99
|
+
return metadata
|
100
|
+
|
101
|
+
def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
|
102
|
+
"""
|
103
|
+
Return the daemon properties for a single job.
|
104
|
+
"""
|
105
|
+
metadata = self.get_job_metadata(name, debug=debug)
|
106
|
+
return metadata.get('daemon', {}).get('properties', {})
|
107
|
+
|
108
|
+
def get_job_status(self, name: str, debug: bool = False) -> str:
|
109
|
+
"""
|
110
|
+
Return the job's status.
|
111
|
+
"""
|
112
|
+
metadata = self.get_job_metadata(name, debug=debug)
|
113
|
+
return metadata.get('status', 'stopped')
|
114
|
+
|
115
|
+
def get_job_began(self, name: str, debug: bool = False) -> Union[str, None]:
|
116
|
+
"""
|
117
|
+
Return a job's `began` timestamp, if it exists.
|
118
|
+
"""
|
119
|
+
properties = self.get_job_properties(name, debug=debug)
|
120
|
+
began_str = properties.get('daemon', {}).get('began', None)
|
121
|
+
if began_str is None:
|
122
|
+
return None
|
123
|
+
|
124
|
+
return began_str
|
125
|
+
|
126
|
+
def get_job_ended(self, name: str, debug: bool = False) -> Union[str, None]:
|
127
|
+
"""
|
128
|
+
Return a job's `ended` timestamp, if it exists.
|
129
|
+
"""
|
130
|
+
properties = self.get_job_properties(name, debug=debug)
|
131
|
+
ended_str = properties.get('daemon', {}).get('ended', None)
|
132
|
+
if ended_str is None:
|
133
|
+
return None
|
134
|
+
|
135
|
+
return ended_str
|
136
|
+
|
137
|
+
def get_job_paused(self, name: str, debug: bool = False) -> Union[str, None]:
|
138
|
+
"""
|
139
|
+
Return a job's `paused` timestamp, if it exists.
|
140
|
+
"""
|
141
|
+
properties = self.get_job_properties(name, debug=debug)
|
142
|
+
paused_str = properties.get('daemon', {}).get('paused', None)
|
143
|
+
if paused_str is None:
|
144
|
+
return None
|
145
|
+
|
146
|
+
return paused_str
|
147
|
+
|
148
|
+
def get_job_exists(self, name: str, debug: bool = False) -> bool:
|
149
|
+
"""
|
150
|
+
Return whether a job exists.
|
151
|
+
"""
|
152
|
+
response = self.get(JOBS_ENDPOINT + f'/{name}/exists', debug=debug)
|
153
|
+
if not response:
|
154
|
+
warn(f"Failed to determine whether job '{name}' exists.")
|
155
|
+
return False
|
156
|
+
|
157
|
+
return response.json()
|
158
|
+
|
159
|
+
|
160
|
+
def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
161
|
+
"""
|
162
|
+
Delete a job.
|
163
|
+
"""
|
164
|
+
response = self.delete(JOBS_ENDPOINT + f"/{name}", debug=debug)
|
165
|
+
if not response:
|
166
|
+
if 'detail' in response.text:
|
167
|
+
return False, response.json()['detail']
|
168
|
+
|
169
|
+
return False, response.text
|
170
|
+
|
171
|
+
return tuple(response.json())
|
172
|
+
|
173
|
+
|
174
|
+
def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
175
|
+
"""
|
176
|
+
Start a job.
|
177
|
+
"""
|
178
|
+
response = self.post(JOBS_ENDPOINT + f"/{name}/start", debug=debug)
|
179
|
+
if not response:
|
180
|
+
if 'detail' in response.text:
|
181
|
+
return False, response.json()['detail']
|
182
|
+
return False, response.text
|
183
|
+
|
184
|
+
return tuple(response.json())
|
185
|
+
|
186
|
+
|
187
|
+
def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
|
188
|
+
"""
|
189
|
+
Create a job.
|
190
|
+
"""
|
191
|
+
response = self.post(JOBS_ENDPOINT + f"/{name}", json=sysargs, debug=debug)
|
192
|
+
if not response:
|
193
|
+
if 'detail' in response.text:
|
194
|
+
return False, response.json()['detail']
|
195
|
+
return False, response.text
|
196
|
+
|
197
|
+
return tuple(response.json())
|
198
|
+
|
199
|
+
|
200
|
+
def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
201
|
+
"""
|
202
|
+
Stop a job.
|
203
|
+
"""
|
204
|
+
response = self.post(JOBS_ENDPOINT + f"/{name}/stop", debug=debug)
|
205
|
+
if not response:
|
206
|
+
if 'detail' in response.text:
|
207
|
+
return False, response.json()['detail']
|
208
|
+
return False, response.text
|
209
|
+
|
210
|
+
return tuple(response.json())
|
211
|
+
|
212
|
+
|
213
|
+
def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
214
|
+
"""
|
215
|
+
Pause a job.
|
216
|
+
"""
|
217
|
+
response = self.post(JOBS_ENDPOINT + f"/{name}/pause", debug=debug)
|
218
|
+
if not response:
|
219
|
+
if 'detail' in response.text:
|
220
|
+
return False, response.json()['detail']
|
221
|
+
return False, response.text
|
222
|
+
|
223
|
+
return tuple(response.json())
|
224
|
+
|
225
|
+
|
226
|
+
def get_logs(self, name: str, debug: bool = False) -> str:
|
227
|
+
"""
|
228
|
+
Return the logs for a job.
|
229
|
+
"""
|
230
|
+
response = self.get(LOGS_ENDPOINT + f"/{name}")
|
231
|
+
if not response:
|
232
|
+
raise ValueError(f"Cannot fetch logs for job '{name}':\n{response.text}")
|
233
|
+
|
234
|
+
return response.json()
|
235
|
+
|
236
|
+
|
237
|
+
def get_job_stop_time(self, name: str, debug: bool = False) -> Union[datetime, None]:
|
238
|
+
"""
|
239
|
+
Return the job's manual stop time.
|
240
|
+
"""
|
241
|
+
response = self.get(JOBS_ENDPOINT + f"/{name}/stop_time")
|
242
|
+
if not response:
|
243
|
+
warn(f"Failed to get stop time for job '{name}':\n{response.text}")
|
244
|
+
return None
|
245
|
+
|
246
|
+
data = response.json()
|
247
|
+
if data is None:
|
248
|
+
return None
|
249
|
+
|
250
|
+
return datetime.fromisoformat(data)
|
251
|
+
|
252
|
+
|
253
|
+
async def monitor_logs_async(
|
254
|
+
self,
|
255
|
+
name: str,
|
256
|
+
callback_function: Callable[[Any], Any],
|
257
|
+
input_callback_function: Callable[[], str],
|
258
|
+
stop_callback_function: Callable[[SuccessTuple], str],
|
259
|
+
stop_on_exit: bool = False,
|
260
|
+
strip_timestamps: bool = False,
|
261
|
+
accept_input: bool = True,
|
262
|
+
debug: bool = False,
|
263
|
+
):
|
264
|
+
"""
|
265
|
+
Monitor a job's log files and await a callback with the changes.
|
266
|
+
"""
|
267
|
+
from meerschaum.jobs import StopMonitoringLogs
|
268
|
+
from meerschaum.utils.formatting._jobs import strip_timestamp_from_line
|
269
|
+
|
270
|
+
websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
|
271
|
+
protocol = 'ws' if self.URI.startswith('http://') else 'wss'
|
272
|
+
port = self.port if 'port' in self.__dict__ else ''
|
273
|
+
uri = f"{protocol}://{self.host}:{port}{LOGS_ENDPOINT}/{name}/ws"
|
274
|
+
|
275
|
+
async def _stdin_callback(client):
|
276
|
+
if input_callback_function is None:
|
277
|
+
return
|
278
|
+
|
279
|
+
if asyncio.iscoroutinefunction(input_callback_function):
|
280
|
+
data = await input_callback_function()
|
281
|
+
else:
|
282
|
+
data = input_callback_function()
|
283
|
+
|
284
|
+
await client.send(data)
|
285
|
+
|
286
|
+
async def _stop_callback(client):
|
287
|
+
try:
|
288
|
+
result = tuple(json.loads(await client.recv()))
|
289
|
+
except Exception as e:
|
290
|
+
warn(traceback.format_exc())
|
291
|
+
result = False, str(e)
|
292
|
+
|
293
|
+
if stop_callback_function is not None:
|
294
|
+
if asyncio.iscoroutinefunction(stop_callback_function):
|
295
|
+
await stop_callback_function(result)
|
296
|
+
else:
|
297
|
+
stop_callback_function(result)
|
298
|
+
|
299
|
+
if stop_on_exit:
|
300
|
+
raise StopMonitoringLogs
|
301
|
+
|
302
|
+
message_callbacks = {
|
303
|
+
JOBS_STDIN_MESSAGE: _stdin_callback,
|
304
|
+
JOBS_STOP_MESSAGE: _stop_callback,
|
305
|
+
}
|
306
|
+
|
307
|
+
async with websockets.connect(uri) as websocket:
|
308
|
+
try:
|
309
|
+
await websocket.send(self.token or 'no-login')
|
310
|
+
except websockets_exceptions.ConnectionClosedOK:
|
311
|
+
pass
|
312
|
+
|
313
|
+
while True:
|
314
|
+
try:
|
315
|
+
response = await websocket.recv()
|
316
|
+
callback = message_callbacks.get(response, None)
|
317
|
+
if callback is not None:
|
318
|
+
await callback(websocket)
|
319
|
+
continue
|
320
|
+
|
321
|
+
if strip_timestamps:
|
322
|
+
response = strip_timestamp_from_line(response)
|
323
|
+
|
324
|
+
if asyncio.iscoroutinefunction(callback_function):
|
325
|
+
await callback_function(response)
|
326
|
+
else:
|
327
|
+
callback_function(response)
|
328
|
+
except (KeyboardInterrupt, StopMonitoringLogs):
|
329
|
+
await websocket.close()
|
330
|
+
break
|
331
|
+
|
332
|
+
|
333
|
+
def monitor_logs(
|
334
|
+
self,
|
335
|
+
name: str,
|
336
|
+
callback_function: Callable[[Any], Any],
|
337
|
+
input_callback_function: Callable[[None], str],
|
338
|
+
stop_callback_function: Callable[[None], str],
|
339
|
+
stop_on_exit: bool = False,
|
340
|
+
strip_timestamps: bool = False,
|
341
|
+
accept_input: bool = True,
|
342
|
+
debug: bool = False,
|
343
|
+
):
|
344
|
+
"""
|
345
|
+
Monitor a job's log files and execute a callback with the changes.
|
346
|
+
"""
|
347
|
+
return asyncio.run(
|
348
|
+
self.monitor_logs_async(
|
349
|
+
name,
|
350
|
+
callback_function,
|
351
|
+
input_callback_function=input_callback_function,
|
352
|
+
stop_callback_function=stop_callback_function,
|
353
|
+
stop_on_exit=stop_on_exit,
|
354
|
+
strip_timestamps=strip_timestamps,
|
355
|
+
accept_input=accept_input,
|
356
|
+
debug=debug
|
357
|
+
)
|
358
|
+
)
|
359
|
+
|
360
|
+
def get_job_is_blocking_on_stdin(self, name: str, debug: bool = False) -> bool:
|
361
|
+
"""
|
362
|
+
Return whether a remote job is blocking on stdin.
|
363
|
+
"""
|
364
|
+
response = self.get(JOBS_ENDPOINT + f'/{name}/is_blocking_on_stdin', debug=debug)
|
365
|
+
if not response:
|
366
|
+
return False
|
367
|
+
|
368
|
+
return response.json()
|
@@ -11,14 +11,15 @@ import time
|
|
11
11
|
import json
|
12
12
|
from io import StringIO
|
13
13
|
from datetime import datetime
|
14
|
+
|
14
15
|
import meerschaum as mrsm
|
15
16
|
from meerschaum.utils.debug import dprint
|
16
17
|
from meerschaum.utils.warnings import warn, error
|
17
18
|
from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
|
18
19
|
|
19
20
|
def pipe_r_url(
|
20
|
-
|
21
|
-
|
21
|
+
pipe: mrsm.Pipe
|
22
|
+
) -> str:
|
22
23
|
"""Return a relative URL path from a Pipe's keys."""
|
23
24
|
from meerschaum.config.static import STATIC_CONFIG
|
24
25
|
location_key = pipe.location_key
|
@@ -30,10 +31,10 @@ def pipe_r_url(
|
|
30
31
|
)
|
31
32
|
|
32
33
|
def register_pipe(
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
self,
|
35
|
+
pipe: mrsm.Pipe,
|
36
|
+
debug: bool = False
|
37
|
+
) -> SuccessTuple:
|
37
38
|
"""Submit a POST to the API to register a new Pipe object.
|
38
39
|
Returns a tuple of (success_bool, response_dict).
|
39
40
|
"""
|
@@ -59,11 +60,11 @@ def register_pipe(
|
|
59
60
|
|
60
61
|
|
61
62
|
def edit_pipe(
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
63
|
+
self,
|
64
|
+
pipe: mrsm.Pipe,
|
65
|
+
patch: bool = False,
|
66
|
+
debug: bool = False,
|
67
|
+
) -> SuccessTuple:
|
67
68
|
"""Submit a PATCH to the API to edit an existing Pipe object.
|
68
69
|
Returns a tuple of (success_bool, response_dict).
|
69
70
|
"""
|
@@ -89,14 +90,14 @@ def edit_pipe(
|
|
89
90
|
|
90
91
|
|
91
92
|
def fetch_pipes_keys(
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
93
|
+
self,
|
94
|
+
connector_keys: Optional[List[str]] = None,
|
95
|
+
metric_keys: Optional[List[str]] = None,
|
96
|
+
location_keys: Optional[List[str]] = None,
|
97
|
+
tags: Optional[List[str]] = None,
|
98
|
+
params: Optional[Dict[str, Any]] = None,
|
99
|
+
debug: bool = False
|
100
|
+
) -> Union[List[Tuple[str, str, Union[str, None]]]]:
|
100
101
|
"""
|
101
102
|
Fetch registered Pipes' keys from the API.
|
102
103
|
|
@@ -158,13 +159,13 @@ def fetch_pipes_keys(
|
|
158
159
|
|
159
160
|
|
160
161
|
def sync_pipe(
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
162
|
+
self,
|
163
|
+
pipe: mrsm.Pipe,
|
164
|
+
df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
|
165
|
+
chunksize: Optional[int] = -1,
|
166
|
+
debug: bool = False,
|
167
|
+
**kw: Any
|
168
|
+
) -> SuccessTuple:
|
168
169
|
"""Sync a DataFrame into a Pipe."""
|
169
170
|
from decimal import Decimal
|
170
171
|
from meerschaum.utils.debug import dprint
|
@@ -303,10 +304,10 @@ def sync_pipe(
|
|
303
304
|
|
304
305
|
|
305
306
|
def delete_pipe(
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
307
|
+
self,
|
308
|
+
pipe: Optional[meerschaum.Pipe] = None,
|
309
|
+
debug: bool = None,
|
310
|
+
) -> SuccessTuple:
|
310
311
|
"""Delete a Pipe and drop its table."""
|
311
312
|
if pipe is None:
|
312
313
|
error(f"Pipe cannot be None.")
|
@@ -327,17 +328,17 @@ def delete_pipe(
|
|
327
328
|
|
328
329
|
|
329
330
|
def get_pipe_data(
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
331
|
+
self,
|
332
|
+
pipe: meerschaum.Pipe,
|
333
|
+
select_columns: Optional[List[str]] = None,
|
334
|
+
omit_columns: Optional[List[str]] = None,
|
335
|
+
begin: Union[str, datetime, int, None] = None,
|
336
|
+
end: Union[str, datetime, int, None] = None,
|
337
|
+
params: Optional[Dict[str, Any]] = None,
|
338
|
+
as_chunks: bool = False,
|
339
|
+
debug: bool = False,
|
340
|
+
**kw: Any
|
341
|
+
) -> Union[pandas.DataFrame, None]:
|
341
342
|
"""Fetch data from the API."""
|
342
343
|
r_url = pipe_r_url(pipe)
|
343
344
|
chunks_list = []
|
@@ -389,10 +390,10 @@ def get_pipe_data(
|
|
389
390
|
|
390
391
|
|
391
392
|
def get_pipe_id(
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
393
|
+
self,
|
394
|
+
pipe: meerschuam.Pipe,
|
395
|
+
debug: bool = False,
|
396
|
+
) -> int:
|
396
397
|
"""Get a Pipe's ID from the API."""
|
397
398
|
from meerschaum.utils.misc import is_int
|
398
399
|
r_url = pipe_r_url(pipe)
|
@@ -411,10 +412,10 @@ def get_pipe_id(
|
|
411
412
|
|
412
413
|
|
413
414
|
def get_pipe_attributes(
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
415
|
+
self,
|
416
|
+
pipe: meerschaum.Pipe,
|
417
|
+
debug: bool = False,
|
418
|
+
) -> Dict[str, Any]:
|
418
419
|
"""Get a Pipe's attributes from the API
|
419
420
|
|
420
421
|
Parameters
|
@@ -437,12 +438,12 @@ def get_pipe_attributes(
|
|
437
438
|
|
438
439
|
|
439
440
|
def get_sync_time(
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
441
|
+
self,
|
442
|
+
pipe: 'meerschaum.Pipe',
|
443
|
+
params: Optional[Dict[str, Any]] = None,
|
444
|
+
newest: bool = True,
|
445
|
+
debug: bool = False,
|
446
|
+
) -> Union[datetime, int, None]:
|
446
447
|
"""Get a Pipe's most recent datetime value from the API.
|
447
448
|
|
448
449
|
Parameters
|
@@ -492,10 +493,10 @@ def get_sync_time(
|
|
492
493
|
|
493
494
|
|
494
495
|
def pipe_exists(
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
496
|
+
self,
|
497
|
+
pipe: mrsm.Pipe,
|
498
|
+
debug: bool = False
|
499
|
+
) -> bool:
|
499
500
|
"""Check the API to see if a Pipe exists.
|
500
501
|
|
501
502
|
Parameters
|
@@ -523,9 +524,9 @@ def pipe_exists(
|
|
523
524
|
|
524
525
|
|
525
526
|
def create_metadata(
|
526
|
-
|
527
|
-
|
528
|
-
|
527
|
+
self,
|
528
|
+
debug: bool = False
|
529
|
+
) -> bool:
|
529
530
|
"""Create metadata tables.
|
530
531
|
|
531
532
|
Returns
|
@@ -547,14 +548,14 @@ def create_metadata(
|
|
547
548
|
|
548
549
|
|
549
550
|
def get_pipe_rowcount(
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
551
|
+
self,
|
552
|
+
pipe: mrsm.Pipe,
|
553
|
+
begin: Optional[datetime] = None,
|
554
|
+
end: Optional[datetime] = None,
|
555
|
+
params: Optional[Dict[str, Any]] = None,
|
556
|
+
remote: bool = False,
|
557
|
+
debug: bool = False,
|
558
|
+
) -> int:
|
558
559
|
"""Get a pipe's row count from the API.
|
559
560
|
|
560
561
|
Parameters
|
@@ -600,10 +601,10 @@ def get_pipe_rowcount(
|
|
600
601
|
|
601
602
|
|
602
603
|
def drop_pipe(
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
604
|
+
self,
|
605
|
+
pipe: mrsm.Pipe,
|
606
|
+
debug: bool = False
|
607
|
+
) -> SuccessTuple:
|
607
608
|
"""
|
608
609
|
Drop a pipe's table but maintain its registration.
|
609
610
|
|
@@ -644,11 +645,11 @@ def drop_pipe(
|
|
644
645
|
|
645
646
|
|
646
647
|
def clear_pipe(
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
648
|
+
self,
|
649
|
+
pipe: mrsm.Pipe,
|
650
|
+
debug: bool = False,
|
651
|
+
**kw
|
652
|
+
) -> SuccessTuple:
|
652
653
|
"""
|
653
654
|
Delete rows in a pipe's table.
|
654
655
|
|
@@ -666,7 +667,7 @@ def clear_pipe(
|
|
666
667
|
kw.pop('location_keys', None)
|
667
668
|
kw.pop('action', None)
|
668
669
|
kw.pop('force', None)
|
669
|
-
return self.
|
670
|
+
return self.do_action_legacy(
|
670
671
|
['clear', 'pipes'],
|
671
672
|
connector_keys = pipe.connector_keys,
|
672
673
|
metric_keys = pipe.metric_key,
|
@@ -678,10 +679,10 @@ def clear_pipe(
|
|
678
679
|
|
679
680
|
|
680
681
|
def get_pipe_columns_types(
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
682
|
+
self,
|
683
|
+
pipe: mrsm.Pipe,
|
684
|
+
debug: bool = False,
|
685
|
+
) -> Union[Dict[str, str], None]:
|
685
686
|
"""
|
686
687
|
Fetch the columns and types of the pipe's table.
|
687
688
|
|