meerschaum 2.9.4__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +5 -2
- meerschaum/_internal/__init__.py +1 -0
- meerschaum/_internal/arguments/_parse_arguments.py +4 -4
- meerschaum/_internal/arguments/_parser.py +17 -1
- meerschaum/_internal/entry.py +6 -6
- meerschaum/_internal/shell/Shell.py +1 -1
- meerschaum/_internal/static.py +372 -0
- meerschaum/actions/api.py +12 -2
- meerschaum/actions/bootstrap.py +7 -7
- meerschaum/actions/edit.py +142 -18
- meerschaum/actions/register.py +137 -6
- meerschaum/actions/show.py +117 -29
- meerschaum/actions/stop.py +4 -1
- meerschaum/actions/sync.py +1 -1
- meerschaum/actions/tag.py +9 -8
- meerschaum/api/__init__.py +9 -2
- meerschaum/api/_events.py +39 -2
- meerschaum/api/_oauth2.py +118 -8
- meerschaum/api/_tokens.py +102 -0
- meerschaum/api/dash/__init__.py +0 -1
- meerschaum/api/dash/callbacks/custom.py +2 -2
- meerschaum/api/dash/callbacks/dashboard.py +133 -18
- meerschaum/api/dash/callbacks/plugins.py +0 -1
- meerschaum/api/dash/callbacks/register.py +1 -1
- meerschaum/api/dash/callbacks/settings/__init__.py +1 -0
- meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
- meerschaum/api/dash/callbacks/settings/tokens.py +388 -0
- meerschaum/api/dash/components.py +30 -8
- meerschaum/api/dash/keys.py +19 -93
- meerschaum/api/dash/pages/dashboard.py +1 -20
- meerschaum/api/dash/pages/settings/__init__.py +1 -0
- meerschaum/api/dash/pages/settings/password_reset.py +1 -1
- meerschaum/api/dash/pages/settings/tokens.py +55 -0
- meerschaum/api/dash/pipes.py +156 -58
- meerschaum/api/dash/sessions.py +12 -0
- meerschaum/api/dash/tokens.py +606 -0
- meerschaum/api/dash/websockets.py +1 -1
- meerschaum/api/dash/webterm.py +4 -0
- meerschaum/api/models/__init__.py +23 -3
- meerschaum/api/models/_actions.py +22 -0
- meerschaum/api/models/_pipes.py +85 -7
- meerschaum/api/models/_tokens.py +81 -0
- meerschaum/api/resources/static/css/dash.css +16 -0
- meerschaum/api/resources/templates/termpage.html +12 -0
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +3 -4
- meerschaum/api/routes/_connectors.py +3 -7
- meerschaum/api/routes/_jobs.py +14 -35
- meerschaum/api/routes/_login.py +49 -12
- meerschaum/api/routes/_misc.py +5 -10
- meerschaum/api/routes/_pipes.py +134 -111
- meerschaum/api/routes/_plugins.py +38 -28
- meerschaum/api/routes/_tokens.py +236 -0
- meerschaum/api/routes/_users.py +47 -35
- meerschaum/api/routes/_version.py +3 -3
- meerschaum/config/__init__.py +43 -20
- meerschaum/config/_default.py +32 -5
- meerschaum/config/_edit.py +28 -24
- meerschaum/config/_environment.py +1 -1
- meerschaum/config/_patch.py +6 -6
- meerschaum/config/_paths.py +5 -1
- meerschaum/config/_read_config.py +65 -34
- meerschaum/config/_sync.py +6 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +24 -5
- meerschaum/config/static.py +18 -0
- meerschaum/connectors/_Connector.py +10 -4
- meerschaum/connectors/__init__.py +4 -20
- meerschaum/connectors/api/_APIConnector.py +34 -6
- meerschaum/connectors/api/_actions.py +2 -2
- meerschaum/connectors/api/_jobs.py +1 -1
- meerschaum/connectors/api/_login.py +33 -7
- meerschaum/connectors/api/_misc.py +2 -2
- meerschaum/connectors/api/_pipes.py +15 -14
- meerschaum/connectors/api/_plugins.py +2 -2
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/api/_tokens.py +146 -0
- meerschaum/connectors/api/_users.py +70 -58
- meerschaum/connectors/instance/_InstanceConnector.py +83 -0
- meerschaum/connectors/instance/__init__.py +10 -0
- meerschaum/connectors/instance/_pipes.py +442 -0
- meerschaum/connectors/instance/_plugins.py +151 -0
- meerschaum/connectors/instance/_tokens.py +296 -0
- meerschaum/connectors/instance/_users.py +181 -0
- meerschaum/connectors/parse.py +4 -1
- meerschaum/connectors/sql/_SQLConnector.py +8 -5
- meerschaum/connectors/sql/_cli.py +12 -11
- meerschaum/connectors/sql/_create_engine.py +6 -154
- meerschaum/connectors/sql/_fetch.py +2 -18
- meerschaum/connectors/sql/_pipes.py +42 -31
- meerschaum/connectors/sql/_plugins.py +29 -0
- meerschaum/connectors/sql/_sql.py +9 -2
- meerschaum/connectors/sql/_users.py +29 -2
- meerschaum/connectors/sql/tables/__init__.py +1 -1
- meerschaum/connectors/valkey/_ValkeyConnector.py +2 -4
- meerschaum/connectors/valkey/_pipes.py +9 -10
- meerschaum/connectors/valkey/_plugins.py +2 -26
- meerschaum/core/Pipe/__init__.py +31 -14
- meerschaum/core/Pipe/_attributes.py +156 -58
- meerschaum/core/Pipe/_bootstrap.py +54 -24
- meerschaum/core/Pipe/_data.py +41 -1
- meerschaum/core/Pipe/_dtypes.py +29 -14
- meerschaum/core/Pipe/_edit.py +12 -4
- meerschaum/core/Pipe/_show.py +5 -5
- meerschaum/core/Pipe/_sync.py +48 -53
- meerschaum/core/Pipe/_verify.py +1 -1
- meerschaum/{plugins → core/Plugin}/_Plugin.py +9 -11
- meerschaum/core/Plugin/__init__.py +1 -1
- meerschaum/core/Token/_Token.py +221 -0
- meerschaum/core/Token/__init__.py +12 -0
- meerschaum/core/User/_User.py +34 -8
- meerschaum/core/User/__init__.py +9 -1
- meerschaum/core/__init__.py +1 -0
- meerschaum/jobs/_Job.py +3 -2
- meerschaum/jobs/__init__.py +3 -2
- meerschaum/jobs/systemd.py +1 -1
- meerschaum/models/__init__.py +35 -0
- meerschaum/models/pipes.py +247 -0
- meerschaum/models/tokens.py +38 -0
- meerschaum/models/users.py +26 -0
- meerschaum/plugins/__init__.py +22 -7
- meerschaum/plugins/bootstrap.py +2 -1
- meerschaum/utils/_get_pipes.py +68 -27
- meerschaum/utils/daemon/Daemon.py +2 -1
- meerschaum/utils/daemon/__init__.py +30 -2
- meerschaum/utils/dataframe.py +96 -15
- meerschaum/utils/dtypes/__init__.py +93 -21
- meerschaum/utils/dtypes/sql.py +44 -0
- meerschaum/utils/formatting/__init__.py +1 -1
- meerschaum/utils/formatting/_pipes.py +5 -4
- meerschaum/utils/formatting/_shell.py +11 -9
- meerschaum/utils/misc.py +237 -80
- meerschaum/utils/packages/__init__.py +3 -6
- meerschaum/utils/packages/_packages.py +34 -32
- meerschaum/utils/pipes.py +181 -0
- meerschaum/utils/process.py +1 -1
- meerschaum/utils/prompt.py +3 -1
- meerschaum/utils/schedule.py +1 -0
- meerschaum/utils/sql.py +115 -39
- meerschaum/utils/typing.py +1 -4
- meerschaum/utils/venv/_Venv.py +2 -2
- meerschaum/utils/venv/__init__.py +5 -7
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0rc1.dist-info}/METADATA +88 -80
- meerschaum-3.0.0rc1.dist-info/RECORD +282 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0rc1.dist-info}/WHEEL +1 -1
- meerschaum/api/models/_interfaces.py +0 -15
- meerschaum/api/models/_locations.py +0 -15
- meerschaum/api/models/_metrics.py +0 -15
- meerschaum/config/static/__init__.py +0 -186
- meerschaum-2.9.4.dist-info/RECORD +0 -263
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0rc1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0rc1.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0rc1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0rc1.dist-info}/zip-safe +0 -0
@@ -14,6 +14,8 @@ import json
|
|
14
14
|
from meerschaum.config._paths import (
|
15
15
|
GRAFANA_DATASOURCE_PATH,
|
16
16
|
GRAFANA_DASHBOARD_PATH,
|
17
|
+
DB_INIT_RESOURCES_PATH,
|
18
|
+
DB_CREATE_EXTENSIONS_PATH,
|
17
19
|
ROOT_DIR_PATH,
|
18
20
|
)
|
19
21
|
from meerschaum.config._paths import STACK_COMPOSE_FILENAME, STACK_ENV_FILENAME
|
@@ -39,7 +41,7 @@ valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
|
|
39
41
|
|
40
42
|
env_dict = {
|
41
43
|
'COMPOSE_PROJECT_NAME': 'mrsm',
|
42
|
-
'TIMESCALEDB_VERSION': '
|
44
|
+
'TIMESCALEDB_VERSION': 'pg17',
|
43
45
|
'POSTGRES_USER': db_user,
|
44
46
|
'POSTGRES_PASSWORD': db_pass,
|
45
47
|
'POSTGRES_DB': db_base,
|
@@ -120,8 +122,9 @@ default_docker_compose_config = {
|
|
120
122
|
'POSTGRES_DB': '<DOLLAR>POSTGRES_DB',
|
121
123
|
'POSTGRES_PASSWORD': '<DOLLAR>POSTGRES_PASSWORD',
|
122
124
|
'ALLOW_IP_RANGE': env_dict['ALLOW_IP_RANGE'],
|
125
|
+
'POSTGRES_INITDB_ARGS': '-c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100'
|
123
126
|
},
|
124
|
-
'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB',
|
127
|
+
# 'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB',
|
125
128
|
'healthcheck': {
|
126
129
|
'test': [
|
127
130
|
'CMD-SHELL', 'pg_isready -d <DOLLAR>POSTGRES_DB -U <DOLLAR>POSTGRES_USER',
|
@@ -131,13 +134,14 @@ default_docker_compose_config = {
|
|
131
134
|
'retries': 5
|
132
135
|
},
|
133
136
|
'restart': 'always',
|
134
|
-
'image': 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
|
137
|
+
'image': 'timescale/timescaledb-ha:' + env_dict['TIMESCALEDB_VERSION'],
|
135
138
|
'ports': [
|
136
139
|
f'{db_port}:5432',
|
137
140
|
],
|
138
141
|
'hostname': db_hostname,
|
139
142
|
'volumes': [
|
140
143
|
'meerschaum_db_data:' + volumes['meerschaum_db_data'],
|
144
|
+
f'{DB_INIT_RESOURCES_PATH.as_posix()}:/docker-entrypoint-initdb.d:z,ro',
|
141
145
|
],
|
142
146
|
'shm_size': '1024m',
|
143
147
|
'networks': [
|
@@ -224,8 +228,8 @@ default_docker_compose_config = {
|
|
224
228
|
'volumes': [
|
225
229
|
'grafana_storage' + ':' + volumes['grafana_storage'],
|
226
230
|
### NOTE: Mount with the 'z' option for SELinux.
|
227
|
-
f'{GRAFANA_DATASOURCE_PATH.parent}:/etc/grafana/provisioning/datasources:z,ro',
|
228
|
-
f'{GRAFANA_DASHBOARD_PATH.parent}:/etc/grafana/provisioning/dashboards:z,ro',
|
231
|
+
f'{GRAFANA_DATASOURCE_PATH.parent.as_posix()}:/etc/grafana/provisioning/datasources:z,ro',
|
232
|
+
f'{GRAFANA_DASHBOARD_PATH.parent.as_posix()}:/etc/grafana/provisioning/dashboards:z,ro',
|
229
233
|
],
|
230
234
|
'environment': {
|
231
235
|
'GF_SECURITY_ALLOW_EMBEDDING': 'true',
|
@@ -273,6 +277,21 @@ def _sync_stack_files():
|
|
273
277
|
substitute = True,
|
274
278
|
)
|
275
279
|
|
280
|
+
_write_initdb()
|
281
|
+
|
282
|
+
def _write_initdb():
|
283
|
+
create_postgis_text = (
|
284
|
+
"CREATE EXTENSION IF NOT EXISTS timescaledb;\n"
|
285
|
+
"CREATE EXTENSION IF NOT EXISTS postgis;\n"
|
286
|
+
"CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit;\n"
|
287
|
+
"CREATE EXTENSION IF NOT EXISTS pg_stat_statements;\n"
|
288
|
+
)
|
289
|
+
if DB_CREATE_EXTENSIONS_PATH.exists():
|
290
|
+
return
|
291
|
+
|
292
|
+
with open(DB_CREATE_EXTENSIONS_PATH, 'w+', encoding='utf-8') as f:
|
293
|
+
f.write(create_postgis_text)
|
294
|
+
|
276
295
|
NECESSARY_FILES = [STACK_COMPOSE_PATH, GRAFANA_DATASOURCE_PATH, GRAFANA_DASHBOARD_PATH]
|
277
296
|
def get_necessary_files():
|
278
297
|
from meerschaum.config import get_config
|
@@ -0,0 +1,18 @@
|
|
1
|
+
#! /usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# vim:fenc=utf-8
|
4
|
+
|
5
|
+
"""
|
6
|
+
Alias import for the internal static configuration dictionary.
|
7
|
+
"""
|
8
|
+
|
9
|
+
from meerschaum._internal.static import SERVER_ID, STATIC_CONFIG
|
10
|
+
|
11
|
+
__all__ = ('STATIC_CONFIG',)
|
12
|
+
|
13
|
+
|
14
|
+
def _static_config():
|
15
|
+
"""
|
16
|
+
Alias function for the global `STATIC_CONFIG` dictionary.
|
17
|
+
"""
|
18
|
+
return STATIC_CONFIG
|
@@ -7,10 +7,12 @@ Define the parent `Connector` class.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
+
|
10
11
|
import abc
|
11
12
|
import copy
|
12
13
|
from meerschaum.utils.typing import Iterable, Optional, Any, Union, List, Dict
|
13
14
|
|
15
|
+
|
14
16
|
class InvalidAttributesError(Exception):
|
15
17
|
"""
|
16
18
|
Raised when the incorrect attributes are set in the Connector.
|
@@ -20,6 +22,9 @@ class Connector(metaclass=abc.ABCMeta):
|
|
20
22
|
"""
|
21
23
|
The base connector class to hold connection attributes.
|
22
24
|
"""
|
25
|
+
|
26
|
+
IS_INSTANCE: bool = False
|
27
|
+
|
23
28
|
def __init__(
|
24
29
|
self,
|
25
30
|
type: Optional[str] = None,
|
@@ -70,7 +75,7 @@ class Connector(metaclass=abc.ABCMeta):
|
|
70
75
|
inherit_default: bool = True,
|
71
76
|
**kw: Any
|
72
77
|
):
|
73
|
-
from meerschaum.
|
78
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
74
79
|
from meerschaum.utils.warnings import error
|
75
80
|
|
76
81
|
self._attributes = {}
|
@@ -151,7 +156,7 @@ class Connector(metaclass=abc.ABCMeta):
|
|
151
156
|
from meerschaum.utils.debug import dprint
|
152
157
|
from meerschaum.utils.misc import items_str
|
153
158
|
if required_attributes is None:
|
154
|
-
required_attributes = ['label']
|
159
|
+
required_attributes = ['type', 'label']
|
155
160
|
|
156
161
|
missing_attributes = set()
|
157
162
|
for a in required_attributes:
|
@@ -213,6 +218,8 @@ class Connector(metaclass=abc.ABCMeta):
|
|
213
218
|
else r'executor$'
|
214
219
|
)
|
215
220
|
_type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
|
221
|
+
if not _type or _type.lower() == 'instance':
|
222
|
+
raise ValueError("No type could be determined for this connector.")
|
216
223
|
self.__dict__['type'] = _type
|
217
224
|
return _type
|
218
225
|
|
@@ -224,8 +231,7 @@ class Connector(metaclass=abc.ABCMeta):
|
|
224
231
|
"""
|
225
232
|
_label = self.__dict__.get('label', None)
|
226
233
|
if _label is None:
|
227
|
-
from meerschaum.
|
234
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
228
235
|
_label = STATIC_CONFIG['connectors']['default_label']
|
229
236
|
self.__dict__['label'] = _label
|
230
237
|
return _label
|
231
|
-
|
@@ -19,13 +19,14 @@ from meerschaum.utils.threading import RLock
|
|
19
19
|
from meerschaum.utils.warnings import warn
|
20
20
|
|
21
21
|
from meerschaum.connectors._Connector import Connector, InvalidAttributesError
|
22
|
+
from meerschaum.connectors.instance._InstanceConnector import InstanceConnector
|
22
23
|
from meerschaum.connectors.sql._SQLConnector import SQLConnector
|
23
24
|
from meerschaum.connectors.api._APIConnector import APIConnector
|
24
|
-
from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
|
25
25
|
|
26
26
|
__all__ = (
|
27
27
|
"make_connector",
|
28
28
|
"Connector",
|
29
|
+
"InstanceConnector",
|
29
30
|
"SQLConnector",
|
30
31
|
"APIConnector",
|
31
32
|
"get_connector",
|
@@ -53,24 +54,7 @@ _locks: Dict[str, RLock] = {
|
|
53
54
|
'_loaded_plugin_connectors': RLock(),
|
54
55
|
'instance_types' : RLock(),
|
55
56
|
}
|
56
|
-
|
57
|
-
'api': {
|
58
|
-
'required': [
|
59
|
-
'host',
|
60
|
-
'username',
|
61
|
-
'password',
|
62
|
-
],
|
63
|
-
'optional': [
|
64
|
-
'port',
|
65
|
-
],
|
66
|
-
'default': {
|
67
|
-
'protocol': 'http',
|
68
|
-
},
|
69
|
-
},
|
70
|
-
'sql': {
|
71
|
-
'flavors': sql_flavor_configs,
|
72
|
-
},
|
73
|
-
}
|
57
|
+
|
74
58
|
### Fill this with objects only when connectors are first referenced.
|
75
59
|
types: Dict[str, Any] = {}
|
76
60
|
custom_types: set = set()
|
@@ -130,7 +114,7 @@ def get_connector(
|
|
130
114
|
"""
|
131
115
|
from meerschaum.connectors.parse import parse_instance_keys
|
132
116
|
from meerschaum.config import get_config
|
133
|
-
from meerschaum.
|
117
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
134
118
|
from meerschaum.utils.warnings import warn
|
135
119
|
global _loaded_plugin_connectors
|
136
120
|
if isinstance(type, str) and not label and ':' in type:
|
@@ -10,7 +10,7 @@ from __future__ import annotations
|
|
10
10
|
|
11
11
|
from datetime import datetime, timedelta, timezone
|
12
12
|
from meerschaum.utils.typing import Optional, List, Union
|
13
|
-
from meerschaum.connectors import
|
13
|
+
from meerschaum.connectors import InstanceConnector
|
14
14
|
from meerschaum.utils.warnings import warn, error
|
15
15
|
from meerschaum.utils.packages import attempt_import
|
16
16
|
|
@@ -18,15 +18,13 @@ required_attributes = {
|
|
18
18
|
'host',
|
19
19
|
}
|
20
20
|
|
21
|
-
class APIConnector(
|
21
|
+
class APIConnector(InstanceConnector):
|
22
22
|
"""
|
23
23
|
Connect to a Meerschaum API instance.
|
24
24
|
"""
|
25
25
|
|
26
|
-
IS_INSTANCE: bool = True
|
27
26
|
IS_THREAD_SAFE: bool = False
|
28
|
-
|
29
|
-
OPTIONAL_ATTRIBUTES: List[str] = ['port']
|
27
|
+
OPTIONAL_ATTRIBUTES: List[str] = ['port', 'client_secret', 'client_id', 'api_key']
|
30
28
|
|
31
29
|
from ._request import (
|
32
30
|
make_request,
|
@@ -82,6 +80,16 @@ class APIConnector(Connector):
|
|
82
80
|
get_user_type,
|
83
81
|
get_user_attributes,
|
84
82
|
)
|
83
|
+
from ._tokens import (
|
84
|
+
register_token,
|
85
|
+
get_token_model,
|
86
|
+
get_tokens,
|
87
|
+
edit_token,
|
88
|
+
invalidate_token,
|
89
|
+
get_token_scopes,
|
90
|
+
token_exists,
|
91
|
+
delete_token,
|
92
|
+
)
|
85
93
|
from ._uri import from_uri
|
86
94
|
from ._jobs import (
|
87
95
|
get_jobs,
|
@@ -154,9 +162,15 @@ class APIConnector(Connector):
|
|
154
162
|
"""
|
155
163
|
Return the fully qualified URI.
|
156
164
|
"""
|
165
|
+
import urllib.parse
|
157
166
|
username = self.__dict__.get('username', None)
|
158
167
|
password = self.__dict__.get('password', None)
|
168
|
+
client_id = self.__dict__.get('client_id', None)
|
169
|
+
client_secret = self.__dict__.get('client_secret', None)
|
170
|
+
api_key = self.__dict__.get('api_key', None)
|
159
171
|
creds = (username + ':' + password + '@') if username and password else ''
|
172
|
+
params = {}
|
173
|
+
params_str = ('?' + urllib.parse.urlencode(params)) if params else ''
|
160
174
|
return (
|
161
175
|
self.protocol
|
162
176
|
+ '://'
|
@@ -167,9 +181,9 @@ class APIConnector(Connector):
|
|
167
181
|
if self.__dict__.get('port', None)
|
168
182
|
else ''
|
169
183
|
)
|
184
|
+
+ params_str
|
170
185
|
)
|
171
186
|
|
172
|
-
|
173
187
|
@property
|
174
188
|
def session(self):
|
175
189
|
if self._session is None:
|
@@ -206,3 +220,17 @@ class APIConnector(Connector):
|
|
206
220
|
Return the instance keys to be sent alongside pipe requests.
|
207
221
|
"""
|
208
222
|
return self._instance_keys
|
223
|
+
|
224
|
+
@property
|
225
|
+
def login_scheme(self) -> str:
|
226
|
+
"""
|
227
|
+
Return the login scheme to use based on the configured credentials.
|
228
|
+
"""
|
229
|
+
if 'username' in self.__dict__:
|
230
|
+
return 'password'
|
231
|
+
if 'client_id' in self.__dict__:
|
232
|
+
return 'client_credentials'
|
233
|
+
elif 'api_key' in self.__dict__:
|
234
|
+
return 'api_key'
|
235
|
+
|
236
|
+
raise ValueError(f"Could not determine the login scheme for '{self}'.")
|
@@ -14,7 +14,7 @@ from functools import partial
|
|
14
14
|
|
15
15
|
import meerschaum as mrsm
|
16
16
|
from meerschaum.utils.typing import SuccessTuple, List, Callable, Optional
|
17
|
-
from meerschaum.
|
17
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
18
18
|
|
19
19
|
ACTIONS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['actions']
|
20
20
|
TEMP_PREFIX: str = STATIC_CONFIG['api']['jobs']['temp_prefix']
|
@@ -88,7 +88,7 @@ def do_action_legacy(
|
|
88
88
|
"""
|
89
89
|
import sys, json
|
90
90
|
from meerschaum.utils.debug import dprint
|
91
|
-
from meerschaum.
|
91
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
92
92
|
from meerschaum.utils.misc import json_serialize_datetime
|
93
93
|
if action is None:
|
94
94
|
action = []
|
@@ -14,7 +14,7 @@ from datetime import datetime
|
|
14
14
|
import meerschaum as mrsm
|
15
15
|
from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable, Optional
|
16
16
|
from meerschaum.jobs import Job
|
17
|
-
from meerschaum.
|
17
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
18
18
|
from meerschaum.utils.warnings import warn, dprint
|
19
19
|
|
20
20
|
JOBS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['jobs']
|
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|
11
11
|
import json
|
12
12
|
import datetime
|
13
13
|
from meerschaum.utils.typing import SuccessTuple, Any, Union
|
14
|
-
from meerschaum.
|
14
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
15
15
|
from meerschaum.utils.warnings import warn as _warn
|
16
16
|
|
17
17
|
|
@@ -22,14 +22,40 @@ def login(
|
|
22
22
|
**kw: Any
|
23
23
|
) -> SuccessTuple:
|
24
24
|
"""Log in and set the session token."""
|
25
|
+
if self.login_scheme == 'api_key':
|
26
|
+
validate_response = self.post(
|
27
|
+
STATIC_CONFIG['api']['endpoints']['tokens'] + '/validate',
|
28
|
+
headers={'Authorization': f'Bearer {self.api_key}'},
|
29
|
+
use_token=False,
|
30
|
+
debug=debug,
|
31
|
+
)
|
32
|
+
if not validate_response:
|
33
|
+
return False, "API key is not valid."
|
34
|
+
return True, "API key is valid."
|
35
|
+
|
25
36
|
try:
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
37
|
+
if self.login_scheme == 'password':
|
38
|
+
login_data = {
|
39
|
+
'username': self.username,
|
40
|
+
'password': self.password,
|
41
|
+
}
|
42
|
+
elif self.login_scheme == 'client_credentials':
|
43
|
+
login_data = {
|
44
|
+
'client_id': self.client_id,
|
45
|
+
'client_secret': self.client_secret,
|
46
|
+
}
|
30
47
|
except AttributeError:
|
48
|
+
login_data = {}
|
49
|
+
|
50
|
+
if not login_data:
|
31
51
|
return False, f"Please login with the command `login {self}`."
|
32
52
|
|
53
|
+
login_scheme_msg = (
|
54
|
+
f" as user '{login_data['username']}'"
|
55
|
+
if self.login_scheme == 'username'
|
56
|
+
else ''
|
57
|
+
)
|
58
|
+
|
33
59
|
response = self.post(
|
34
60
|
STATIC_CONFIG['api']['endpoints']['login'],
|
35
61
|
data=login_data,
|
@@ -37,7 +63,7 @@ def login(
|
|
37
63
|
debug=debug,
|
38
64
|
)
|
39
65
|
if response:
|
40
|
-
msg = f"Successfully logged into '{self}'
|
66
|
+
msg = f"Successfully logged into '{self}'{login_scheme_msg}'."
|
41
67
|
self._token = json.loads(response.text)['access_token']
|
42
68
|
self._expires = datetime.datetime.strptime(
|
43
69
|
json.loads(response.text)['expires'],
|
@@ -45,7 +71,7 @@ def login(
|
|
45
71
|
)
|
46
72
|
else:
|
47
73
|
msg = (
|
48
|
-
f"Failed to log into '{self}'
|
74
|
+
f"Failed to log into '{self}'{login_scheme_msg}.\n" +
|
49
75
|
f" Please verify login details for connector '{self}'."
|
50
76
|
)
|
51
77
|
if warn and not self.__dict__.get('_emitted_warning', False):
|
@@ -13,7 +13,7 @@ def get_mrsm_version(self, **kw) -> Optional[str]:
|
|
13
13
|
"""
|
14
14
|
Return the Meerschaum version of the API instance.
|
15
15
|
"""
|
16
|
-
from meerschaum.
|
16
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
17
17
|
try:
|
18
18
|
j = self.get(
|
19
19
|
STATIC_CONFIG['api']['endpoints']['version'] + '/mrsm',
|
@@ -31,7 +31,7 @@ def get_chaining_status(self, **kw) -> Optional[bool]:
|
|
31
31
|
"""
|
32
32
|
Fetch the chaining status of the API instance.
|
33
33
|
"""
|
34
|
-
from meerschaum.
|
34
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
35
35
|
try:
|
36
36
|
response = self.get(
|
37
37
|
STATIC_CONFIG['api']['endpoints']['chaining'],
|
@@ -21,7 +21,7 @@ def pipe_r_url(
|
|
21
21
|
pipe: mrsm.Pipe
|
22
22
|
) -> str:
|
23
23
|
"""Return a relative URL path from a Pipe's keys."""
|
24
|
-
from meerschaum.
|
24
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
25
25
|
location_key = pipe.location_key
|
26
26
|
if location_key is None:
|
27
27
|
location_key = '[None]'
|
@@ -87,7 +87,7 @@ def edit_pipe(
|
|
87
87
|
response = self.patch(
|
88
88
|
r_url + '/edit',
|
89
89
|
params={'patch': patch, 'instance_keys': self.get_pipe_instance_keys(pipe)},
|
90
|
-
json=pipe.
|
90
|
+
json=pipe.get_parameters(apply_symlinks=False),
|
91
91
|
debug=debug,
|
92
92
|
)
|
93
93
|
if debug:
|
@@ -142,7 +142,7 @@ def fetch_pipes_keys(
|
|
142
142
|
-------
|
143
143
|
A list of tuples containing pipes' keys.
|
144
144
|
"""
|
145
|
-
from meerschaum.
|
145
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
146
146
|
if connector_keys is None:
|
147
147
|
connector_keys = []
|
148
148
|
if metric_keys is None:
|
@@ -167,6 +167,8 @@ def fetch_pipes_keys(
|
|
167
167
|
debug=debug
|
168
168
|
).json()
|
169
169
|
except Exception as e:
|
170
|
+
import traceback
|
171
|
+
traceback.print_exc()
|
170
172
|
error(str(e))
|
171
173
|
|
172
174
|
if 'detail' in j:
|
@@ -185,7 +187,8 @@ def sync_pipe(
|
|
185
187
|
"""Sync a DataFrame into a Pipe."""
|
186
188
|
from decimal import Decimal
|
187
189
|
from meerschaum.utils.debug import dprint
|
188
|
-
from meerschaum.utils.
|
190
|
+
from meerschaum.utils.dtypes import json_serialize_value
|
191
|
+
from meerschaum.utils.misc import items_str, interval_str
|
189
192
|
from meerschaum.config import get_config
|
190
193
|
from meerschaum.utils.packages import attempt_import
|
191
194
|
from meerschaum.utils.dataframe import get_numeric_cols, to_json
|
@@ -197,8 +200,10 @@ def sync_pipe(
|
|
197
200
|
|
198
201
|
def get_json_str(c):
|
199
202
|
### allow syncing dict or JSON without needing to import pandas (for IOT devices)
|
200
|
-
if isinstance(c,
|
201
|
-
return
|
203
|
+
if isinstance(c, str):
|
204
|
+
return c
|
205
|
+
if isinstance(c, (dict, list, tuple)):
|
206
|
+
return json.dumps(c, default=json_serialize_value)
|
202
207
|
return to_json(c, orient='columns')
|
203
208
|
|
204
209
|
df = json.loads(df) if isinstance(df, str) else df
|
@@ -228,11 +233,7 @@ def sync_pipe(
|
|
228
233
|
for col in numeric_cols
|
229
234
|
if pipe_dtypes.get(col, None) != 'numeric'
|
230
235
|
]
|
231
|
-
pipe.
|
232
|
-
col: 'numeric'
|
233
|
-
for col in new_numeric_cols
|
234
|
-
})
|
235
|
-
edit_success, edit_msg = pipe.edit(debug=debug)
|
236
|
+
edit_success, edit_msg = pipe.update_parameters({'dtypes': {col: 'numeric' for col in new_numeric_cols}})
|
236
237
|
if not edit_success:
|
237
238
|
warn(
|
238
239
|
"Failed to update new numeric columns "
|
@@ -313,7 +314,7 @@ def sync_pipe(
|
|
313
314
|
|
314
315
|
success_tuple = True, (
|
315
316
|
f"It took {interval_str(timedelta(seconds=(time.perf_counter() - begin)))} "
|
316
|
-
+ "to sync {rowcount:,} row"
|
317
|
+
+ f"to sync {rowcount:,} row"
|
317
318
|
+ ('s' if rowcount != 1 else '')
|
318
319
|
+ f" across {num_success_chunks:,} chunk" + ('s' if num_success_chunks != 1 else '') +
|
319
320
|
f" to {pipe}."
|
@@ -576,7 +577,7 @@ def create_metadata(
|
|
576
577
|
A bool indicating success.
|
577
578
|
"""
|
578
579
|
from meerschaum.utils.debug import dprint
|
579
|
-
from meerschaum.
|
580
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
580
581
|
r_url = STATIC_CONFIG['api']['endpoints']['metadata']
|
581
582
|
response = self.post(r_url, debug=debug)
|
582
583
|
if debug:
|
@@ -663,7 +664,7 @@ def drop_pipe(
|
|
663
664
|
from meerschaum.utils.warnings import error
|
664
665
|
from meerschaum.utils.debug import dprint
|
665
666
|
if pipe is None:
|
666
|
-
error(
|
667
|
+
error("Pipe cannot be None.")
|
667
668
|
r_url = pipe_r_url(pipe)
|
668
669
|
response = self.delete(
|
669
670
|
r_url + '/drop',
|
@@ -16,7 +16,7 @@ def plugin_r_url(
|
|
16
16
|
plugin: Union[mrsm.core.Plugin, str],
|
17
17
|
) -> str:
|
18
18
|
"""Generate a relative URL path from a Plugin."""
|
19
|
-
from meerschaum.
|
19
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
20
20
|
return f"{STATIC_CONFIG['api']['endpoints']['plugins']}/{plugin}"
|
21
21
|
|
22
22
|
|
@@ -111,7 +111,7 @@ def get_plugins(
|
|
111
111
|
"""
|
112
112
|
import json
|
113
113
|
from meerschaum.utils.warnings import error
|
114
|
-
from meerschaum.
|
114
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
115
115
|
response = self.get(
|
116
116
|
STATIC_CONFIG['api']['endpoints']['plugins'],
|
117
117
|
params = {'user_id': user_id, 'search_term': search_term},
|
@@ -11,7 +11,7 @@ import urllib.parse
|
|
11
11
|
import pathlib
|
12
12
|
from meerschaum.utils.typing import Any, Optional, Dict, Union
|
13
13
|
from meerschaum.utils.debug import dprint
|
14
|
-
from meerschaum.
|
14
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
15
15
|
|
16
16
|
METHODS = {
|
17
17
|
'GET',
|
@@ -0,0 +1,146 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# vim:fenc=utf-8
|
3
|
+
|
4
|
+
"""
|
5
|
+
Implement the `APIConnector` token methods.
|
6
|
+
"""
|
7
|
+
|
8
|
+
import json
|
9
|
+
import uuid
|
10
|
+
from datetime import datetime
|
11
|
+
from typing import Union, List, Optional
|
12
|
+
|
13
|
+
import meerschaum as mrsm
|
14
|
+
from meerschaum.core import Token
|
15
|
+
from meerschaum.models import TokenModel
|
16
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
17
|
+
tokens_endpoint = STATIC_CONFIG['api']['endpoints']['tokens']
|
18
|
+
|
19
|
+
|
20
|
+
def register_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple:
|
21
|
+
"""
|
22
|
+
Register the provided token to the API.
|
23
|
+
"""
|
24
|
+
from meerschaum.utils.dtypes import json_serialize_value
|
25
|
+
r_url = tokens_endpoint + '/register'
|
26
|
+
response = self.post(
|
27
|
+
r_url,
|
28
|
+
data=json.dumps({
|
29
|
+
'label': token.label,
|
30
|
+
'scopes': token.scopes,
|
31
|
+
'expiration': token.expiration,
|
32
|
+
}, default=json_serialize_value),
|
33
|
+
debug=debug,
|
34
|
+
)
|
35
|
+
if not response:
|
36
|
+
return False, f"Failed to register token:\n{response.text}"
|
37
|
+
|
38
|
+
data = response.json()
|
39
|
+
token.label = data['label']
|
40
|
+
token.secret = data['secret']
|
41
|
+
token.id = uuid.UUID(data['id'])
|
42
|
+
if data.get('expiration', None):
|
43
|
+
token.expiration = datetime.fromisoformat(data['expiration'])
|
44
|
+
|
45
|
+
return True, f"Registered token '{token.label}'."
|
46
|
+
|
47
|
+
|
48
|
+
def get_token_model(self, token_id: uuid.UUID, debug: bool = False) -> Union[TokenModel, None]:
|
49
|
+
"""
|
50
|
+
Return a token's model from the API instance.
|
51
|
+
"""
|
52
|
+
r_url = tokens_endpoint + f'/{token_id}'
|
53
|
+
response = self.get(r_url, debug=debug)
|
54
|
+
if not response:
|
55
|
+
return None
|
56
|
+
data = response.json()
|
57
|
+
return TokenModel(**data)
|
58
|
+
|
59
|
+
|
60
|
+
def get_tokens(self, labels: Optional[List[str]] = None, debug: bool = False) -> List[Token]:
|
61
|
+
"""
|
62
|
+
Return the tokens registered to the current user.
|
63
|
+
"""
|
64
|
+
from meerschaum.utils.warnings import warn
|
65
|
+
r_url = tokens_endpoint
|
66
|
+
params = {}
|
67
|
+
if labels:
|
68
|
+
params['labels'] = ','.join(labels)
|
69
|
+
response = self.get(r_url, params={'labels': labels}, debug=debug)
|
70
|
+
if not response:
|
71
|
+
warn(f"Could not get tokens from '{self}':\n{response.text}")
|
72
|
+
return []
|
73
|
+
|
74
|
+
tokens = [
|
75
|
+
Token(instance=self, **payload)
|
76
|
+
for payload in response.json()
|
77
|
+
]
|
78
|
+
return tokens
|
79
|
+
|
80
|
+
|
81
|
+
def edit_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple:
|
82
|
+
"""
|
83
|
+
Persist the token's in-memory state to the API.
|
84
|
+
"""
|
85
|
+
r_url = tokens_endpoint + f"/{token.id}/edit"
|
86
|
+
response = self.post(
|
87
|
+
r_url,
|
88
|
+
json={
|
89
|
+
'creation': token.creation.isoformat() if token.creation else None,
|
90
|
+
'expiration': token.expiration.isoformat() if token.expiration else None,
|
91
|
+
'label': token.label,
|
92
|
+
'is_valid': token.is_valid,
|
93
|
+
'scopes': token.scopes,
|
94
|
+
},
|
95
|
+
)
|
96
|
+
if not response:
|
97
|
+
return False, f"Failed to edit token:\n{response.text}"
|
98
|
+
|
99
|
+
success, msg = response.json()
|
100
|
+
return success, msg
|
101
|
+
|
102
|
+
|
103
|
+
def invalidate_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple:
|
104
|
+
"""
|
105
|
+
Invalidate the token, disabling it for future requests.
|
106
|
+
"""
|
107
|
+
r_url = tokens_endpoint + f"/{token.id}/invalidate"
|
108
|
+
response = self.post(r_url)
|
109
|
+
if not response:
|
110
|
+
return False, f"Failed to invalidate token:\n{response.text}"
|
111
|
+
|
112
|
+
success, msg = response.json()
|
113
|
+
return success, msg
|
114
|
+
|
115
|
+
|
116
|
+
def get_token_scopes(self, token_id: Union[uuid.UUID, Token], debug: bool = False) -> List[str]:
|
117
|
+
"""
|
118
|
+
Return the scopes for a token.
|
119
|
+
"""
|
120
|
+
_token_id = (token_id.id if isinstance(token_id, Token) else token_id)
|
121
|
+
model = self.get_token_model(_token_id, debug=debug).scopes
|
122
|
+
return getattr(model, 'scopes', [])
|
123
|
+
|
124
|
+
|
125
|
+
def token_exists(self, token_id: Union[uuid.UUID, Token], debug: bool = False) -> bool:
|
126
|
+
"""
|
127
|
+
Return `True` if a token exists.
|
128
|
+
"""
|
129
|
+
_token_id = (token_id.id if isinstance(token_id, Token) else token_id)
|
130
|
+
model = self.get_token_model(_token_id, debug=debug)
|
131
|
+
if model is None:
|
132
|
+
return False
|
133
|
+
return model.creation is not None
|
134
|
+
|
135
|
+
|
136
|
+
def delete_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple:
|
137
|
+
"""
|
138
|
+
Delete the token from the API.
|
139
|
+
"""
|
140
|
+
r_url = tokens_endpoint + f"/{token.id}"
|
141
|
+
response = self.delete(r_url, debug=debug)
|
142
|
+
if not response:
|
143
|
+
return False, f"Failed to delete token:\n{response.text}"
|
144
|
+
|
145
|
+
success, msg = response.json()
|
146
|
+
return success, msg
|