meerschaum 2.9.5__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. meerschaum/__init__.py +5 -2
  2. meerschaum/_internal/__init__.py +1 -0
  3. meerschaum/_internal/arguments/_parse_arguments.py +4 -4
  4. meerschaum/_internal/arguments/_parser.py +17 -1
  5. meerschaum/_internal/entry.py +6 -6
  6. meerschaum/_internal/shell/Shell.py +1 -1
  7. meerschaum/_internal/static.py +372 -0
  8. meerschaum/actions/api.py +12 -2
  9. meerschaum/actions/bootstrap.py +7 -7
  10. meerschaum/actions/edit.py +142 -18
  11. meerschaum/actions/register.py +137 -6
  12. meerschaum/actions/show.py +117 -29
  13. meerschaum/actions/stop.py +4 -1
  14. meerschaum/actions/sync.py +1 -1
  15. meerschaum/actions/tag.py +9 -8
  16. meerschaum/api/__init__.py +9 -2
  17. meerschaum/api/_events.py +39 -2
  18. meerschaum/api/_oauth2.py +118 -8
  19. meerschaum/api/_tokens.py +102 -0
  20. meerschaum/api/dash/__init__.py +0 -1
  21. meerschaum/api/dash/callbacks/custom.py +2 -2
  22. meerschaum/api/dash/callbacks/dashboard.py +102 -18
  23. meerschaum/api/dash/callbacks/plugins.py +0 -1
  24. meerschaum/api/dash/callbacks/register.py +1 -1
  25. meerschaum/api/dash/callbacks/settings/__init__.py +1 -0
  26. meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
  27. meerschaum/api/dash/callbacks/settings/tokens.py +388 -0
  28. meerschaum/api/dash/components.py +30 -8
  29. meerschaum/api/dash/keys.py +19 -93
  30. meerschaum/api/dash/pages/dashboard.py +1 -20
  31. meerschaum/api/dash/pages/settings/__init__.py +1 -0
  32. meerschaum/api/dash/pages/settings/password_reset.py +1 -1
  33. meerschaum/api/dash/pages/settings/tokens.py +55 -0
  34. meerschaum/api/dash/pipes.py +94 -59
  35. meerschaum/api/dash/sessions.py +12 -0
  36. meerschaum/api/dash/tokens.py +606 -0
  37. meerschaum/api/dash/websockets.py +1 -1
  38. meerschaum/api/dash/webterm.py +4 -0
  39. meerschaum/api/models/__init__.py +23 -3
  40. meerschaum/api/models/_actions.py +22 -0
  41. meerschaum/api/models/_pipes.py +85 -7
  42. meerschaum/api/models/_tokens.py +81 -0
  43. meerschaum/api/resources/templates/termpage.html +12 -0
  44. meerschaum/api/routes/__init__.py +1 -0
  45. meerschaum/api/routes/_actions.py +3 -4
  46. meerschaum/api/routes/_connectors.py +3 -7
  47. meerschaum/api/routes/_jobs.py +14 -35
  48. meerschaum/api/routes/_login.py +49 -12
  49. meerschaum/api/routes/_misc.py +5 -10
  50. meerschaum/api/routes/_pipes.py +134 -111
  51. meerschaum/api/routes/_plugins.py +38 -28
  52. meerschaum/api/routes/_tokens.py +236 -0
  53. meerschaum/api/routes/_users.py +47 -35
  54. meerschaum/api/routes/_version.py +3 -3
  55. meerschaum/config/__init__.py +43 -20
  56. meerschaum/config/_default.py +32 -5
  57. meerschaum/config/_edit.py +28 -24
  58. meerschaum/config/_environment.py +1 -1
  59. meerschaum/config/_patch.py +6 -6
  60. meerschaum/config/_paths.py +5 -1
  61. meerschaum/config/_read_config.py +65 -34
  62. meerschaum/config/_sync.py +6 -3
  63. meerschaum/config/_version.py +1 -1
  64. meerschaum/config/stack/__init__.py +24 -5
  65. meerschaum/config/static.py +18 -0
  66. meerschaum/connectors/_Connector.py +10 -4
  67. meerschaum/connectors/__init__.py +4 -20
  68. meerschaum/connectors/api/_APIConnector.py +34 -6
  69. meerschaum/connectors/api/_actions.py +2 -2
  70. meerschaum/connectors/api/_jobs.py +1 -1
  71. meerschaum/connectors/api/_login.py +33 -7
  72. meerschaum/connectors/api/_misc.py +2 -2
  73. meerschaum/connectors/api/_pipes.py +15 -14
  74. meerschaum/connectors/api/_plugins.py +2 -2
  75. meerschaum/connectors/api/_request.py +1 -1
  76. meerschaum/connectors/api/_tokens.py +146 -0
  77. meerschaum/connectors/api/_users.py +70 -58
  78. meerschaum/connectors/instance/_InstanceConnector.py +83 -0
  79. meerschaum/connectors/instance/__init__.py +10 -0
  80. meerschaum/connectors/instance/_pipes.py +442 -0
  81. meerschaum/connectors/instance/_plugins.py +151 -0
  82. meerschaum/connectors/instance/_tokens.py +296 -0
  83. meerschaum/connectors/instance/_users.py +181 -0
  84. meerschaum/connectors/parse.py +4 -1
  85. meerschaum/connectors/sql/_SQLConnector.py +8 -5
  86. meerschaum/connectors/sql/_cli.py +12 -11
  87. meerschaum/connectors/sql/_create_engine.py +6 -154
  88. meerschaum/connectors/sql/_fetch.py +2 -18
  89. meerschaum/connectors/sql/_pipes.py +42 -31
  90. meerschaum/connectors/sql/_plugins.py +29 -0
  91. meerschaum/connectors/sql/_sql.py +8 -1
  92. meerschaum/connectors/sql/_users.py +29 -2
  93. meerschaum/connectors/sql/tables/__init__.py +1 -1
  94. meerschaum/connectors/valkey/_ValkeyConnector.py +2 -4
  95. meerschaum/connectors/valkey/_pipes.py +9 -10
  96. meerschaum/connectors/valkey/_plugins.py +2 -26
  97. meerschaum/core/Pipe/__init__.py +31 -14
  98. meerschaum/core/Pipe/_attributes.py +156 -58
  99. meerschaum/core/Pipe/_bootstrap.py +54 -24
  100. meerschaum/core/Pipe/_data.py +41 -1
  101. meerschaum/core/Pipe/_dtypes.py +29 -14
  102. meerschaum/core/Pipe/_edit.py +12 -4
  103. meerschaum/core/Pipe/_show.py +5 -5
  104. meerschaum/core/Pipe/_sync.py +48 -53
  105. meerschaum/core/Pipe/_verify.py +1 -1
  106. meerschaum/{plugins → core/Plugin}/_Plugin.py +9 -11
  107. meerschaum/core/Plugin/__init__.py +1 -1
  108. meerschaum/core/Token/_Token.py +221 -0
  109. meerschaum/core/Token/__init__.py +12 -0
  110. meerschaum/core/User/_User.py +34 -8
  111. meerschaum/core/User/__init__.py +9 -1
  112. meerschaum/core/__init__.py +1 -0
  113. meerschaum/jobs/_Job.py +3 -2
  114. meerschaum/jobs/__init__.py +3 -2
  115. meerschaum/jobs/systemd.py +1 -1
  116. meerschaum/models/__init__.py +35 -0
  117. meerschaum/models/pipes.py +247 -0
  118. meerschaum/models/tokens.py +38 -0
  119. meerschaum/models/users.py +26 -0
  120. meerschaum/plugins/__init__.py +22 -7
  121. meerschaum/plugins/bootstrap.py +2 -1
  122. meerschaum/utils/_get_pipes.py +68 -27
  123. meerschaum/utils/daemon/Daemon.py +2 -1
  124. meerschaum/utils/daemon/__init__.py +30 -2
  125. meerschaum/utils/dataframe.py +95 -14
  126. meerschaum/utils/dtypes/__init__.py +91 -18
  127. meerschaum/utils/dtypes/sql.py +44 -0
  128. meerschaum/utils/formatting/__init__.py +1 -1
  129. meerschaum/utils/formatting/_pipes.py +5 -4
  130. meerschaum/utils/formatting/_shell.py +11 -9
  131. meerschaum/utils/misc.py +237 -80
  132. meerschaum/utils/packages/__init__.py +3 -6
  133. meerschaum/utils/packages/_packages.py +34 -32
  134. meerschaum/utils/pipes.py +181 -0
  135. meerschaum/utils/process.py +1 -1
  136. meerschaum/utils/prompt.py +3 -1
  137. meerschaum/utils/schedule.py +1 -0
  138. meerschaum/utils/sql.py +114 -37
  139. meerschaum/utils/typing.py +1 -4
  140. meerschaum/utils/venv/_Venv.py +2 -2
  141. meerschaum/utils/venv/__init__.py +5 -7
  142. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc1.dist-info}/METADATA +88 -80
  143. meerschaum-3.0.0rc1.dist-info/RECORD +282 -0
  144. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc1.dist-info}/WHEEL +1 -1
  145. meerschaum/api/models/_interfaces.py +0 -15
  146. meerschaum/api/models/_locations.py +0 -15
  147. meerschaum/api/models/_metrics.py +0 -15
  148. meerschaum/config/static/__init__.py +0 -186
  149. meerschaum-2.9.5.dist-info/RECORD +0 -263
  150. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc1.dist-info}/entry_points.txt +0 -0
  151. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc1.dist-info}/licenses/LICENSE +0 -0
  152. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc1.dist-info}/top_level.txt +0 -0
  153. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc1.dist-info}/zip-safe +0 -0
@@ -14,6 +14,8 @@ import json
14
14
  from meerschaum.config._paths import (
15
15
  GRAFANA_DATASOURCE_PATH,
16
16
  GRAFANA_DASHBOARD_PATH,
17
+ DB_INIT_RESOURCES_PATH,
18
+ DB_CREATE_EXTENSIONS_PATH,
17
19
  ROOT_DIR_PATH,
18
20
  )
19
21
  from meerschaum.config._paths import STACK_COMPOSE_FILENAME, STACK_ENV_FILENAME
@@ -39,7 +41,7 @@ valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
39
41
 
40
42
  env_dict = {
41
43
  'COMPOSE_PROJECT_NAME': 'mrsm',
42
- 'TIMESCALEDB_VERSION': 'latest-pg16',
44
+ 'TIMESCALEDB_VERSION': 'pg17',
43
45
  'POSTGRES_USER': db_user,
44
46
  'POSTGRES_PASSWORD': db_pass,
45
47
  'POSTGRES_DB': db_base,
@@ -120,8 +122,9 @@ default_docker_compose_config = {
120
122
  'POSTGRES_DB': '<DOLLAR>POSTGRES_DB',
121
123
  'POSTGRES_PASSWORD': '<DOLLAR>POSTGRES_PASSWORD',
122
124
  'ALLOW_IP_RANGE': env_dict['ALLOW_IP_RANGE'],
125
+ 'POSTGRES_INITDB_ARGS': '-c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100'
123
126
  },
124
- 'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB',
127
+ # 'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB',
125
128
  'healthcheck': {
126
129
  'test': [
127
130
  'CMD-SHELL', 'pg_isready -d <DOLLAR>POSTGRES_DB -U <DOLLAR>POSTGRES_USER',
@@ -131,13 +134,14 @@ default_docker_compose_config = {
131
134
  'retries': 5
132
135
  },
133
136
  'restart': 'always',
134
- 'image': 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
137
+ 'image': 'timescale/timescaledb-ha:' + env_dict['TIMESCALEDB_VERSION'],
135
138
  'ports': [
136
139
  f'{db_port}:5432',
137
140
  ],
138
141
  'hostname': db_hostname,
139
142
  'volumes': [
140
143
  'meerschaum_db_data:' + volumes['meerschaum_db_data'],
144
+ f'{DB_INIT_RESOURCES_PATH.as_posix()}:/docker-entrypoint-initdb.d:z,ro',
141
145
  ],
142
146
  'shm_size': '1024m',
143
147
  'networks': [
@@ -224,8 +228,8 @@ default_docker_compose_config = {
224
228
  'volumes': [
225
229
  'grafana_storage' + ':' + volumes['grafana_storage'],
226
230
  ### NOTE: Mount with the 'z' option for SELinux.
227
- f'{GRAFANA_DATASOURCE_PATH.parent}:/etc/grafana/provisioning/datasources:z,ro',
228
- f'{GRAFANA_DASHBOARD_PATH.parent}:/etc/grafana/provisioning/dashboards:z,ro',
231
+ f'{GRAFANA_DATASOURCE_PATH.parent.as_posix()}:/etc/grafana/provisioning/datasources:z,ro',
232
+ f'{GRAFANA_DASHBOARD_PATH.parent.as_posix()}:/etc/grafana/provisioning/dashboards:z,ro',
229
233
  ],
230
234
  'environment': {
231
235
  'GF_SECURITY_ALLOW_EMBEDDING': 'true',
@@ -273,6 +277,21 @@ def _sync_stack_files():
273
277
  substitute = True,
274
278
  )
275
279
 
280
+ _write_initdb()
281
+
282
+ def _write_initdb():
283
+ create_postgis_text = (
284
+ "CREATE EXTENSION IF NOT EXISTS timescaledb;\n"
285
+ "CREATE EXTENSION IF NOT EXISTS postgis;\n"
286
+ "CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit;\n"
287
+ "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;\n"
288
+ )
289
+ if DB_CREATE_EXTENSIONS_PATH.exists():
290
+ return
291
+
292
+ with open(DB_CREATE_EXTENSIONS_PATH, 'w+', encoding='utf-8') as f:
293
+ f.write(create_postgis_text)
294
+
276
295
  NECESSARY_FILES = [STACK_COMPOSE_PATH, GRAFANA_DATASOURCE_PATH, GRAFANA_DASHBOARD_PATH]
277
296
  def get_necessary_files():
278
297
  from meerschaum.config import get_config
@@ -0,0 +1,18 @@
1
+ #! /usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ # vim:fenc=utf-8
4
+
5
+ """
6
+ Alias import for the internal static configuration dictionary.
7
+ """
8
+
9
+ from meerschaum._internal.static import SERVER_ID, STATIC_CONFIG
10
+
11
+ __all__ = ('STATIC_CONFIG',)
12
+
13
+
14
+ def _static_config():
15
+ """
16
+ Alias function for the global `STATIC_CONFIG` dictionary.
17
+ """
18
+ return STATIC_CONFIG
@@ -7,10 +7,12 @@ Define the parent `Connector` class.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
+
10
11
  import abc
11
12
  import copy
12
13
  from meerschaum.utils.typing import Iterable, Optional, Any, Union, List, Dict
13
14
 
15
+
14
16
  class InvalidAttributesError(Exception):
15
17
  """
16
18
  Raised when the incorrect attributes are set in the Connector.
@@ -20,6 +22,9 @@ class Connector(metaclass=abc.ABCMeta):
20
22
  """
21
23
  The base connector class to hold connection attributes.
22
24
  """
25
+
26
+ IS_INSTANCE: bool = False
27
+
23
28
  def __init__(
24
29
  self,
25
30
  type: Optional[str] = None,
@@ -70,7 +75,7 @@ class Connector(metaclass=abc.ABCMeta):
70
75
  inherit_default: bool = True,
71
76
  **kw: Any
72
77
  ):
73
- from meerschaum.config.static import STATIC_CONFIG
78
+ from meerschaum._internal.static import STATIC_CONFIG
74
79
  from meerschaum.utils.warnings import error
75
80
 
76
81
  self._attributes = {}
@@ -151,7 +156,7 @@ class Connector(metaclass=abc.ABCMeta):
151
156
  from meerschaum.utils.debug import dprint
152
157
  from meerschaum.utils.misc import items_str
153
158
  if required_attributes is None:
154
- required_attributes = ['label']
159
+ required_attributes = ['type', 'label']
155
160
 
156
161
  missing_attributes = set()
157
162
  for a in required_attributes:
@@ -213,6 +218,8 @@ class Connector(metaclass=abc.ABCMeta):
213
218
  else r'executor$'
214
219
  )
215
220
  _type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
221
+ if not _type or _type.lower() == 'instance':
222
+ raise ValueError("No type could be determined for this connector.")
216
223
  self.__dict__['type'] = _type
217
224
  return _type
218
225
 
@@ -224,8 +231,7 @@ class Connector(metaclass=abc.ABCMeta):
224
231
  """
225
232
  _label = self.__dict__.get('label', None)
226
233
  if _label is None:
227
- from meerschaum.config.static import STATIC_CONFIG
234
+ from meerschaum._internal.static import STATIC_CONFIG
228
235
  _label = STATIC_CONFIG['connectors']['default_label']
229
236
  self.__dict__['label'] = _label
230
237
  return _label
231
-
@@ -19,13 +19,14 @@ from meerschaum.utils.threading import RLock
19
19
  from meerschaum.utils.warnings import warn
20
20
 
21
21
  from meerschaum.connectors._Connector import Connector, InvalidAttributesError
22
+ from meerschaum.connectors.instance._InstanceConnector import InstanceConnector
22
23
  from meerschaum.connectors.sql._SQLConnector import SQLConnector
23
24
  from meerschaum.connectors.api._APIConnector import APIConnector
24
- from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
25
25
 
26
26
  __all__ = (
27
27
  "make_connector",
28
28
  "Connector",
29
+ "InstanceConnector",
29
30
  "SQLConnector",
30
31
  "APIConnector",
31
32
  "get_connector",
@@ -53,24 +54,7 @@ _locks: Dict[str, RLock] = {
53
54
  '_loaded_plugin_connectors': RLock(),
54
55
  'instance_types' : RLock(),
55
56
  }
56
- attributes: Dict[str, Dict[str, Any]] = {
57
- 'api': {
58
- 'required': [
59
- 'host',
60
- 'username',
61
- 'password',
62
- ],
63
- 'optional': [
64
- 'port',
65
- ],
66
- 'default': {
67
- 'protocol': 'http',
68
- },
69
- },
70
- 'sql': {
71
- 'flavors': sql_flavor_configs,
72
- },
73
- }
57
+
74
58
  ### Fill this with objects only when connectors are first referenced.
75
59
  types: Dict[str, Any] = {}
76
60
  custom_types: set = set()
@@ -130,7 +114,7 @@ def get_connector(
130
114
  """
131
115
  from meerschaum.connectors.parse import parse_instance_keys
132
116
  from meerschaum.config import get_config
133
- from meerschaum.config.static import STATIC_CONFIG
117
+ from meerschaum._internal.static import STATIC_CONFIG
134
118
  from meerschaum.utils.warnings import warn
135
119
  global _loaded_plugin_connectors
136
120
  if isinstance(type, str) and not label and ':' in type:
@@ -10,7 +10,7 @@ from __future__ import annotations
10
10
 
11
11
  from datetime import datetime, timedelta, timezone
12
12
  from meerschaum.utils.typing import Optional, List, Union
13
- from meerschaum.connectors import Connector
13
+ from meerschaum.connectors import InstanceConnector
14
14
  from meerschaum.utils.warnings import warn, error
15
15
  from meerschaum.utils.packages import attempt_import
16
16
 
@@ -18,15 +18,13 @@ required_attributes = {
18
18
  'host',
19
19
  }
20
20
 
21
- class APIConnector(Connector):
21
+ class APIConnector(InstanceConnector):
22
22
  """
23
23
  Connect to a Meerschaum API instance.
24
24
  """
25
25
 
26
- IS_INSTANCE: bool = True
27
26
  IS_THREAD_SAFE: bool = False
28
-
29
- OPTIONAL_ATTRIBUTES: List[str] = ['port']
27
+ OPTIONAL_ATTRIBUTES: List[str] = ['port', 'client_secret', 'client_id', 'api_key']
30
28
 
31
29
  from ._request import (
32
30
  make_request,
@@ -82,6 +80,16 @@ class APIConnector(Connector):
82
80
  get_user_type,
83
81
  get_user_attributes,
84
82
  )
83
+ from ._tokens import (
84
+ register_token,
85
+ get_token_model,
86
+ get_tokens,
87
+ edit_token,
88
+ invalidate_token,
89
+ get_token_scopes,
90
+ token_exists,
91
+ delete_token,
92
+ )
85
93
  from ._uri import from_uri
86
94
  from ._jobs import (
87
95
  get_jobs,
@@ -154,9 +162,15 @@ class APIConnector(Connector):
154
162
  """
155
163
  Return the fully qualified URI.
156
164
  """
165
+ import urllib.parse
157
166
  username = self.__dict__.get('username', None)
158
167
  password = self.__dict__.get('password', None)
168
+ client_id = self.__dict__.get('client_id', None)
169
+ client_secret = self.__dict__.get('client_secret', None)
170
+ api_key = self.__dict__.get('api_key', None)
159
171
  creds = (username + ':' + password + '@') if username and password else ''
172
+ params = {}
173
+ params_str = ('?' + urllib.parse.urlencode(params)) if params else ''
160
174
  return (
161
175
  self.protocol
162
176
  + '://'
@@ -167,9 +181,9 @@ class APIConnector(Connector):
167
181
  if self.__dict__.get('port', None)
168
182
  else ''
169
183
  )
184
+ + params_str
170
185
  )
171
186
 
172
-
173
187
  @property
174
188
  def session(self):
175
189
  if self._session is None:
@@ -206,3 +220,17 @@ class APIConnector(Connector):
206
220
  Return the instance keys to be sent alongside pipe requests.
207
221
  """
208
222
  return self._instance_keys
223
+
224
+ @property
225
+ def login_scheme(self) -> str:
226
+ """
227
+ Return the login scheme to use based on the configured credentials.
228
+ """
229
+ if 'username' in self.__dict__:
230
+ return 'password'
231
+ if 'client_id' in self.__dict__:
232
+ return 'client_credentials'
233
+ elif 'api_key' in self.__dict__:
234
+ return 'api_key'
235
+
236
+ raise ValueError(f"Could not determine the login scheme for '{self}'.")
@@ -14,7 +14,7 @@ from functools import partial
14
14
 
15
15
  import meerschaum as mrsm
16
16
  from meerschaum.utils.typing import SuccessTuple, List, Callable, Optional
17
- from meerschaum.config.static import STATIC_CONFIG
17
+ from meerschaum._internal.static import STATIC_CONFIG
18
18
 
19
19
  ACTIONS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['actions']
20
20
  TEMP_PREFIX: str = STATIC_CONFIG['api']['jobs']['temp_prefix']
@@ -88,7 +88,7 @@ def do_action_legacy(
88
88
  """
89
89
  import sys, json
90
90
  from meerschaum.utils.debug import dprint
91
- from meerschaum.config.static import STATIC_CONFIG
91
+ from meerschaum._internal.static import STATIC_CONFIG
92
92
  from meerschaum.utils.misc import json_serialize_datetime
93
93
  if action is None:
94
94
  action = []
@@ -14,7 +14,7 @@ from datetime import datetime
14
14
  import meerschaum as mrsm
15
15
  from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable, Optional
16
16
  from meerschaum.jobs import Job
17
- from meerschaum.config.static import STATIC_CONFIG
17
+ from meerschaum._internal.static import STATIC_CONFIG
18
18
  from meerschaum.utils.warnings import warn, dprint
19
19
 
20
20
  JOBS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['jobs']
@@ -11,7 +11,7 @@ from __future__ import annotations
11
11
  import json
12
12
  import datetime
13
13
  from meerschaum.utils.typing import SuccessTuple, Any, Union
14
- from meerschaum.config.static import STATIC_CONFIG
14
+ from meerschaum._internal.static import STATIC_CONFIG
15
15
  from meerschaum.utils.warnings import warn as _warn
16
16
 
17
17
 
@@ -22,14 +22,40 @@ def login(
22
22
  **kw: Any
23
23
  ) -> SuccessTuple:
24
24
  """Log in and set the session token."""
25
+ if self.login_scheme == 'api_key':
26
+ validate_response = self.post(
27
+ STATIC_CONFIG['api']['endpoints']['tokens'] + '/validate',
28
+ headers={'Authorization': f'Bearer {self.api_key}'},
29
+ use_token=False,
30
+ debug=debug,
31
+ )
32
+ if not validate_response:
33
+ return False, "API key is not valid."
34
+ return True, "API key is valid."
35
+
25
36
  try:
26
- login_data = {
27
- 'username': self.username,
28
- 'password': self.password,
29
- }
37
+ if self.login_scheme == 'password':
38
+ login_data = {
39
+ 'username': self.username,
40
+ 'password': self.password,
41
+ }
42
+ elif self.login_scheme == 'client_credentials':
43
+ login_data = {
44
+ 'client_id': self.client_id,
45
+ 'client_secret': self.client_secret,
46
+ }
30
47
  except AttributeError:
48
+ login_data = {}
49
+
50
+ if not login_data:
31
51
  return False, f"Please login with the command `login {self}`."
32
52
 
53
+ login_scheme_msg = (
54
+ f" as user '{login_data['username']}'"
55
+ if self.login_scheme == 'username'
56
+ else ''
57
+ )
58
+
33
59
  response = self.post(
34
60
  STATIC_CONFIG['api']['endpoints']['login'],
35
61
  data=login_data,
@@ -37,7 +63,7 @@ def login(
37
63
  debug=debug,
38
64
  )
39
65
  if response:
40
- msg = f"Successfully logged into '{self}' as user '{login_data['username']}'."
66
+ msg = f"Successfully logged into '{self}'{login_scheme_msg}'."
41
67
  self._token = json.loads(response.text)['access_token']
42
68
  self._expires = datetime.datetime.strptime(
43
69
  json.loads(response.text)['expires'],
@@ -45,7 +71,7 @@ def login(
45
71
  )
46
72
  else:
47
73
  msg = (
48
- f"Failed to log into '{self}' as user '{login_data['username']}'.\n" +
74
+ f"Failed to log into '{self}'{login_scheme_msg}.\n" +
49
75
  f" Please verify login details for connector '{self}'."
50
76
  )
51
77
  if warn and not self.__dict__.get('_emitted_warning', False):
@@ -13,7 +13,7 @@ def get_mrsm_version(self, **kw) -> Optional[str]:
13
13
  """
14
14
  Return the Meerschaum version of the API instance.
15
15
  """
16
- from meerschaum.config.static import STATIC_CONFIG
16
+ from meerschaum._internal.static import STATIC_CONFIG
17
17
  try:
18
18
  j = self.get(
19
19
  STATIC_CONFIG['api']['endpoints']['version'] + '/mrsm',
@@ -31,7 +31,7 @@ def get_chaining_status(self, **kw) -> Optional[bool]:
31
31
  """
32
32
  Fetch the chaining status of the API instance.
33
33
  """
34
- from meerschaum.config.static import STATIC_CONFIG
34
+ from meerschaum._internal.static import STATIC_CONFIG
35
35
  try:
36
36
  response = self.get(
37
37
  STATIC_CONFIG['api']['endpoints']['chaining'],
@@ -21,7 +21,7 @@ def pipe_r_url(
21
21
  pipe: mrsm.Pipe
22
22
  ) -> str:
23
23
  """Return a relative URL path from a Pipe's keys."""
24
- from meerschaum.config.static import STATIC_CONFIG
24
+ from meerschaum._internal.static import STATIC_CONFIG
25
25
  location_key = pipe.location_key
26
26
  if location_key is None:
27
27
  location_key = '[None]'
@@ -87,7 +87,7 @@ def edit_pipe(
87
87
  response = self.patch(
88
88
  r_url + '/edit',
89
89
  params={'patch': patch, 'instance_keys': self.get_pipe_instance_keys(pipe)},
90
- json=pipe.parameters,
90
+ json=pipe.get_parameters(apply_symlinks=False),
91
91
  debug=debug,
92
92
  )
93
93
  if debug:
@@ -142,7 +142,7 @@ def fetch_pipes_keys(
142
142
  -------
143
143
  A list of tuples containing pipes' keys.
144
144
  """
145
- from meerschaum.config.static import STATIC_CONFIG
145
+ from meerschaum._internal.static import STATIC_CONFIG
146
146
  if connector_keys is None:
147
147
  connector_keys = []
148
148
  if metric_keys is None:
@@ -167,6 +167,8 @@ def fetch_pipes_keys(
167
167
  debug=debug
168
168
  ).json()
169
169
  except Exception as e:
170
+ import traceback
171
+ traceback.print_exc()
170
172
  error(str(e))
171
173
 
172
174
  if 'detail' in j:
@@ -185,7 +187,8 @@ def sync_pipe(
185
187
  """Sync a DataFrame into a Pipe."""
186
188
  from decimal import Decimal
187
189
  from meerschaum.utils.debug import dprint
188
- from meerschaum.utils.misc import json_serialize_datetime, items_str, interval_str
190
+ from meerschaum.utils.dtypes import json_serialize_value
191
+ from meerschaum.utils.misc import items_str, interval_str
189
192
  from meerschaum.config import get_config
190
193
  from meerschaum.utils.packages import attempt_import
191
194
  from meerschaum.utils.dataframe import get_numeric_cols, to_json
@@ -197,8 +200,10 @@ def sync_pipe(
197
200
 
198
201
  def get_json_str(c):
199
202
  ### allow syncing dict or JSON without needing to import pandas (for IOT devices)
200
- if isinstance(c, (dict, list)):
201
- return json.dumps(c, default=json_serialize_datetime)
203
+ if isinstance(c, str):
204
+ return c
205
+ if isinstance(c, (dict, list, tuple)):
206
+ return json.dumps(c, default=json_serialize_value)
202
207
  return to_json(c, orient='columns')
203
208
 
204
209
  df = json.loads(df) if isinstance(df, str) else df
@@ -228,11 +233,7 @@ def sync_pipe(
228
233
  for col in numeric_cols
229
234
  if pipe_dtypes.get(col, None) != 'numeric'
230
235
  ]
231
- pipe.dtypes.update({
232
- col: 'numeric'
233
- for col in new_numeric_cols
234
- })
235
- edit_success, edit_msg = pipe.edit(debug=debug)
236
+ edit_success, edit_msg = pipe.update_parameters({'dtypes': {col: 'numeric' for col in new_numeric_cols}})
236
237
  if not edit_success:
237
238
  warn(
238
239
  "Failed to update new numeric columns "
@@ -313,7 +314,7 @@ def sync_pipe(
313
314
 
314
315
  success_tuple = True, (
315
316
  f"It took {interval_str(timedelta(seconds=(time.perf_counter() - begin)))} "
316
- + "to sync {rowcount:,} row"
317
+ + f"to sync {rowcount:,} row"
317
318
  + ('s' if rowcount != 1 else '')
318
319
  + f" across {num_success_chunks:,} chunk" + ('s' if num_success_chunks != 1 else '') +
319
320
  f" to {pipe}."
@@ -576,7 +577,7 @@ def create_metadata(
576
577
  A bool indicating success.
577
578
  """
578
579
  from meerschaum.utils.debug import dprint
579
- from meerschaum.config.static import STATIC_CONFIG
580
+ from meerschaum._internal.static import STATIC_CONFIG
580
581
  r_url = STATIC_CONFIG['api']['endpoints']['metadata']
581
582
  response = self.post(r_url, debug=debug)
582
583
  if debug:
@@ -663,7 +664,7 @@ def drop_pipe(
663
664
  from meerschaum.utils.warnings import error
664
665
  from meerschaum.utils.debug import dprint
665
666
  if pipe is None:
666
- error(f"Pipe cannot be None.")
667
+ error("Pipe cannot be None.")
667
668
  r_url = pipe_r_url(pipe)
668
669
  response = self.delete(
669
670
  r_url + '/drop',
@@ -16,7 +16,7 @@ def plugin_r_url(
16
16
  plugin: Union[mrsm.core.Plugin, str],
17
17
  ) -> str:
18
18
  """Generate a relative URL path from a Plugin."""
19
- from meerschaum.config.static import STATIC_CONFIG
19
+ from meerschaum._internal.static import STATIC_CONFIG
20
20
  return f"{STATIC_CONFIG['api']['endpoints']['plugins']}/{plugin}"
21
21
 
22
22
 
@@ -111,7 +111,7 @@ def get_plugins(
111
111
  """
112
112
  import json
113
113
  from meerschaum.utils.warnings import error
114
- from meerschaum.config.static import STATIC_CONFIG
114
+ from meerschaum._internal.static import STATIC_CONFIG
115
115
  response = self.get(
116
116
  STATIC_CONFIG['api']['endpoints']['plugins'],
117
117
  params = {'user_id': user_id, 'search_term': search_term},
@@ -11,7 +11,7 @@ import urllib.parse
11
11
  import pathlib
12
12
  from meerschaum.utils.typing import Any, Optional, Dict, Union
13
13
  from meerschaum.utils.debug import dprint
14
- from meerschaum.config.static import STATIC_CONFIG
14
+ from meerschaum._internal.static import STATIC_CONFIG
15
15
 
16
16
  METHODS = {
17
17
  'GET',
@@ -0,0 +1,146 @@
1
+ #! /usr/bin/env python3
2
+ # vim:fenc=utf-8
3
+
4
+ """
5
+ Implement the `APIConnector` token methods.
6
+ """
7
+
8
+ import json
9
+ import uuid
10
+ from datetime import datetime
11
+ from typing import Union, List, Optional
12
+
13
+ import meerschaum as mrsm
14
+ from meerschaum.core import Token
15
+ from meerschaum.models import TokenModel
16
+ from meerschaum._internal.static import STATIC_CONFIG
17
+ tokens_endpoint = STATIC_CONFIG['api']['endpoints']['tokens']
18
+
19
+
20
+ def register_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple:
21
+ """
22
+ Register the provided token to the API.
23
+ """
24
+ from meerschaum.utils.dtypes import json_serialize_value
25
+ r_url = tokens_endpoint + '/register'
26
+ response = self.post(
27
+ r_url,
28
+ data=json.dumps({
29
+ 'label': token.label,
30
+ 'scopes': token.scopes,
31
+ 'expiration': token.expiration,
32
+ }, default=json_serialize_value),
33
+ debug=debug,
34
+ )
35
+ if not response:
36
+ return False, f"Failed to register token:\n{response.text}"
37
+
38
+ data = response.json()
39
+ token.label = data['label']
40
+ token.secret = data['secret']
41
+ token.id = uuid.UUID(data['id'])
42
+ if data.get('expiration', None):
43
+ token.expiration = datetime.fromisoformat(data['expiration'])
44
+
45
+ return True, f"Registered token '{token.label}'."
46
+
47
+
48
+ def get_token_model(self, token_id: uuid.UUID, debug: bool = False) -> Union[TokenModel, None]:
49
+ """
50
+ Return a token's model from the API instance.
51
+ """
52
+ r_url = tokens_endpoint + f'/{token_id}'
53
+ response = self.get(r_url, debug=debug)
54
+ if not response:
55
+ return None
56
+ data = response.json()
57
+ return TokenModel(**data)
58
+
59
+
60
+ def get_tokens(self, labels: Optional[List[str]] = None, debug: bool = False) -> List[Token]:
61
+ """
62
+ Return the tokens registered to the current user.
63
+ """
64
+ from meerschaum.utils.warnings import warn
65
+ r_url = tokens_endpoint
66
+ params = {}
67
+ if labels:
68
+ params['labels'] = ','.join(labels)
69
+ response = self.get(r_url, params={'labels': labels}, debug=debug)
70
+ if not response:
71
+ warn(f"Could not get tokens from '{self}':\n{response.text}")
72
+ return []
73
+
74
+ tokens = [
75
+ Token(instance=self, **payload)
76
+ for payload in response.json()
77
+ ]
78
+ return tokens
79
+
80
+
81
+ def edit_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple:
82
+ """
83
+ Persist the token's in-memory state to the API.
84
+ """
85
+ r_url = tokens_endpoint + f"/{token.id}/edit"
86
+ response = self.post(
87
+ r_url,
88
+ json={
89
+ 'creation': token.creation.isoformat() if token.creation else None,
90
+ 'expiration': token.expiration.isoformat() if token.expiration else None,
91
+ 'label': token.label,
92
+ 'is_valid': token.is_valid,
93
+ 'scopes': token.scopes,
94
+ },
95
+ )
96
+ if not response:
97
+ return False, f"Failed to edit token:\n{response.text}"
98
+
99
+ success, msg = response.json()
100
+ return success, msg
101
+
102
+
103
+ def invalidate_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple:
104
+ """
105
+ Invalidate the token, disabling it for future requests.
106
+ """
107
+ r_url = tokens_endpoint + f"/{token.id}/invalidate"
108
+ response = self.post(r_url)
109
+ if not response:
110
+ return False, f"Failed to invalidate token:\n{response.text}"
111
+
112
+ success, msg = response.json()
113
+ return success, msg
114
+
115
+
116
+ def get_token_scopes(self, token_id: Union[uuid.UUID, Token], debug: bool = False) -> List[str]:
117
+ """
118
+ Return the scopes for a token.
119
+ """
120
+ _token_id = (token_id.id if isinstance(token_id, Token) else token_id)
121
+ model = self.get_token_model(_token_id, debug=debug).scopes
122
+ return getattr(model, 'scopes', [])
123
+
124
+
125
+ def token_exists(self, token_id: Union[uuid.UUID, Token], debug: bool = False) -> bool:
126
+ """
127
+ Return `True` if a token exists.
128
+ """
129
+ _token_id = (token_id.id if isinstance(token_id, Token) else token_id)
130
+ model = self.get_token_model(_token_id, debug=debug)
131
+ if model is None:
132
+ return False
133
+ return model.creation is not None
134
+
135
+
136
+ def delete_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple:
137
+ """
138
+ Delete the token from the API.
139
+ """
140
+ r_url = tokens_endpoint + f"/{token.id}"
141
+ response = self.delete(r_url, debug=debug)
142
+ if not response:
143
+ return False, f"Failed to delete token:\n{response.text}"
144
+
145
+ success, msg = response.json()
146
+ return success, msg