meerschaum 2.7.9__py3-none-any.whl → 2.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. meerschaum/_internal/arguments/_parser.py +17 -5
  2. meerschaum/_internal/term/TermPageHandler.py +1 -1
  3. meerschaum/_internal/term/__init__.py +1 -1
  4. meerschaum/actions/api.py +36 -10
  5. meerschaum/actions/copy.py +3 -1
  6. meerschaum/actions/index.py +1 -1
  7. meerschaum/actions/show.py +7 -7
  8. meerschaum/actions/sync.py +5 -1
  9. meerschaum/actions/verify.py +14 -1
  10. meerschaum/api/__init__.py +77 -41
  11. meerschaum/api/_exceptions.py +18 -0
  12. meerschaum/api/dash/__init__.py +4 -2
  13. meerschaum/api/dash/callbacks/dashboard.py +30 -1
  14. meerschaum/api/dash/components.py +2 -2
  15. meerschaum/api/dash/webterm.py +23 -4
  16. meerschaum/api/models/_pipes.py +8 -8
  17. meerschaum/api/resources/static/css/dash.css +2 -2
  18. meerschaum/api/resources/templates/termpage.html +5 -1
  19. meerschaum/api/routes/__init__.py +15 -12
  20. meerschaum/api/routes/_connectors.py +30 -28
  21. meerschaum/api/routes/_index.py +16 -7
  22. meerschaum/api/routes/_misc.py +30 -22
  23. meerschaum/api/routes/_pipes.py +244 -148
  24. meerschaum/api/routes/_plugins.py +58 -47
  25. meerschaum/api/routes/_users.py +39 -31
  26. meerschaum/api/routes/_version.py +8 -10
  27. meerschaum/api/routes/_webterm.py +2 -2
  28. meerschaum/config/_default.py +10 -0
  29. meerschaum/config/_version.py +1 -1
  30. meerschaum/config/static/__init__.py +5 -2
  31. meerschaum/connectors/api/_APIConnector.py +4 -3
  32. meerschaum/connectors/api/_login.py +21 -17
  33. meerschaum/connectors/api/_pipes.py +1 -0
  34. meerschaum/connectors/api/_request.py +9 -10
  35. meerschaum/connectors/sql/_cli.py +11 -3
  36. meerschaum/connectors/sql/_instance.py +1 -1
  37. meerschaum/connectors/sql/_pipes.py +77 -57
  38. meerschaum/connectors/sql/_sql.py +26 -9
  39. meerschaum/core/Pipe/__init__.py +2 -0
  40. meerschaum/core/Pipe/_attributes.py +13 -2
  41. meerschaum/core/Pipe/_data.py +85 -0
  42. meerschaum/core/Pipe/_deduplicate.py +6 -8
  43. meerschaum/core/Pipe/_sync.py +63 -30
  44. meerschaum/core/Pipe/_verify.py +242 -77
  45. meerschaum/core/User/__init__.py +2 -6
  46. meerschaum/jobs/_Job.py +1 -1
  47. meerschaum/jobs/__init__.py +15 -0
  48. meerschaum/utils/dataframe.py +2 -0
  49. meerschaum/utils/dtypes/sql.py +26 -0
  50. meerschaum/utils/formatting/_pipes.py +1 -1
  51. meerschaum/utils/misc.py +11 -7
  52. meerschaum/utils/packages/_packages.py +1 -1
  53. meerschaum/utils/sql.py +6 -2
  54. {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/METADATA +4 -4
  55. {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/RECORD +61 -60
  56. {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/LICENSE +0 -0
  57. {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/NOTICE +0 -0
  58. {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/WHEEL +0 -0
  59. {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/entry_points.txt +0 -0
  60. {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/top_level.txt +0 -0
  61. {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/zip-safe +0 -0
@@ -302,14 +302,17 @@ groups['sync'].add_argument(
302
302
  help="Run the action asynchronously, if possible. Alias for --unblock",
303
303
  )
304
304
  groups['sync'].add_argument(
305
- '--begin', type=parse_datetime, help="Specify a begin datetime for syncing or displaying data."
305
+ '--begin', type=parse_datetime, help="The begin datetime when syncing, fetching data."
306
306
  )
307
307
  groups['sync'].add_argument(
308
- '--end', type=parse_datetime, help="Specify an end datetime for syncing or displaying data."
308
+ '--end', type=parse_datetime, help="The end datetime when syncing, fetching data."
309
309
  )
310
310
  groups['sync'].add_argument(
311
- '--chunksize', type=int, help=(
312
- "Specify the database chunksize. Defaults to 100,000."
311
+ '--chunksize', type=int, help="How many rows per chunk. Defaults to 100,000."
312
+ )
313
+ groups['sync'].add_argument(
314
+ '--batchsize', type=int, help=(
315
+ "How many chunks to process in parallel. Defaults to number of CPUs."
313
316
  ),
314
317
  )
315
318
  groups['sync'].add_argument(
@@ -355,9 +358,18 @@ groups['sync'].add_argument(
355
358
  "This improves performance when all rows are expected to already be of the correct type."
356
359
  ),
357
360
  )
361
+ groups['sync'].add_argument(
362
+ '--skip-chunks-with-greater-rowcounts', action='store_true',
363
+ help="When verifying, skip chunks with rowcounts greater than the remote's."
364
+ )
365
+ groups['sync'].add_argument(
366
+ '--check-rowcounts-only', action='store_true', help=(
367
+ "Only compare row-counts when verifying pipes."
368
+ ),
369
+ )
358
370
  groups['sync'].add_argument(
359
371
  '--cache', action='store_true',
360
- help = (
372
+ help=(
361
373
  "When syncing or viewing a pipe's data, sync to a local database for later analysis."
362
374
  )
363
375
  )
@@ -32,7 +32,7 @@ class TermPageHandler(tornado_web.RequestHandler):
32
32
  return self.render(
33
33
  "termpage.html",
34
34
  static=self.static_url,
35
- ws_url_path=f"/_websocket/{term_name}",
35
+ ws_url_path=f"/websocket/{term_name}",
36
36
  )
37
37
 
38
38
 
@@ -51,7 +51,7 @@ def get_webterm_app_and_manager(
51
51
  term_manager = terminado.NamedTermManager(shell_command=commands)
52
52
  handlers = [
53
53
  (
54
- r"/_websocket/(.+)/?",
54
+ r"/websocket/(.+)/?",
55
55
  CustomTermSocket,
56
56
  {'term_manager': term_manager}
57
57
  ),
meerschaum/actions/api.py CHANGED
@@ -140,16 +140,28 @@ def _api_start(
140
140
  If provided, serve over HTTPS with this certfile.
141
141
  Requires `--keyfile`.
142
142
  """
143
+ import json
144
+ import sys
145
+ import shutil
146
+ import pathlib
147
+ from copy import deepcopy
148
+
143
149
  from meerschaum.utils.packages import (
144
- attempt_import, venv_contains_package, pip_install, run_python_package
150
+ attempt_import,
151
+ venv_contains_package,
152
+ pip_install,
153
+ run_python_package,
145
154
  )
146
155
  from meerschaum.utils.misc import is_int, filter_keywords
156
+ from meerschaum.utils.dtypes import json_serialize_value
147
157
  from meerschaum.utils.formatting import pprint, ANSI, _init
148
158
  from meerschaum.utils.debug import dprint
149
159
  from meerschaum.utils.warnings import error, warn
150
160
  from meerschaum.config import get_config, _config
151
161
  from meerschaum.config._paths import (
152
- API_UVICORN_RESOURCES_PATH, API_UVICORN_CONFIG_PATH, CACHE_RESOURCES_PATH,
162
+ API_UVICORN_RESOURCES_PATH,
163
+ API_UVICORN_CONFIG_PATH,
164
+ CACHE_RESOURCES_PATH,
153
165
  PACKAGE_ROOT_PATH,
154
166
  )
155
167
  from meerschaum.config._patch import apply_patch_to_config
@@ -157,8 +169,6 @@ def _api_start(
157
169
  from meerschaum.config.static import STATIC_CONFIG, SERVER_ID
158
170
  from meerschaum.connectors.parse import parse_instance_keys
159
171
  from meerschaum.utils.pool import get_pool
160
- import shutil
161
- from copy import deepcopy
162
172
 
163
173
  if action is None:
164
174
  action = []
@@ -256,7 +266,6 @@ def _api_start(
256
266
  custom_keys = ['mrsm_instance', 'no_dash', 'no_auth', 'private', 'debug', 'production']
257
267
 
258
268
  ### write config to a temporary file to communicate with uvicorn threads
259
- import json, sys
260
269
  try:
261
270
  if uvicorn_config_path.exists():
262
271
  os.remove(uvicorn_config_path)
@@ -275,12 +284,25 @@ def _api_start(
275
284
  MRSM_RUNTIME = STATIC_CONFIG['environment']['runtime']
276
285
  MRSM_PATCH = STATIC_CONFIG['environment']['patch']
277
286
  MRSM_ROOT_DIR = STATIC_CONFIG['environment']['root']
278
- env_dict = {
287
+ env_dict = {}
288
+ env_dict.update({
279
289
  MRSM_SERVER_ID: SERVER_ID,
280
290
  MRSM_RUNTIME: 'api',
281
291
  MRSM_CONFIG: json.loads(os.environ.get(MRSM_CONFIG, '{}')),
282
292
  'FORWARDED_ALLOW_IPS': forwarded_allow_ips,
283
- }
293
+ 'TERM': os.environ.get('TERM', 'screen-256color'),
294
+ 'SHELL': os.environ.get('SHELL', '/bin/bash'),
295
+ 'LANG': os.environ.get('LANG', 'C.UTF-8'),
296
+ 'HOME': os.environ.get('HOME', pathlib.Path.home().as_posix()),
297
+ 'PATH': os.environ.get(
298
+ 'PATH',
299
+ (
300
+ '/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin:'
301
+ f'{pathlib.Path.home().as_posix().rstrip("/")}/.local/bin'
302
+ )
303
+ ),
304
+ 'HOSTNAME': os.environ.get('HOSTNAME', 'api'),
305
+ })
284
306
  for env_var in get_env_vars():
285
307
  if env_var in env_dict:
286
308
  continue
@@ -294,10 +316,10 @@ def _api_start(
294
316
  env_text = ''
295
317
  for key, val in env_dict.items():
296
318
  value = str(
297
- json.dumps(val)
319
+ json.dumps(val, default=json_serialize_value)
298
320
  if isinstance(val, (dict))
299
321
  else val
300
- ).replace('\\', '\\\\')
322
+ ).replace('\\', '\\\\').replace("'", "'\\''")
301
323
  env_text += f"{key}='{value}'\n"
302
324
  with open(uvicorn_env_path, 'w+', encoding='utf-8') as f:
303
325
  if debug:
@@ -329,7 +351,11 @@ def _api_start(
329
351
  for key, val in env_dict.items():
330
352
  gunicorn_args += [
331
353
  '--env', key + "="
332
- + (json.dumps(val) if isinstance(val, (dict, list)) else val)
354
+ + (
355
+ json.dumps(val, default=json_serialize_value)
356
+ if isinstance(val, (dict, list))
357
+ else val
358
+ )
333
359
  ]
334
360
  if workers is not None:
335
361
  gunicorn_args += ['--workers', str(workers)]
@@ -70,6 +70,7 @@ def _copy_pipes(
70
70
  Copy pipes' attributes and make new pipes.
71
71
  """
72
72
  from meerschaum import get_pipes, Pipe
73
+ from meerschaum.connectors import instance_types
73
74
  from meerschaum.utils.prompt import prompt, yes_no, get_connectors_completer
74
75
  from meerschaum.utils.warnings import warn
75
76
  from meerschaum.utils.formatting import print_tuple
@@ -92,7 +93,8 @@ def _copy_pipes(
92
93
 
93
94
  instance_keys = prompt(
94
95
  f"Meerschaum instance for copy of {pipe}:",
95
- default=pipe.instance_keys
96
+ default=pipe.instance_keys,
97
+ completer=get_connectors_completer(*instance_types),
96
98
  )
97
99
  new_pipe = Pipe(
98
100
  ck, mk, lk,
@@ -48,7 +48,7 @@ def _index_pipes(
48
48
 
49
49
  for pipe in pipes:
50
50
  info(f"Creating indices for {pipe}...")
51
- index_success, index_msg = pipe.create_indices(debug=debug)
51
+ index_success, index_msg = pipe.create_indices(columns=(action or None), debug=debug)
52
52
  success_dict[pipe] = index_msg
53
53
  if index_success:
54
54
  successes += 1
@@ -442,7 +442,7 @@ def _show_rowcounts(
442
442
 
443
443
  msgs = []
444
444
  for p, rc in rc_dict.items():
445
- msgs.append(f'{p}\n{rc}\n')
445
+ msgs.append(f'{p}\n{rc:,}\n')
446
446
 
447
447
  header = "Remote row-counts:" if remote else "Pipe row-counts:"
448
448
 
@@ -581,10 +581,10 @@ def _show_jobs(
581
581
  from meerschaum.utils.warnings import info
582
582
  info('No running or stopped jobs.')
583
583
  print(
584
- f" You can start a background job with `-d` or `--daemon`,\n" +
585
- " or run the command `start job` before action commands.\n\n" +
586
- " Examples:\n" +
587
- " - start api -d\n" +
584
+ " You can start a background job with `-d` or `--daemon`,\n"
585
+ " or run the command `start job` before action commands.\n\n"
586
+ " Examples:\n"
587
+ " - start api -d\n"
588
588
  " - start job sync pipes --loop"
589
589
  )
590
590
  return True, "No jobs to show."
@@ -610,7 +610,7 @@ def _show_logs(
610
610
  `show logs --nopretty`
611
611
  `show logs myjob myotherjob`
612
612
  """
613
- import os, pathlib, random, asyncio
613
+ import asyncio
614
614
  from functools import partial
615
615
  from datetime import datetime, timezone
616
616
  from meerschaum.utils.packages import attempt_import, import_rich
@@ -676,7 +676,7 @@ def _show_logs(
676
676
  try:
677
677
  line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
678
678
  previous_line_timestamp = line_timestamp
679
- except Exception as e:
679
+ except Exception:
680
680
  line_timestamp = None
681
681
  if line_timestamp:
682
682
  line = line[(len(now_str) + 3):]
@@ -49,6 +49,7 @@ def _pipes_lap(
49
49
  deduplicate: bool = False,
50
50
  bounded: Optional[bool] = None,
51
51
  chunk_interval: Union[timedelta, int, None] = None,
52
+ check_rowcounts_only: bool = False,
52
53
  mrsm_instance: Optional[str] = None,
53
54
  timeout_seconds: Optional[int] = None,
54
55
  nopretty: bool = False,
@@ -93,6 +94,7 @@ def _pipes_lap(
93
94
  'deduplicate': deduplicate,
94
95
  'bounded': bounded,
95
96
  'chunk_interval': chunk_interval,
97
+ 'check_rowcounts_only': check_rowcounts_only,
96
98
  })
97
99
  locks = {'remaining_count': Lock(), 'results_dict': Lock(), 'pipes_threads': Lock(),}
98
100
  pipes = get_pipes(
@@ -254,6 +256,7 @@ def _sync_pipes(
254
256
  deduplicate: bool = False,
255
257
  bounded: Optional[bool] = None,
256
258
  chunk_interval: Union[timedelta, int, None] = None,
259
+ check_rowcounts_only: bool = False,
257
260
  shell: bool = False,
258
261
  nopretty: bool = False,
259
262
  debug: bool = False,
@@ -308,6 +311,7 @@ def _sync_pipes(
308
311
  deduplicate=deduplicate,
309
312
  bounded=bounded,
310
313
  chunk_interval=chunk_interval,
314
+ check_rowcounts_only=check_rowcounts_only,
311
315
  unblock=unblock,
312
316
  debug=debug,
313
317
  nopretty=nopretty,
@@ -447,7 +451,7 @@ def _wrap_pipe(
447
451
  sync_hook_result = sync_hook(pipe, **filter_keywords(sync_hook, **sync_kwargs))
448
452
  if is_success_tuple(sync_hook_result):
449
453
  return sync_hook_result
450
- except Exception as e:
454
+ except Exception:
451
455
  msg = (
452
456
  f"Failed to execute sync hook '{sync_hook.__name__}' "
453
457
  + f"from plugin '{plugin}':\n{traceback.format_exc()}"
@@ -7,7 +7,9 @@ Verify the states of pipes, pacakages, and more.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional, Tuple, List
10
+
11
+ from meerschaum.utils.typing import Any, SuccessTuple, Optional, List
12
+
11
13
 
12
14
  def verify(
13
15
  action: Optional[List[str]] = None,
@@ -22,6 +24,7 @@ def verify(
22
24
  'packages': _verify_packages,
23
25
  'venvs': _verify_venvs,
24
26
  'plugins': _verify_plugins,
27
+ 'rowcounts': _verify_rowcounts,
25
28
  }
26
29
  return choose_subaction(action, options, **kwargs)
27
30
 
@@ -35,6 +38,16 @@ def _verify_pipes(**kwargs) -> SuccessTuple:
35
38
  return _sync_pipes(**kwargs)
36
39
 
37
40
 
41
+ def _verify_rowcounts(**kwargs) -> SuccessTuple:
42
+ """
43
+ Verify the contents of pipes, syncing across their entire datetime intervals.
44
+ """
45
+ from meerschaum.actions.sync import _sync_pipes
46
+ kwargs['verify'] = True
47
+ kwargs['check_rowcounts_only'] = True
48
+ return _sync_pipes(**kwargs)
49
+
50
+
38
51
  def _verify_packages(
39
52
  debug: bool = False,
40
53
  venv: Optional[str] = 'mrsm',
@@ -6,15 +6,12 @@
6
6
  Meerschaum API backend. Start an API instance with `start api`.
7
7
  """
8
8
  from __future__ import annotations
9
- import os
10
- from meerschaum.utils.typing import Dict, Any, Optional
11
9
 
12
- from meerschaum import __version__ as version
13
- __version__ = version
14
- __doc__ = """
15
- The Meerschaum Web API lets you access and control your data over the Internet.
16
- """
10
+ import os
11
+ from collections import defaultdict
17
12
 
13
+ import meerschaum as mrsm
14
+ from meerschaum.utils.typing import Dict, Any, Optional, PipesDict
18
15
  from meerschaum.config import get_config
19
16
  from meerschaum.config.static import STATIC_CONFIG, SERVER_ID
20
17
  from meerschaum.utils.packages import attempt_import
@@ -23,8 +20,14 @@ from meerschaum.config._paths import API_UVICORN_CONFIG_PATH, API_UVICORN_RESOUR
23
20
  from meerschaum.plugins import _api_plugins
24
21
  from meerschaum.utils.warnings import warn, dprint
25
22
  from meerschaum.utils.threading import RLock
23
+ from meerschaum.utils.misc import is_pipe_registered
24
+
25
+ from meerschaum import __version__ as version
26
+ __version__ = version
27
+ __doc__ = """The Meerschaum Web API lets you manage your pipes over the Internet."""
28
+
26
29
 
27
- _locks = {'pipes': RLock(), 'connector': RLock(), 'uvicorn_config': RLock()}
30
+ _locks = defaultdict(lambda: RLock())
28
31
 
29
32
  ### Skip verifying packages in the docker image.
30
33
  CHECK_UPDATE = os.environ.get(STATIC_CONFIG['environment']['runtime'], None) != 'docker'
@@ -58,6 +61,7 @@ uv = attempt_import('uv', lazy=False, check_update=CHECK_UPDATE)
58
61
  venv=None,
59
62
  )
60
63
  from meerschaum.api._chain import check_allow_chaining, DISALLOW_CHAINING_MESSAGE
64
+ from meerschaum.api._exceptions import APIPermissionError
61
65
  uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
62
66
 
63
67
  uvicorn_config = None
@@ -78,13 +82,11 @@ def get_uvicorn_config() -> Dict[str, Any]:
78
82
  with open(uvicorn_config_path, 'r', encoding='utf-8') as f:
79
83
  uvicorn_config = json.load(f)
80
84
  _uvicorn_config = uvicorn_config
81
- except Exception as e:
85
+ except Exception:
82
86
  _uvicorn_config = sys_config.get('uvicorn', None)
83
87
 
84
88
  if _uvicorn_config is None:
85
89
  _uvicorn_config = {}
86
-
87
- ### Default: main SQL connector
88
90
  if 'mrsm_instance' not in _uvicorn_config:
89
91
  _uvicorn_config['mrsm_instance'] = get_config('meerschaum', 'api_instance')
90
92
  return _uvicorn_config
@@ -95,22 +97,47 @@ no_auth = get_uvicorn_config().get('no_auth', False)
95
97
  private = get_uvicorn_config().get('private', False)
96
98
  production = get_uvicorn_config().get('production', False)
97
99
  _include_dash = (not no_dash)
100
+ docs_enabled = not production or sys_config.get('endpoints', {}).get('docs_in_production', True)
98
101
 
99
102
  connector = None
103
+ default_instance_keys = None
104
+ _instance_connectors = defaultdict(lambda: None)
100
105
  def get_api_connector(instance_keys: Optional[str] = None):
101
- """Create the instance connector."""
102
- global connector
103
- with _locks['connector']:
104
- if connector is None:
105
- if instance_keys is None:
106
- instance_keys = get_uvicorn_config().get('mrsm_instance', None)
106
+ """Create the instance connectors."""
107
+ global default_instance_keys
108
+ if instance_keys is None:
109
+ if default_instance_keys is None:
110
+ default_instance_keys = get_uvicorn_config().get('mrsm_instance', None)
111
+ instance_keys = default_instance_keys
112
+
113
+ allow_multiple_instances = permissions_config.get(
114
+ 'instances', {}
115
+ ).get('allow_multiple_instances', False)
116
+ if not allow_multiple_instances and instance_keys != default_instance_keys:
117
+ raise APIPermissionError(
118
+ "This API instance does not allow for accessing additional instances."
119
+ )
120
+
121
+ allowed_instance_keys = permissions_config.get(
122
+ 'instance', {}
123
+ ).get(
124
+ 'allowed_instance_keys',
125
+ ['*']
126
+ )
127
+ if allowed_instance_keys != ['*'] and instance_keys not in allowed_instance_keys:
128
+ raise APIPermissionError(
129
+ f"Instance keys '{instance_keys}' not in list of allowed instances."
130
+ )
107
131
 
132
+ with _locks[f'instance-{instance_keys}']:
133
+ connector = _instance_connectors[instance_keys]
134
+ if connector is None:
108
135
  from meerschaum.connectors.parse import parse_instance_keys
109
136
  connector = parse_instance_keys(instance_keys, debug=debug)
110
- if debug:
111
- dprint(f"API instance connector: {connector}")
137
+ _instance_connectors[instance_keys] = connector
112
138
  return connector
113
139
 
140
+
114
141
  cache_connector = None
115
142
  def get_cache_connector(connector_keys: Optional[str] = None):
116
143
  """Return the `valkey` connector if running in production."""
@@ -146,28 +173,35 @@ def get_cache_connector(connector_keys: Optional[str] = None):
146
173
  return cache_connector
147
174
 
148
175
 
149
- _pipes = None
150
- def pipes(refresh=False):
176
+ _instance_pipes = defaultdict(lambda: None)
177
+ def pipes(instance_keys: Optional[str] = None, refresh: bool = False) -> PipesDict:
151
178
  """
152
- Manage the global pipes dictionary.
179
+ Manage the global pipes dictionaries.
153
180
  """
154
- global _pipes
155
- with _locks['pipes']:
156
- if _pipes is None or refresh:
157
- _pipes = _get_pipes(mrsm_instance=get_api_connector())
158
- return _pipes
159
-
160
-
161
- def get_pipe(connector_keys, metric_key, location_key, refresh=False):
181
+ instance_keys = str(get_api_connector(instance_keys))
182
+ with _locks['pipes-' + instance_keys]:
183
+ pipes = _instance_pipes[instance_keys]
184
+ if pipes is None or refresh:
185
+ pipes = _get_pipes(mrsm_instance=instance_keys)
186
+ _instance_pipes[instance_keys] = pipes
187
+ return pipes
188
+
189
+
190
+ def get_pipe(
191
+ connector_keys: str,
192
+ metric_key: str,
193
+ location_key: Optional[str],
194
+ instance_keys: Optional[str] = None,
195
+ refresh: bool = False
196
+ ) -> mrsm.Pipe:
162
197
  """Index the pipes dictionary or create a new Pipe object."""
163
- from meerschaum.utils.misc import is_pipe_registered
164
- from meerschaum import Pipe
165
198
  if location_key in ('[None]', 'None', 'null'):
166
199
  location_key = None
167
- p = Pipe(connector_keys, metric_key, location_key, mrsm_instance=get_api_connector())
168
- if is_pipe_registered(p, pipes()):
169
- return pipes(refresh=refresh)[connector_keys][metric_key][location_key]
170
- return p
200
+ instance_keys = str(get_api_connector(instance_keys))
201
+ pipe = mrsm.Pipe(connector_keys, metric_key, location_key, mrsm_instance=instance_keys)
202
+ if is_pipe_registered(pipe, pipes(instance_keys)):
203
+ return pipes(instance_keys, refresh=refresh)[connector_keys][metric_key][location_key]
204
+ return pipe
171
205
 
172
206
 
173
207
  app = fastapi.FastAPI(
@@ -182,6 +216,9 @@ app = fastapi.FastAPI(
182
216
  'name': 'Apache 2.0',
183
217
  'url': 'https://www.apache.org/licenses/LICENSE-2.0.html',
184
218
  },
219
+ docs_url=(None if not docs_enabled else endpoints['docs']),
220
+ redoc_url=(None if not docs_enabled else endpoints['redoc']),
221
+ openapi_url=endpoints['openapi'],
185
222
  open_api_tags=[
186
223
  {
187
224
  'name': 'Pipes',
@@ -193,7 +230,7 @@ app = fastapi.FastAPI(
193
230
  },
194
231
  {
195
232
  'name': 'Connectors',
196
- 'description': 'Get information about the registered connectors.'
233
+ 'description': 'Get information about the registered connectors.',
197
234
  },
198
235
  {
199
236
  'name': 'Users',
@@ -209,8 +246,8 @@ app = fastapi.FastAPI(
209
246
  },
210
247
  {
211
248
  'name': 'Version',
212
- 'description': 'Version information.'
213
- }
249
+ 'description': 'Version information.',
250
+ },
214
251
  ],
215
252
  )
216
253
 
@@ -228,7 +265,7 @@ app = fastapi.FastAPI(
228
265
  HTMLResponse = fastapi_responses.HTMLResponse
229
266
  Request = fastapi.Request
230
267
 
231
- from meerschaum.config._paths import API_RESOURCES_PATH, API_STATIC_PATH, API_TEMPLATES_PATH
268
+ from meerschaum.config.paths import API_RESOURCES_PATH, API_STATIC_PATH, API_TEMPLATES_PATH
232
269
  app.mount('/static', fastapi_staticfiles.StaticFiles(directory=str(API_STATIC_PATH)), name='static')
233
270
 
234
271
  _custom_kwargs = {'mrsm_instance'}
@@ -252,7 +289,6 @@ if _include_dash:
252
289
  import meerschaum.api.dash
253
290
 
254
291
  ### Execute the API plugins functions.
255
- import meerschaum as mrsm
256
292
  for module_name, functions_list in _api_plugins.items():
257
293
  plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
258
294
  plugin = mrsm.Plugin(plugin_name) if plugin_name else None
@@ -0,0 +1,18 @@
1
+ #! /usr/bin/env python3
2
+ # vim:fenc=utf-8
3
+
4
+ """
5
+ Define custom API exceptions.
6
+ """
7
+
8
+ import meerschaum as mrsm
9
+
10
+ _ = mrsm.attempt_import('fastapi', lazy=False)
11
+ from fastapi import HTTPException
12
+
13
+
14
+ class APIPermissionError(HTTPException):
15
+ """Raise if the configured Meerschaum API permissions disallow an action."""
16
+
17
+ def __init__(self, detail: str = "Permission denied.", status_code: int = 403):
18
+ super().__init__(status_code=status_code, detail=detail)
@@ -7,10 +7,12 @@ Build the Dash app to be hooked into FastAPI.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- import uuid
11
10
 
12
11
  from meerschaum.utils.packages import (
13
- attempt_import, import_dcc, import_html, _monkey_patch_get_distribution
12
+ attempt_import,
13
+ import_dcc,
14
+ import_html,
15
+ _monkey_patch_get_distribution,
14
16
  )
15
17
  flask_compress = attempt_import('flask_compress', lazy=False)
16
18
  _monkey_patch_get_distribution('flask-compress', flask_compress.__version__)
@@ -665,7 +665,8 @@ dash_app.clientside_callback(
665
665
 
666
666
  iframe.contentWindow.postMessage(
667
667
  {
668
- action: "__TMUX_NEW_WINDOW"
668
+ action: "__TMUX_NEW_WINDOW",
669
+ instance: window.instance
669
670
  },
670
671
  url
671
672
  );
@@ -692,6 +693,34 @@ dash_app.clientside_callback(
692
693
  State('mrsm-location', 'href'),
693
694
  )
694
695
 
696
+ dash_app.clientside_callback(
697
+ """
698
+ function(n_clicks){
699
+ console.log('fullscreen');
700
+ if (!n_clicks) { return dash_clientside.no_update; }
701
+ iframe = document.getElementById('webterm-iframe');
702
+ if (!iframe){ return dash_clientside.no_update; }
703
+ const leftCol = document.getElementById('content-col-left');
704
+ const rightCol = document.getElementById('content-col-right');
705
+ const button = document.getElementById('webterm-fullscreen-button');
706
+
707
+ if (leftCol.style.display === 'none') {
708
+ leftCol.style.display = '';
709
+ rightCol.className = 'col-6';
710
+ button.innerHTML = "Full View";
711
+ } else {
712
+ leftCol.style.display = 'none';
713
+ rightCol.className = 'col-12';
714
+ button.innerHTML = "Side-by-side View";
715
+ }
716
+
717
+ return dash_clientside.no_update;
718
+ }
719
+ """,
720
+ Output('webterm-fullscreen-button', 'n_clicks'),
721
+ Input('webterm-fullscreen-button', 'n_clicks'),
722
+ )
723
+
695
724
  @dash_app.callback(
696
725
  Output(component_id='connector-keys-input', component_property='value'),
697
726
  Input(component_id='clear-connector-keys-input-button', component_property='n_clicks'),
@@ -13,7 +13,7 @@ from meerschaum.utils.typing import SuccessTuple, List
13
13
  from meerschaum.config.static import STATIC_CONFIG
14
14
  from meerschaum.utils.misc import remove_ansi
15
15
  from meerschaum._internal.shell.Shell import get_shell_intro
16
- from meerschaum.api import endpoints, CHECK_UPDATE
16
+ from meerschaum.api import endpoints, CHECK_UPDATE, docs_enabled
17
17
  from meerschaum.connectors import instance_types, _load_builtin_custom_connectors
18
18
  from meerschaum.utils.misc import get_connector_labels
19
19
  from meerschaum.config import __doc__ as doc
@@ -127,7 +127,7 @@ navbar = dbc.Navbar(
127
127
  align='center',
128
128
  className='g-0 navbar-logo-row',
129
129
  ),
130
- href='/docs',
130
+ href=('/docs' if docs_enabled else '#'),
131
131
  style={"textDecoration": "none"},
132
132
  ),
133
133
  dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
@@ -7,6 +7,8 @@ Functions for interacting with the Webterm via the dashboard.
7
7
  """
8
8
 
9
9
  import time
10
+
11
+ import meerschaum as mrsm
10
12
  from meerschaum.api import CHECK_UPDATE, get_api_connector
11
13
  from meerschaum.api.dash.sessions import is_session_authenticated, get_username_from_session
12
14
  from meerschaum.api.dash.components import alert_from_success_tuple, console_div
@@ -19,7 +21,9 @@ dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHEC
19
21
  dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
20
22
 
21
23
  MAX_WEBTERM_ATTEMPTS: int = 10
22
- TMUX_IS_AVAILABLE: bool = is_tmux_available()
24
+ TMUX_IS_ENABLED: bool = (
25
+ is_tmux_available() and mrsm.get_config('system', 'webterm', 'tmux', 'enabled')
26
+ )
23
27
 
24
28
  _locks = {'webterm_thread': RLock()}
25
29
 
@@ -48,10 +52,25 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
48
52
  [
49
53
  html.Div(
50
54
  [
51
- dbc.Button('Refresh', color='black', id='webterm-refresh-button'),
52
- dbc.Button('New Tab', color='black', id='webterm-new-tab-button'),
53
- ],
55
+ dbc.Button(
56
+ 'Refresh',
57
+ color='black',
58
+ id='webterm-refresh-button',
59
+ ),
60
+ dbc.Button(
61
+ 'Full View',
62
+ color='black',
63
+ id='webterm-fullscreen-button',
64
+ ),
65
+ ] + [
66
+ dbc.Button(
67
+ 'New Tab',
68
+ color='black',
69
+ id='webterm-new-tab-button',
70
+ ),
71
+ ] if TMUX_IS_ENABLED else [],
54
72
  id='webterm-controls-div',
73
+ style={'text-align': 'right'},
55
74
  ),
56
75
  html.Iframe(
57
76
  src=f"/webterm/{session_id}",