meerschaum 2.9.5__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. meerschaum/__init__.py +5 -2
  2. meerschaum/_internal/__init__.py +1 -0
  3. meerschaum/_internal/arguments/_parse_arguments.py +4 -4
  4. meerschaum/_internal/arguments/_parser.py +33 -4
  5. meerschaum/_internal/cli/__init__.py +6 -0
  6. meerschaum/_internal/cli/daemons.py +103 -0
  7. meerschaum/_internal/cli/entry.py +220 -0
  8. meerschaum/_internal/cli/workers.py +435 -0
  9. meerschaum/_internal/docs/index.py +48 -2
  10. meerschaum/_internal/entry.py +50 -14
  11. meerschaum/_internal/shell/Shell.py +121 -29
  12. meerschaum/_internal/shell/__init__.py +4 -1
  13. meerschaum/_internal/static.py +359 -0
  14. meerschaum/_internal/term/TermPageHandler.py +1 -2
  15. meerschaum/_internal/term/__init__.py +40 -6
  16. meerschaum/_internal/term/tools.py +33 -8
  17. meerschaum/actions/__init__.py +6 -4
  18. meerschaum/actions/api.py +53 -13
  19. meerschaum/actions/attach.py +1 -0
  20. meerschaum/actions/bootstrap.py +8 -8
  21. meerschaum/actions/delete.py +4 -2
  22. meerschaum/actions/edit.py +171 -25
  23. meerschaum/actions/login.py +8 -8
  24. meerschaum/actions/register.py +143 -6
  25. meerschaum/actions/reload.py +22 -5
  26. meerschaum/actions/restart.py +14 -0
  27. meerschaum/actions/show.py +184 -31
  28. meerschaum/actions/start.py +166 -17
  29. meerschaum/actions/stop.py +38 -2
  30. meerschaum/actions/sync.py +7 -2
  31. meerschaum/actions/tag.py +9 -8
  32. meerschaum/actions/verify.py +5 -8
  33. meerschaum/api/__init__.py +45 -15
  34. meerschaum/api/_events.py +46 -4
  35. meerschaum/api/_oauth2.py +162 -9
  36. meerschaum/api/_tokens.py +102 -0
  37. meerschaum/api/dash/__init__.py +0 -3
  38. meerschaum/api/dash/callbacks/__init__.py +1 -0
  39. meerschaum/api/dash/callbacks/custom.py +4 -3
  40. meerschaum/api/dash/callbacks/dashboard.py +198 -118
  41. meerschaum/api/dash/callbacks/jobs.py +14 -7
  42. meerschaum/api/dash/callbacks/login.py +10 -1
  43. meerschaum/api/dash/callbacks/pipes.py +194 -14
  44. meerschaum/api/dash/callbacks/plugins.py +0 -1
  45. meerschaum/api/dash/callbacks/register.py +10 -3
  46. meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
  47. meerschaum/api/dash/callbacks/tokens.py +389 -0
  48. meerschaum/api/dash/components.py +36 -15
  49. meerschaum/api/dash/jobs.py +1 -1
  50. meerschaum/api/dash/keys.py +35 -93
  51. meerschaum/api/dash/pages/__init__.py +2 -1
  52. meerschaum/api/dash/pages/dashboard.py +1 -20
  53. meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
  54. meerschaum/api/dash/pages/login.py +2 -2
  55. meerschaum/api/dash/pages/pipes.py +16 -5
  56. meerschaum/api/dash/pages/settings/password_reset.py +1 -1
  57. meerschaum/api/dash/pages/tokens.py +53 -0
  58. meerschaum/api/dash/pipes.py +382 -95
  59. meerschaum/api/dash/sessions.py +12 -0
  60. meerschaum/api/dash/tokens.py +603 -0
  61. meerschaum/api/dash/websockets.py +1 -1
  62. meerschaum/api/dash/webterm.py +18 -6
  63. meerschaum/api/models/__init__.py +23 -3
  64. meerschaum/api/models/_actions.py +22 -0
  65. meerschaum/api/models/_pipes.py +91 -7
  66. meerschaum/api/models/_tokens.py +81 -0
  67. meerschaum/api/resources/static/js/terminado.js +3 -0
  68. meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
  69. meerschaum/api/resources/templates/termpage.html +13 -0
  70. meerschaum/api/routes/__init__.py +1 -0
  71. meerschaum/api/routes/_actions.py +3 -4
  72. meerschaum/api/routes/_connectors.py +3 -7
  73. meerschaum/api/routes/_jobs.py +26 -35
  74. meerschaum/api/routes/_login.py +120 -15
  75. meerschaum/api/routes/_misc.py +5 -10
  76. meerschaum/api/routes/_pipes.py +178 -143
  77. meerschaum/api/routes/_plugins.py +38 -28
  78. meerschaum/api/routes/_tokens.py +236 -0
  79. meerschaum/api/routes/_users.py +47 -35
  80. meerschaum/api/routes/_version.py +3 -3
  81. meerschaum/api/routes/_webterm.py +3 -3
  82. meerschaum/config/__init__.py +100 -30
  83. meerschaum/config/_default.py +132 -64
  84. meerschaum/config/_edit.py +38 -32
  85. meerschaum/config/_formatting.py +2 -0
  86. meerschaum/config/_patch.py +10 -8
  87. meerschaum/config/_paths.py +133 -13
  88. meerschaum/config/_read_config.py +87 -36
  89. meerschaum/config/_sync.py +6 -3
  90. meerschaum/config/_version.py +1 -1
  91. meerschaum/config/environment.py +262 -0
  92. meerschaum/config/stack/__init__.py +37 -15
  93. meerschaum/config/static.py +18 -0
  94. meerschaum/connectors/_Connector.py +11 -6
  95. meerschaum/connectors/__init__.py +41 -22
  96. meerschaum/connectors/api/_APIConnector.py +34 -6
  97. meerschaum/connectors/api/_actions.py +2 -2
  98. meerschaum/connectors/api/_jobs.py +12 -1
  99. meerschaum/connectors/api/_login.py +33 -7
  100. meerschaum/connectors/api/_misc.py +2 -2
  101. meerschaum/connectors/api/_pipes.py +23 -32
  102. meerschaum/connectors/api/_plugins.py +2 -2
  103. meerschaum/connectors/api/_request.py +1 -1
  104. meerschaum/connectors/api/_tokens.py +146 -0
  105. meerschaum/connectors/api/_users.py +70 -58
  106. meerschaum/connectors/instance/_InstanceConnector.py +83 -0
  107. meerschaum/connectors/instance/__init__.py +10 -0
  108. meerschaum/connectors/instance/_pipes.py +442 -0
  109. meerschaum/connectors/instance/_plugins.py +159 -0
  110. meerschaum/connectors/instance/_tokens.py +317 -0
  111. meerschaum/connectors/instance/_users.py +188 -0
  112. meerschaum/connectors/parse.py +5 -2
  113. meerschaum/connectors/sql/_SQLConnector.py +22 -5
  114. meerschaum/connectors/sql/_cli.py +12 -11
  115. meerschaum/connectors/sql/_create_engine.py +12 -168
  116. meerschaum/connectors/sql/_fetch.py +2 -18
  117. meerschaum/connectors/sql/_pipes.py +295 -278
  118. meerschaum/connectors/sql/_plugins.py +29 -0
  119. meerschaum/connectors/sql/_sql.py +46 -21
  120. meerschaum/connectors/sql/_users.py +36 -2
  121. meerschaum/connectors/sql/tables/__init__.py +254 -122
  122. meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
  123. meerschaum/connectors/valkey/_pipes.py +60 -31
  124. meerschaum/connectors/valkey/_plugins.py +2 -26
  125. meerschaum/core/Pipe/__init__.py +115 -85
  126. meerschaum/core/Pipe/_attributes.py +425 -124
  127. meerschaum/core/Pipe/_bootstrap.py +54 -24
  128. meerschaum/core/Pipe/_cache.py +555 -0
  129. meerschaum/core/Pipe/_clear.py +0 -11
  130. meerschaum/core/Pipe/_data.py +96 -68
  131. meerschaum/core/Pipe/_deduplicate.py +0 -13
  132. meerschaum/core/Pipe/_delete.py +12 -21
  133. meerschaum/core/Pipe/_drop.py +11 -23
  134. meerschaum/core/Pipe/_dtypes.py +49 -19
  135. meerschaum/core/Pipe/_edit.py +14 -4
  136. meerschaum/core/Pipe/_fetch.py +1 -1
  137. meerschaum/core/Pipe/_index.py +8 -14
  138. meerschaum/core/Pipe/_show.py +5 -5
  139. meerschaum/core/Pipe/_sync.py +123 -204
  140. meerschaum/core/Pipe/_verify.py +4 -4
  141. meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
  142. meerschaum/core/Plugin/__init__.py +1 -1
  143. meerschaum/core/Token/_Token.py +220 -0
  144. meerschaum/core/Token/__init__.py +12 -0
  145. meerschaum/core/User/_User.py +35 -10
  146. meerschaum/core/User/__init__.py +9 -1
  147. meerschaum/core/__init__.py +1 -0
  148. meerschaum/jobs/_Executor.py +88 -4
  149. meerschaum/jobs/_Job.py +149 -38
  150. meerschaum/jobs/__init__.py +3 -2
  151. meerschaum/jobs/systemd.py +8 -3
  152. meerschaum/models/__init__.py +35 -0
  153. meerschaum/models/pipes.py +247 -0
  154. meerschaum/models/tokens.py +38 -0
  155. meerschaum/models/users.py +26 -0
  156. meerschaum/plugins/__init__.py +301 -88
  157. meerschaum/plugins/bootstrap.py +510 -4
  158. meerschaum/utils/_get_pipes.py +97 -30
  159. meerschaum/utils/daemon/Daemon.py +199 -43
  160. meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
  161. meerschaum/utils/daemon/RotatingFile.py +63 -36
  162. meerschaum/utils/daemon/StdinFile.py +53 -13
  163. meerschaum/utils/daemon/__init__.py +47 -6
  164. meerschaum/utils/daemon/_names.py +6 -3
  165. meerschaum/utils/dataframe.py +479 -81
  166. meerschaum/utils/debug.py +49 -19
  167. meerschaum/utils/dtypes/__init__.py +476 -34
  168. meerschaum/utils/dtypes/sql.py +369 -29
  169. meerschaum/utils/formatting/__init__.py +5 -2
  170. meerschaum/utils/formatting/_jobs.py +1 -1
  171. meerschaum/utils/formatting/_pipes.py +52 -50
  172. meerschaum/utils/formatting/_pprint.py +1 -0
  173. meerschaum/utils/formatting/_shell.py +44 -18
  174. meerschaum/utils/misc.py +268 -186
  175. meerschaum/utils/packages/__init__.py +25 -40
  176. meerschaum/utils/packages/_packages.py +42 -34
  177. meerschaum/utils/pipes.py +213 -0
  178. meerschaum/utils/process.py +2 -2
  179. meerschaum/utils/prompt.py +175 -144
  180. meerschaum/utils/schedule.py +2 -1
  181. meerschaum/utils/sql.py +134 -47
  182. meerschaum/utils/threading.py +42 -0
  183. meerschaum/utils/typing.py +1 -4
  184. meerschaum/utils/venv/_Venv.py +2 -2
  185. meerschaum/utils/venv/__init__.py +7 -7
  186. meerschaum/utils/warnings.py +19 -13
  187. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
  188. meerschaum-3.0.0.dist-info/RECORD +289 -0
  189. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
  190. meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
  191. meerschaum/api/models/_interfaces.py +0 -15
  192. meerschaum/api/models/_locations.py +0 -15
  193. meerschaum/api/models/_metrics.py +0 -15
  194. meerschaum/config/_environment.py +0 -145
  195. meerschaum/config/static/__init__.py +0 -186
  196. meerschaum-2.9.5.dist-info/RECORD +0 -263
  197. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
  198. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
  199. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
  200. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
@@ -11,10 +11,11 @@ from __future__ import annotations
11
11
  import textwrap
12
12
  import json
13
13
  import uuid
14
- from datetime import datetime, timezone
15
14
 
16
15
  from dash.dependencies import Input, Output, State, ALL, MATCH
17
16
  from dash.exceptions import PreventUpdate
17
+
18
+ import meerschaum as mrsm
18
19
  from meerschaum.utils.typing import List, Optional, Any, Tuple
19
20
  from meerschaum.api import get_api_connector, endpoints, no_auth, CHECK_UPDATE
20
21
  from meerschaum.api.dash import dash_app, debug
@@ -26,7 +27,12 @@ from meerschaum.api.dash.sessions import (
26
27
  from meerschaum.api.dash.sessions import is_session_authenticated
27
28
  from meerschaum.api.dash.connectors import get_web_connector
28
29
  from meerschaum.connectors.parse import parse_instance_keys
29
- from meerschaum.api.dash.pipes import get_pipes_cards, pipe_from_ctx, accordion_items_from_pipe
30
+ from meerschaum.api.dash.pipes import (
31
+ get_pipes_cards,
32
+ pipe_from_ctx,
33
+ accordion_items_from_pipe,
34
+ get_backtrack_text,
35
+ )
30
36
  from meerschaum.api.dash.jobs import get_jobs_cards
31
37
  from meerschaum.api.dash.plugins import get_plugins_cards
32
38
  from meerschaum.api.dash.users import get_users_cards
@@ -41,7 +47,7 @@ from meerschaum.api.dash.components import (
41
47
  from meerschaum.api.dash import pages
42
48
  from meerschaum.utils.typing import Dict
43
49
  from meerschaum.utils.packages import attempt_import, import_html, import_dcc
44
- from meerschaum.utils.misc import filter_keywords, flatten_list
50
+ from meerschaum.utils.misc import filter_keywords, flatten_list, string_to_dict
45
51
  from meerschaum.utils.yaml import yaml
46
52
  from meerschaum.actions import get_subactions, actions
47
53
  from meerschaum._internal.arguments._parser import parser
@@ -55,11 +61,8 @@ keys_state = (
55
61
  State('connector-keys-dropdown', 'value'),
56
62
  State('metric-keys-dropdown', 'value'),
57
63
  State('location-keys-dropdown', 'value'),
58
- State('connector-keys-input', 'value'),
59
- State('metric-keys-input', 'value'),
60
- State('location-keys-input', 'value'),
61
- State('search-parameters-editor', 'value'),
62
- State('pipes-filter-tabs', 'active_tab'),
64
+ State('tags-dropdown', 'value'),
65
+ State('tags-dropdown-div', 'style'),
63
66
  State('action-dropdown', 'value'),
64
67
  State('subaction-dropdown', 'value'),
65
68
  State('subaction-dropdown', 'options'),
@@ -69,8 +72,6 @@ keys_state = (
69
72
  State({'type': 'input-flags-dropdown', 'index': ALL}, 'value'),
70
73
  State({'type': 'input-flags-dropdown-text', 'index': ALL}, 'value'),
71
74
  State('instance-select', 'value'),
72
- State('content-div-right', 'children'),
73
- State('success-alert-div', 'children'),
74
75
  State('session-store', 'data'),
75
76
  )
76
77
 
@@ -98,17 +99,21 @@ omit_actions = {
98
99
 
99
100
  ### Map endpoints to page layouts.
100
101
  _paths = {
101
- 'login' : pages.login.layout,
102
- '' : pages.dashboard.layout,
103
- 'plugins' : pages.plugins.layout,
104
- 'register': pages.register.layout,
105
- 'pipes' : pages.pipes.layout,
106
- 'job' : pages.job.layout,
102
+ '/dash/login' : pages.login.layout,
103
+ '/dash' : pages.dashboard.layout,
104
+ '/dash/plugins' : pages.plugins.layout,
105
+ '/dash/tokens' : pages.tokens.layout,
106
+ '/dash/register': pages.register.layout,
107
+ '/dash/pipes' : pages.pipes.layout,
108
+ '/dash/jobs' : pages.jobs.layout,
107
109
  }
108
- _required_login = {''}
110
+ _required_login = {'', '/dash', '/dash/', '/dash/tokens', '/dash/jobs', '/dash/pipes'}
109
111
  _pages = {
110
112
  'Web Console': '/dash/',
113
+ 'Pipes': '/dash/pipes',
111
114
  'Plugins': '/dash/plugins',
115
+ 'Tokens': '/dash/tokens',
116
+ 'Jobs': '/dash/jobs',
112
117
  }
113
118
 
114
119
 
@@ -139,7 +144,6 @@ def update_page_layout_div(
139
144
  -------
140
145
  A tuple of the page layout and new session store data.
141
146
  """
142
- dash_endpoint = endpoints['dash']
143
147
  try:
144
148
  session_id = session_store_data.get('session-id', None)
145
149
  except AttributeError:
@@ -156,18 +160,9 @@ def update_page_layout_div(
156
160
  else:
157
161
  session_store_to_return = dash.no_update
158
162
 
159
- base_path = (
160
- pathname.rstrip('/') + '/'
161
- ).replace(
162
- (dash_endpoint + '/'),
163
- ''
164
- ).rstrip('/').split('/')[0]
165
-
163
+ base_path = '/'.join(pathname.split('/')[:2])
166
164
  complete_path = (
167
165
  pathname.rstrip('/') + '/'
168
- ).replace(
169
- dash_endpoint + '/',
170
- ''
171
166
  ).rstrip('/')
172
167
 
173
168
  if complete_path in _paths:
@@ -175,14 +170,14 @@ def update_page_layout_div(
175
170
  elif base_path in _paths:
176
171
  path_str = base_path
177
172
  else:
178
- path_str = ''
173
+ path_str = '/dash'
179
174
 
180
175
  path = (
181
176
  path_str
182
177
  if no_auth or path_str not in _required_login else (
183
178
  path_str
184
179
  if is_session_active(session_id)
185
- else 'login'
180
+ else '/dash/login'
186
181
  )
187
182
  )
188
183
  layout = _paths.get(path, pages.error.layout)
@@ -295,6 +290,7 @@ dash_app.clientside_callback(
295
290
  connector_keys,
296
291
  metric_keys,
297
292
  location_keys,
293
+ tags,
298
294
  flags,
299
295
  input_flags,
300
296
  input_flags_texts,
@@ -317,6 +313,7 @@ dash_app.clientside_callback(
317
313
  connector_keys: connector_keys,
318
314
  metric_keys: metric_keys,
319
315
  location_keys: location_keys,
316
+ tags: tags,
320
317
  flags: flags,
321
318
  input_flags: input_flags,
322
319
  input_flags_texts: input_flags_texts,
@@ -333,6 +330,7 @@ dash_app.clientside_callback(
333
330
  State('connector-keys-dropdown', 'value'),
334
331
  State('metric-keys-dropdown', 'value'),
335
332
  State('location-keys-dropdown', 'value'),
333
+ State('tags-dropdown', 'value'),
336
334
  State('flags-dropdown', 'value'),
337
335
  State({'type': 'input-flags-dropdown', 'index': ALL}, 'value'),
338
336
  State({'type': 'input-flags-dropdown-text', 'index': ALL}, 'value'),
@@ -358,7 +356,7 @@ def update_actions(action: str, subaction: str):
358
356
  _actions_options = sorted([
359
357
  {
360
358
  'label': a.replace('_', ' '),
361
- 'value': a,
359
+ 'value': a.replace('_', ' '),
362
360
  'title': (textwrap.dedent(f.__doc__).lstrip() if f.__doc__ else 'No help available.'),
363
361
  }
364
362
  for a, f in actions.items() if a not in omit_actions
@@ -366,7 +364,7 @@ def update_actions(action: str, subaction: str):
366
364
  _subactions_options = sorted([
367
365
  {
368
366
  'label': sa.replace('_', ' '),
369
- 'value': sa,
367
+ 'value': sa.replace('_', ' '),
370
368
  'title': (textwrap.dedent(f.__doc__).lstrip() if f.__doc__ else 'No help available.'),
371
369
  }
372
370
  for sa, f in get_subactions(action).items()
@@ -489,19 +487,15 @@ def update_flags(input_flags_dropdown_values, n_clicks, input_flags_texts):
489
487
 
490
488
  @dash_app.callback(
491
489
  Output('connector-keys-dropdown', 'options'),
492
- Output('connector-keys-list', 'children'),
493
- Output('connector-keys-dropdown', 'value'),
494
490
  Output('metric-keys-dropdown', 'options'),
495
- Output('metric-keys-list', 'children'),
496
- Output('metric-keys-dropdown', 'value'),
497
491
  Output('location-keys-dropdown', 'options'),
498
- Output('location-keys-list', 'children'),
499
- Output('location-keys-dropdown', 'value'),
492
+ Output('tags-dropdown', 'options'),
500
493
  Output('instance-select', 'value'),
501
494
  Output('instance-alert-div', 'children'),
502
495
  Input('connector-keys-dropdown', 'value'),
503
496
  Input('metric-keys-dropdown', 'value'),
504
497
  Input('location-keys-dropdown', 'value'),
498
+ Input('tags-dropdown', 'value'),
505
499
  Input('instance-select', 'value'),
506
500
  *keys_state ### NOTE: Necessary for `ctx.states`.
507
501
  )
@@ -509,6 +503,7 @@ def update_keys_options(
509
503
  connector_keys: Optional[List[str]],
510
504
  metric_keys: Optional[List[str]],
511
505
  location_keys: Optional[List[str]],
506
+ tags: Optional[List[str]],
512
507
  instance_keys: Optional[str],
513
508
  *keys
514
509
  ):
@@ -536,13 +531,6 @@ def update_keys_options(
536
531
  except Exception as e:
537
532
  instance_alerts += [alert_from_success_tuple((False, str(e)))]
538
533
 
539
- ### Update the keys filters.
540
- if connector_keys is None:
541
- connector_keys = []
542
- if metric_keys is None:
543
- metric_keys = []
544
- if location_keys is None:
545
- location_keys = []
546
534
  num_filter = 0
547
535
  if connector_keys:
548
536
  num_filter += 1
@@ -550,13 +538,13 @@ def update_keys_options(
550
538
  num_filter += 1
551
539
  if location_keys:
552
540
  num_filter += 1
541
+ if tags:
542
+ num_filter += 1
553
543
 
554
- _ck_filter = connector_keys
555
- _mk_filter = metric_keys
556
- _lk_filter = location_keys
557
544
  _ck_alone = (connector_keys and num_filter == 1) or instance_click
558
545
  _mk_alone = (metric_keys and num_filter == 1) or instance_click
559
546
  _lk_alone = (location_keys and num_filter == 1) or instance_click
547
+ _tags_alone = (tags and num_filter == 1) or instance_click
560
548
 
561
549
  from meerschaum.utils import fetch_pipes_keys
562
550
 
@@ -565,74 +553,95 @@ def update_keys_options(
565
553
  _keys = fetch_pipes_keys(
566
554
  'registered',
567
555
  get_web_connector(ctx.states),
568
- connector_keys=_ck_filter,
569
- metric_keys=_mk_filter,
570
- location_keys=_lk_filter,
556
+ connector_keys=connector_keys,
557
+ metric_keys=metric_keys,
558
+ location_keys=location_keys,
559
+ tags=tags,
560
+ )
561
+ _tags_pipes = mrsm.get_pipes(
562
+ connector_keys=connector_keys,
563
+ metric_keys=metric_keys,
564
+ location_keys=location_keys,
565
+ tags=tags,
566
+ instance=get_web_connector(ctx.states),
567
+ as_tags_dict=True,
571
568
  )
569
+ _all_tags = list(
570
+ set(
571
+ mrsm.get_pipes(
572
+ instance=get_web_connector(ctx.states),
573
+ as_tags_dict=True,
574
+ )
575
+ ).union(tags or [])
576
+ ) if _tags_alone else []
572
577
  except Exception as e:
573
578
  instance_alerts += [alert_from_success_tuple((False, str(e)))]
574
- _all_keys, _keys = [], []
575
- _connectors_options = []
576
- _metrics_options = []
577
- _locations_options = []
578
-
579
- _seen_keys = {'ck' : set(), 'mk' : set(), 'lk' : set()}
580
-
581
- def add_options(options, keys, key_type):
582
- for ck, mk, lk in keys:
583
- k = locals()[key_type]
584
- if k not in _seen_keys[key_type]:
585
- _k = 'None' if k in (None, '[None]', 'None', 'null') else k
586
- options.append({'label': _k, 'value': _k})
587
- _seen_keys[key_type].add(k)
588
-
589
- add_options(_connectors_options, _all_keys if _ck_alone else _keys, 'ck')
590
- add_options(_metrics_options, _all_keys if _mk_alone else _keys, 'mk')
591
- add_options(_locations_options, _all_keys if _lk_alone else _keys, 'lk')
592
- _connectors_options.sort(key=lambda x: str(x).lower())
593
- _metrics_options.sort(key=lambda x: str(x).lower())
594
- _locations_options.sort(key=lambda x: str(x).lower())
595
- connector_keys = [
596
- ck
597
- for ck in connector_keys
598
- if ck in [
599
- _ck['value']
600
- for _ck in _connectors_options
601
- ]
602
- ]
603
- metric_keys = [
604
- mk
605
- for mk in metric_keys
606
- if mk in [
607
- _mk['value']
608
- for _mk in _metrics_options
609
- ]
610
- ]
611
- location_keys = [
612
- lk
613
- for lk in location_keys
614
- if lk in [
615
- _lk['value']
616
- for _lk in _locations_options
617
- ]
618
- ]
619
- _connectors_datalist = [html.Option(value=o['value']) for o in _connectors_options]
620
- _metrics_datalist = [html.Option(value=o['value']) for o in _metrics_options]
621
- _locations_datalist = [html.Option(value=o['value']) for o in _locations_options]
579
+ _all_keys, _all_tags, _keys = [], [], []
580
+
581
+ connectors_options = sorted(
582
+ list(
583
+ set(
584
+ keys_tuple[0] for keys_tuple in (_all_keys if _ck_alone else _keys)
585
+ ).union(set(connector_keys or []))
586
+ ),
587
+ key=(lambda x: str(x).lower()),
588
+ )
589
+ metrics_options = sorted(
590
+ list(
591
+ set(
592
+ keys_tuple[1] for keys_tuple in (_all_keys if _mk_alone else _keys)
593
+ ).union(set(metric_keys or []))
594
+ ),
595
+ key=(lambda x: str(x).lower()),
596
+ )
597
+ locations_options = sorted(
598
+ list(
599
+ set(
600
+ (
601
+ str(keys_tuple[2])
602
+ for keys_tuple in (_all_keys if _lk_alone else _keys)
603
+ )
604
+ ).union(set((str(_lk) for _lk in (location_keys or []))))
605
+ ),
606
+ key=(lambda x: str(x).lower()),
607
+ )
608
+
609
+ tags_options = sorted(
610
+ list(
611
+ set(
612
+ (_all_tags if _tags_alone else _tags_pipes)
613
+ ).union(set(tags or []))
614
+ ),
615
+ key=(lambda x: str(x).lower()),
616
+ )
617
+
622
618
  return (
623
- _connectors_options,
624
- _connectors_datalist,
625
- connector_keys,
626
- _metrics_options,
627
- _metrics_datalist,
628
- metric_keys,
629
- _locations_options,
630
- _locations_datalist,
631
- location_keys,
619
+ connectors_options,
620
+ metrics_options,
621
+ locations_options,
622
+ tags_options,
632
623
  (instance_keys if update_instance_keys else dash.no_update),
633
624
  instance_alerts,
634
625
  )
635
626
 
627
+
628
+ @dash_app.callback(
629
+ Output('connector-keys-dropdown', 'value'),
630
+ Output('metric-keys-dropdown', 'value'),
631
+ Output('location-keys-dropdown', 'value'),
632
+ Output('tags-dropdown', 'value'),
633
+ Input('clear-all-keys-button', 'n_clicks'),
634
+ prevent_initial_call=True,
635
+ )
636
+ def clear_all_keys_button_click(n_clicks):
637
+ """
638
+ Clear the keys dropdowns when the `Clear all` button is clicked.
639
+ """
640
+ if not n_clicks:
641
+ raise PreventUpdate
642
+
643
+ return [], [], [], []
644
+
636
645
  dash_app.clientside_callback(
637
646
  """
638
647
  function(
@@ -804,6 +813,7 @@ dash_app.clientside_callback(
804
813
  @dash_app.callback(
805
814
  Output("download-dataframe-csv", "data"),
806
815
  Input({'type': 'pipe-download-csv-button', 'index': ALL}, 'n_clicks'),
816
+ prevent_initial_call=True,
807
817
  )
808
818
  def download_pipe_csv(n_clicks):
809
819
  """
@@ -833,6 +843,7 @@ def download_pipe_csv(n_clicks):
833
843
  Output({'type': 'pipe-accordion', 'index': MATCH}, 'children'),
834
844
  Input({'type': 'pipe-accordion', 'index': MATCH}, 'active_item'),
835
845
  State('session-store', 'data'),
846
+ prevent_initial_call=True,
836
847
  )
837
848
  def update_pipe_accordion(item, session_store_data):
838
849
  """
@@ -856,12 +867,12 @@ def update_pipe_accordion(item, session_store_data):
856
867
  @dash_app.callback(
857
868
  Output({'type': 'update-parameters-success-div', 'index': MATCH}, 'children'),
858
869
  Input({'type': 'update-parameters-button', 'index': MATCH}, 'n_clicks'),
859
- State({'type': 'parameters-editor', 'index': MATCH}, 'value')
870
+ State({'type': 'parameters-editor', 'index': MATCH}, 'value'),
871
+ prevent_initial_call=True,
860
872
  )
861
873
  def update_pipe_parameters_click(n_clicks, parameters_editor_text):
862
874
  if not n_clicks:
863
875
  raise PreventUpdate
864
- ctx = dash.callback_context
865
876
  triggered = dash.callback_context.triggered
866
877
  if triggered[0]['value'] is None:
867
878
  raise PreventUpdate
@@ -890,12 +901,12 @@ def update_pipe_parameters_click(n_clicks, parameters_editor_text):
890
901
  @dash_app.callback(
891
902
  Output({'type': 'update-sql-success-div', 'index': MATCH}, 'children'),
892
903
  Input({'type': 'update-sql-button', 'index': MATCH}, 'n_clicks'),
893
- State({'type': 'sql-editor', 'index': MATCH}, 'value')
904
+ State({'type': 'sql-editor', 'index': MATCH}, 'value'),
905
+ prevent_initial_call=True,
894
906
  )
895
907
  def update_pipe_sql_click(n_clicks, sql_editor_text):
896
908
  if not n_clicks:
897
909
  raise PreventUpdate
898
- ctx = dash.callback_context
899
910
  triggered = dash.callback_context.triggered
900
911
  if triggered[0]['value'] is None:
901
912
  raise PreventUpdate
@@ -918,12 +929,12 @@ def update_pipe_sql_click(n_clicks, sql_editor_text):
918
929
  @dash_app.callback(
919
930
  Output({'type': 'sync-success-div', 'index': MATCH}, 'children'),
920
931
  Input({'type': 'update-sync-button', 'index': MATCH}, 'n_clicks'),
921
- State({'type': 'sync-editor', 'index': MATCH}, 'value')
932
+ State({'type': 'sync-editor', 'index': MATCH}, 'value'),
933
+ prevent_initial_call=True,
922
934
  )
923
935
  def sync_documents_click(n_clicks, sync_editor_text):
924
936
  if not n_clicks:
925
937
  raise PreventUpdate
926
- ctx = dash.callback_context
927
938
  triggered = dash.callback_context.triggered
928
939
  if triggered[0]['value'] is None:
929
940
  raise PreventUpdate
@@ -937,6 +948,15 @@ def sync_documents_click(n_clicks, sync_editor_text):
937
948
  except Exception as e:
938
949
  docs = None
939
950
  msg = str(e)
951
+
952
+ if docs is None:
953
+ try:
954
+ lines = sync_editor_text.splitlines()
955
+ docs = [string_to_dict(line) for line in lines]
956
+ msg = '... '
957
+ except Exception:
958
+ docs = None
959
+
940
960
  if docs is None:
941
961
  success, msg = False, (msg + f"Unable to sync documents to {pipe}.")
942
962
  else:
@@ -956,6 +976,7 @@ def sync_documents_click(n_clicks, sync_editor_text):
956
976
  State({'type': 'limit-input', 'index': MATCH}, 'value'),
957
977
  State({'type': 'query-data-begin-input', 'index': MATCH}, 'value'),
958
978
  State({'type': 'query-data-end-input', 'index': MATCH}, 'value'),
979
+ prevent_initial_call=True,
959
980
  )
960
981
  def query_data_click(n_clicks, query_editor_text, limit_value, begin, end):
961
982
  triggered = dash.callback_context.triggered
@@ -966,7 +987,7 @@ def query_data_click(n_clicks, query_editor_text, limit_value, begin, end):
966
987
  raise PreventUpdate
967
988
 
968
989
  try:
969
- params_query = json.loads(query_editor_text)
990
+ params_query = string_to_dict(query_editor_text)
970
991
  except Exception as e:
971
992
  return alert_from_success_tuple((False, f"Invalid query:\n{e}"))
972
993
 
@@ -1043,8 +1064,8 @@ dash_app.clientside_callback(
1043
1064
 
1044
1065
  @dash_app.callback(
1045
1066
  Output("navbar-collapse", "is_open"),
1046
- [Input("navbar-toggler", "n_clicks")],
1047
- [State("navbar-collapse", "is_open")],
1067
+ Input("navbar-toggler", "n_clicks"),
1068
+ State("navbar-collapse", "is_open"),
1048
1069
  )
1049
1070
  def toggle_navbar_collapse(n_clicks: Optional[int], is_open: bool) -> bool:
1050
1071
  """
@@ -1072,6 +1093,7 @@ def toggle_navbar_collapse(n_clicks: Optional[int], is_open: bool) -> bool:
1072
1093
  Output('session-store', 'data'),
1073
1094
  Input("sign-out-button", "n_clicks"),
1074
1095
  State('session-store', 'data'),
1096
+ prevent_initial_call=True,
1075
1097
  )
1076
1098
  def sign_out_button_click(
1077
1099
  n_clicks: Optional[int],
@@ -1092,6 +1114,7 @@ def sign_out_button_click(
1092
1114
  Output({'type': 'parameters-editor', 'index': MATCH}, 'value'),
1093
1115
  Input({'type': 'parameters-as-yaml-button', 'index': MATCH}, 'n_clicks'),
1094
1116
  Input({'type': 'parameters-as-json-button', 'index': MATCH}, 'n_clicks'),
1117
+ prevent_initial_call=True,
1095
1118
  )
1096
1119
  def parameters_as_yaml_or_json_click(
1097
1120
  yaml_n_clicks: Optional[int],
@@ -1103,7 +1126,6 @@ def parameters_as_yaml_or_json_click(
1103
1126
  if not yaml_n_clicks and not json_n_clicks:
1104
1127
  raise PreventUpdate
1105
1128
 
1106
- ctx = dash.callback_context
1107
1129
  triggered = dash.callback_context.triggered
1108
1130
  if triggered[0]['value'] is None:
1109
1131
  raise PreventUpdate
@@ -1117,11 +1139,40 @@ def parameters_as_yaml_or_json_click(
1117
1139
  return json.dumps(pipe.parameters, indent=4, separators=(',', ': '), sort_keys=True)
1118
1140
 
1119
1141
 
1142
+ @dash_app.callback(
1143
+ Output({'type': 'sync-editor', 'index': MATCH}, 'value'),
1144
+ Input({'type': 'sync-as-json-button', 'index': MATCH}, 'n_clicks'),
1145
+ Input({'type': 'sync-as-lines-button', 'index': MATCH}, 'n_clicks'),
1146
+ prevent_initial_call=True,
1147
+ )
1148
+ def sync_as_json_or_lines_click(
1149
+ json_n_clicks: Optional[int],
1150
+ lines_n_clicks: Optional[int],
1151
+ ):
1152
+ """
1153
+ When the `YAML` button is clicked under the parameters editor, switch the content to YAML.
1154
+ """
1155
+ if not json_n_clicks and not lines_n_clicks:
1156
+ raise PreventUpdate
1157
+
1158
+ triggered = dash.callback_context.triggered
1159
+ if triggered[0]['value'] is None:
1160
+ raise PreventUpdate
1161
+
1162
+ as_lines = 'lines' in triggered[0]['prop_id']
1163
+ pipe = pipe_from_ctx(triggered, 'n_clicks')
1164
+ if pipe is None:
1165
+ raise PreventUpdate
1166
+
1167
+ return get_backtrack_text(pipe, lines=as_lines)
1168
+
1169
+
1120
1170
  @dash_app.callback(
1121
1171
  Output('pages-offcanvas', 'is_open'),
1122
1172
  Output('pages-offcanvas', 'children'),
1123
1173
  Input('logo-img', 'n_clicks'),
1124
1174
  State('pages-offcanvas', 'is_open'),
1175
+ prevent_initial_call=True,
1125
1176
  )
1126
1177
  def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
1127
1178
  """
@@ -1131,3 +1182,32 @@ def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
1131
1182
  if n_clicks:
1132
1183
  return not is_open, pages_children
1133
1184
  return is_open, pages_children
1185
+
1186
+
1187
+ @dash_app.callback(
1188
+ Output({'type': 'calculate-rowcount-div', 'index': MATCH}, 'children'),
1189
+ Input({'type': 'calculate-rowcount-button', 'index': MATCH}, 'n_clicks'),
1190
+ prevent_initial_call=True,
1191
+ )
1192
+ def calculate_rowcount_button_click(n_clicks: int):
1193
+ """
1194
+ Calculate the rowcount for the pipe.
1195
+ """
1196
+ if not n_clicks:
1197
+ raise PreventUpdate
1198
+
1199
+ triggered = dash.callback_context.triggered
1200
+ if triggered[0]['value'] is None:
1201
+ raise PreventUpdate
1202
+
1203
+ pipe = pipe_from_ctx(triggered, 'n_clicks')
1204
+ if pipe is None:
1205
+ raise PreventUpdate
1206
+
1207
+ try:
1208
+ rowcount = pipe.get_rowcount(debug=debug)
1209
+ return f"{rowcount:,}"
1210
+ except Exception as e:
1211
+ return (
1212
+ alert_from_success_tuple((False, f"Failed to calculate row count: {e}"))
1213
+ )
@@ -13,12 +13,14 @@ import time
13
13
  import traceback
14
14
  from datetime import datetime, timezone
15
15
 
16
+ import meerschaum as mrsm
17
+ from meerschaum.jobs import get_jobs
16
18
  from meerschaum.utils.typing import Optional, Dict, Any
17
19
  from meerschaum.api import CHECK_UPDATE
18
20
  from meerschaum.api.dash import dash_app
19
21
  from meerschaum.api.dash.sessions import get_username_from_session
20
22
  from meerschaum.utils.packages import attempt_import, import_dcc, import_html
21
- from meerschaum.api.dash.components import alert_from_success_tuple
23
+ from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
22
24
  from meerschaum.api.dash.jobs import (
23
25
  build_job_card,
24
26
  build_manage_job_buttons_div_children,
@@ -135,12 +137,12 @@ def manage_job_button_click(
135
137
  old_status = job.status
136
138
  try:
137
139
  success, msg = manage_functions[manage_job_action]()
138
- except Exception as e:
140
+ except Exception:
139
141
  success, msg = False, traceback.format_exc()
140
142
 
141
143
  ### Wait for a status change before building the elements.
142
144
  timeout_seconds = 1.0
143
- check_interval_seconds = 0.01
145
+ check_interval_seconds = mrsm.get_config('system', 'cli', 'refresh_seconds')
144
146
  begin = time.perf_counter()
145
147
  while (time.perf_counter() - begin) < timeout_seconds:
146
148
  if job.status != old_status:
@@ -251,17 +253,22 @@ def render_job_page_from_url(
251
253
  session_data: Optional[Dict[str, Any]],
252
254
  ):
253
255
  """
254
- Load the `/job/{name}` page.
256
+ Load the `/dash/jobs/{name}` page.
255
257
  """
256
- if not str(pathname).startswith('/dash/job'):
258
+ if not str(pathname).startswith('/dash/jobs'):
257
259
  return no_update
258
260
 
259
261
  session_id = (session_data or {}).get('session-id', None)
260
262
  authenticated = is_session_authenticated(str(session_id))
261
263
 
262
- job_name = pathname.replace('/dash/job', '').lstrip('/').rstrip('/')
264
+ job_name = pathname.replace('/dash/jobs', '').lstrip('/').rstrip('/')
263
265
  if not job_name:
264
- return no_update
266
+ jobs = get_jobs(executor_keys='local', combine_local_and_systemd=True)
267
+ cards = [
268
+ build_job_card(job, authenticated=authenticated, include_follow=False)
269
+ for job in jobs.values()
270
+ ]
271
+ return [html.Br(), build_cards_grid(cards, 3), html.Br()]
265
272
 
266
273
  job = _get_job(job_name)
267
274
  if not job.exists():
@@ -16,11 +16,14 @@ from meerschaum.utils.typing import Optional
16
16
  dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
17
17
  from dash.exceptions import PreventUpdate
18
18
  from dash.dependencies import Input, Output, State
19
+
19
20
  from meerschaum.api.dash import dash_app, debug, pipes, _get_pipes
20
21
  from meerschaum.api.dash.sessions import set_session
21
22
  from meerschaum.api.dash.connectors import get_web_connector
22
23
  from meerschaum.api.routes._login import login
23
24
  from meerschaum.api.dash.components import alert_from_success_tuple
25
+ from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
26
+ from meerschaum._internal.static import STATIC_CONFIG
24
27
  from fastapi_login.exceptions import InvalidCredentialsException
25
28
  from fastapi.exceptions import HTTPException
26
29
  dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
@@ -73,7 +76,13 @@ def login_button_click(
73
76
  raise PreventUpdate
74
77
 
75
78
  try:
76
- _ = login({'username': username, 'password': password})
79
+ form = CustomOAuth2PasswordRequestForm(
80
+ grant_type='password',
81
+ username=username,
82
+ password=password,
83
+ scope=' '.join(STATIC_CONFIG['tokens']['scopes'])
84
+ )
85
+ _ = login(form)
77
86
  session_id = str(uuid.uuid4())
78
87
  session_data = {
79
88
  'session-id': session_id,