meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. meerschaum/__init__.py +5 -2
  2. meerschaum/_internal/__init__.py +1 -0
  3. meerschaum/_internal/arguments/_parse_arguments.py +4 -4
  4. meerschaum/_internal/arguments/_parser.py +33 -4
  5. meerschaum/_internal/cli/__init__.py +6 -0
  6. meerschaum/_internal/cli/daemons.py +103 -0
  7. meerschaum/_internal/cli/entry.py +220 -0
  8. meerschaum/_internal/cli/workers.py +435 -0
  9. meerschaum/_internal/docs/index.py +48 -2
  10. meerschaum/_internal/entry.py +50 -14
  11. meerschaum/_internal/shell/Shell.py +121 -29
  12. meerschaum/_internal/shell/__init__.py +4 -1
  13. meerschaum/_internal/static.py +359 -0
  14. meerschaum/_internal/term/TermPageHandler.py +1 -2
  15. meerschaum/_internal/term/__init__.py +40 -6
  16. meerschaum/_internal/term/tools.py +33 -8
  17. meerschaum/actions/__init__.py +6 -4
  18. meerschaum/actions/api.py +53 -13
  19. meerschaum/actions/attach.py +1 -0
  20. meerschaum/actions/bootstrap.py +8 -8
  21. meerschaum/actions/delete.py +4 -2
  22. meerschaum/actions/edit.py +171 -25
  23. meerschaum/actions/login.py +8 -8
  24. meerschaum/actions/register.py +143 -6
  25. meerschaum/actions/reload.py +22 -5
  26. meerschaum/actions/restart.py +14 -0
  27. meerschaum/actions/show.py +184 -31
  28. meerschaum/actions/start.py +166 -17
  29. meerschaum/actions/stop.py +38 -2
  30. meerschaum/actions/sync.py +7 -2
  31. meerschaum/actions/tag.py +9 -8
  32. meerschaum/actions/verify.py +5 -8
  33. meerschaum/api/__init__.py +45 -15
  34. meerschaum/api/_events.py +46 -4
  35. meerschaum/api/_oauth2.py +162 -9
  36. meerschaum/api/_tokens.py +102 -0
  37. meerschaum/api/dash/__init__.py +0 -3
  38. meerschaum/api/dash/callbacks/__init__.py +1 -0
  39. meerschaum/api/dash/callbacks/custom.py +4 -3
  40. meerschaum/api/dash/callbacks/dashboard.py +228 -117
  41. meerschaum/api/dash/callbacks/jobs.py +14 -7
  42. meerschaum/api/dash/callbacks/login.py +10 -1
  43. meerschaum/api/dash/callbacks/pipes.py +194 -14
  44. meerschaum/api/dash/callbacks/plugins.py +0 -1
  45. meerschaum/api/dash/callbacks/register.py +10 -3
  46. meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
  47. meerschaum/api/dash/callbacks/tokens.py +389 -0
  48. meerschaum/api/dash/components.py +36 -15
  49. meerschaum/api/dash/jobs.py +1 -1
  50. meerschaum/api/dash/keys.py +35 -93
  51. meerschaum/api/dash/pages/__init__.py +2 -1
  52. meerschaum/api/dash/pages/dashboard.py +1 -20
  53. meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
  54. meerschaum/api/dash/pages/login.py +2 -2
  55. meerschaum/api/dash/pages/pipes.py +16 -5
  56. meerschaum/api/dash/pages/settings/password_reset.py +1 -1
  57. meerschaum/api/dash/pages/tokens.py +53 -0
  58. meerschaum/api/dash/pipes.py +438 -88
  59. meerschaum/api/dash/sessions.py +12 -0
  60. meerschaum/api/dash/tokens.py +603 -0
  61. meerschaum/api/dash/websockets.py +1 -1
  62. meerschaum/api/dash/webterm.py +18 -6
  63. meerschaum/api/models/__init__.py +23 -3
  64. meerschaum/api/models/_actions.py +22 -0
  65. meerschaum/api/models/_pipes.py +91 -7
  66. meerschaum/api/models/_tokens.py +81 -0
  67. meerschaum/api/resources/static/css/dash.css +16 -0
  68. meerschaum/api/resources/static/js/terminado.js +3 -0
  69. meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
  70. meerschaum/api/resources/templates/termpage.html +13 -0
  71. meerschaum/api/routes/__init__.py +1 -0
  72. meerschaum/api/routes/_actions.py +3 -4
  73. meerschaum/api/routes/_connectors.py +3 -7
  74. meerschaum/api/routes/_jobs.py +26 -35
  75. meerschaum/api/routes/_login.py +120 -15
  76. meerschaum/api/routes/_misc.py +5 -10
  77. meerschaum/api/routes/_pipes.py +178 -143
  78. meerschaum/api/routes/_plugins.py +38 -28
  79. meerschaum/api/routes/_tokens.py +236 -0
  80. meerschaum/api/routes/_users.py +47 -35
  81. meerschaum/api/routes/_version.py +3 -3
  82. meerschaum/api/routes/_webterm.py +3 -3
  83. meerschaum/config/__init__.py +100 -30
  84. meerschaum/config/_default.py +132 -64
  85. meerschaum/config/_edit.py +38 -32
  86. meerschaum/config/_formatting.py +2 -0
  87. meerschaum/config/_patch.py +10 -8
  88. meerschaum/config/_paths.py +133 -13
  89. meerschaum/config/_read_config.py +87 -36
  90. meerschaum/config/_sync.py +6 -3
  91. meerschaum/config/_version.py +1 -1
  92. meerschaum/config/environment.py +262 -0
  93. meerschaum/config/stack/__init__.py +37 -15
  94. meerschaum/config/static.py +18 -0
  95. meerschaum/connectors/_Connector.py +11 -6
  96. meerschaum/connectors/__init__.py +41 -22
  97. meerschaum/connectors/api/_APIConnector.py +34 -6
  98. meerschaum/connectors/api/_actions.py +2 -2
  99. meerschaum/connectors/api/_jobs.py +12 -1
  100. meerschaum/connectors/api/_login.py +33 -7
  101. meerschaum/connectors/api/_misc.py +2 -2
  102. meerschaum/connectors/api/_pipes.py +23 -32
  103. meerschaum/connectors/api/_plugins.py +2 -2
  104. meerschaum/connectors/api/_request.py +1 -1
  105. meerschaum/connectors/api/_tokens.py +146 -0
  106. meerschaum/connectors/api/_users.py +70 -58
  107. meerschaum/connectors/instance/_InstanceConnector.py +83 -0
  108. meerschaum/connectors/instance/__init__.py +10 -0
  109. meerschaum/connectors/instance/_pipes.py +442 -0
  110. meerschaum/connectors/instance/_plugins.py +159 -0
  111. meerschaum/connectors/instance/_tokens.py +317 -0
  112. meerschaum/connectors/instance/_users.py +188 -0
  113. meerschaum/connectors/parse.py +5 -2
  114. meerschaum/connectors/sql/_SQLConnector.py +22 -5
  115. meerschaum/connectors/sql/_cli.py +12 -11
  116. meerschaum/connectors/sql/_create_engine.py +12 -168
  117. meerschaum/connectors/sql/_fetch.py +2 -18
  118. meerschaum/connectors/sql/_pipes.py +295 -278
  119. meerschaum/connectors/sql/_plugins.py +29 -0
  120. meerschaum/connectors/sql/_sql.py +47 -22
  121. meerschaum/connectors/sql/_users.py +36 -2
  122. meerschaum/connectors/sql/tables/__init__.py +254 -122
  123. meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
  124. meerschaum/connectors/valkey/_pipes.py +60 -31
  125. meerschaum/connectors/valkey/_plugins.py +2 -26
  126. meerschaum/core/Pipe/__init__.py +115 -85
  127. meerschaum/core/Pipe/_attributes.py +425 -124
  128. meerschaum/core/Pipe/_bootstrap.py +54 -24
  129. meerschaum/core/Pipe/_cache.py +555 -0
  130. meerschaum/core/Pipe/_clear.py +0 -11
  131. meerschaum/core/Pipe/_data.py +96 -68
  132. meerschaum/core/Pipe/_deduplicate.py +0 -13
  133. meerschaum/core/Pipe/_delete.py +12 -21
  134. meerschaum/core/Pipe/_drop.py +11 -23
  135. meerschaum/core/Pipe/_dtypes.py +49 -19
  136. meerschaum/core/Pipe/_edit.py +14 -4
  137. meerschaum/core/Pipe/_fetch.py +1 -1
  138. meerschaum/core/Pipe/_index.py +8 -14
  139. meerschaum/core/Pipe/_show.py +5 -5
  140. meerschaum/core/Pipe/_sync.py +123 -204
  141. meerschaum/core/Pipe/_verify.py +4 -4
  142. meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
  143. meerschaum/core/Plugin/__init__.py +1 -1
  144. meerschaum/core/Token/_Token.py +220 -0
  145. meerschaum/core/Token/__init__.py +12 -0
  146. meerschaum/core/User/_User.py +35 -10
  147. meerschaum/core/User/__init__.py +9 -1
  148. meerschaum/core/__init__.py +1 -0
  149. meerschaum/jobs/_Executor.py +88 -4
  150. meerschaum/jobs/_Job.py +149 -38
  151. meerschaum/jobs/__init__.py +3 -2
  152. meerschaum/jobs/systemd.py +8 -3
  153. meerschaum/models/__init__.py +35 -0
  154. meerschaum/models/pipes.py +247 -0
  155. meerschaum/models/tokens.py +38 -0
  156. meerschaum/models/users.py +26 -0
  157. meerschaum/plugins/__init__.py +301 -88
  158. meerschaum/plugins/bootstrap.py +510 -4
  159. meerschaum/utils/_get_pipes.py +97 -30
  160. meerschaum/utils/daemon/Daemon.py +199 -43
  161. meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
  162. meerschaum/utils/daemon/RotatingFile.py +63 -36
  163. meerschaum/utils/daemon/StdinFile.py +53 -13
  164. meerschaum/utils/daemon/__init__.py +47 -6
  165. meerschaum/utils/daemon/_names.py +6 -3
  166. meerschaum/utils/dataframe.py +480 -82
  167. meerschaum/utils/debug.py +49 -19
  168. meerschaum/utils/dtypes/__init__.py +478 -37
  169. meerschaum/utils/dtypes/sql.py +369 -29
  170. meerschaum/utils/formatting/__init__.py +5 -2
  171. meerschaum/utils/formatting/_jobs.py +1 -1
  172. meerschaum/utils/formatting/_pipes.py +52 -50
  173. meerschaum/utils/formatting/_pprint.py +1 -0
  174. meerschaum/utils/formatting/_shell.py +44 -18
  175. meerschaum/utils/misc.py +268 -186
  176. meerschaum/utils/packages/__init__.py +25 -40
  177. meerschaum/utils/packages/_packages.py +42 -34
  178. meerschaum/utils/pipes.py +213 -0
  179. meerschaum/utils/process.py +2 -2
  180. meerschaum/utils/prompt.py +175 -144
  181. meerschaum/utils/schedule.py +2 -1
  182. meerschaum/utils/sql.py +135 -49
  183. meerschaum/utils/threading.py +42 -0
  184. meerschaum/utils/typing.py +1 -4
  185. meerschaum/utils/venv/_Venv.py +2 -2
  186. meerschaum/utils/venv/__init__.py +7 -7
  187. meerschaum/utils/warnings.py +19 -13
  188. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
  189. meerschaum-3.0.0.dist-info/RECORD +289 -0
  190. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
  191. meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
  192. meerschaum/api/models/_interfaces.py +0 -15
  193. meerschaum/api/models/_locations.py +0 -15
  194. meerschaum/api/models/_metrics.py +0 -15
  195. meerschaum/config/_environment.py +0 -145
  196. meerschaum/config/static/__init__.py +0 -186
  197. meerschaum-2.9.4.dist-info/RECORD +0 -263
  198. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
  199. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
  200. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
  201. {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
@@ -12,18 +12,26 @@ import shlex
12
12
  from textwrap import dedent
13
13
  from urllib.parse import urlencode
14
14
 
15
+ from meerschaum.utils import fetch_pipes_keys
15
16
  from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
16
- from meerschaum.utils.misc import string_to_dict
17
+ from meerschaum.utils.misc import get_connector_labels
18
+ from meerschaum.connectors import instance_types
17
19
  from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
18
20
  from meerschaum.utils.sql import get_pd_type
19
21
  from meerschaum.utils.yaml import yaml
20
22
  from meerschaum.utils.warnings import warn
21
- from meerschaum.utils.dataframe import to_json
23
+ from meerschaum.utils.dataframe import to_json, to_simple_lines
22
24
  from meerschaum.connectors.sql._fetch import get_pipe_query
23
- from meerschaum.api import CHECK_UPDATE
25
+ from meerschaum.api import CHECK_UPDATE, get_api_connector
24
26
  from meerschaum.api.dash import debug, _get_pipes
25
27
  from meerschaum.api.dash.connectors import get_web_connector
26
- from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
28
+ from meerschaum.api.dash.components import (
29
+ alert_from_success_tuple,
30
+ build_cards_grid,
31
+ sign_out_button,
32
+ logo_row,
33
+ pages_offcanvas,
34
+ )
27
35
  from meerschaum.api.dash.sessions import is_session_authenticated
28
36
  from meerschaum.config import get_config
29
37
  import meerschaum as mrsm
@@ -58,28 +66,20 @@ def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, N
58
66
 
59
67
  def keys_from_state(
60
68
  state: Dict[str, Any],
61
- with_params: bool = False
69
+ with_tags: bool = False,
62
70
  ) -> Union[
63
71
  Tuple[List[str], List[str], List[str]],
64
- Tuple[List[str], List[str], List[str], str],
72
+ Tuple[List[str], List[str], List[str], List[str]],
65
73
  ]:
66
74
  """
67
75
  Read the current state and return the selected keys lists.
68
76
  """
69
77
  _filters = {
70
- 'ck' : state.get(f"connector-keys-{state['pipes-filter-tabs.active_tab']}.value", None),
71
- 'mk' : state.get(f"metric-keys-{state['pipes-filter-tabs.active_tab']}.value", None),
72
- 'lk' : state.get(f"location-keys-{state['pipes-filter-tabs.active_tab']}.value", None),
78
+ 'ck': state.get("connector-keys-dropdown.value", None),
79
+ 'mk': state.get("metric-keys-dropdown.value", None),
80
+ 'lk': state.get("location-keys-dropdown.value", None),
81
+ 'tags': state.get("tags-dropdown.value", None),
73
82
  }
74
- if state['pipes-filter-tabs.active_tab'] == 'input':
75
- try:
76
- # params = string_to_dict(state['params-textarea.value'])
77
- params = string_to_dict(state['search-parameters-editor.value'])
78
- except Exception:
79
- params = None
80
- else:
81
- params = None
82
-
83
83
  for k in _filters:
84
84
  _filters[k] = [] if _filters[k] is None else _filters[k]
85
85
  if not isinstance(_filters[k], list):
@@ -89,8 +89,8 @@ def keys_from_state(
89
89
  print(e)
90
90
  _filters[k] = []
91
91
  keys = [_filters['ck'], _filters['mk'], _filters['lk']]
92
- if with_params:
93
- keys.append(params)
92
+ if with_tags:
93
+ keys.append(_filters['tags'])
94
94
  return tuple(keys)
95
95
 
96
96
 
@@ -98,12 +98,12 @@ def pipes_from_state(
98
98
  state: Dict[str, Any],
99
99
  **kw
100
100
  ):
101
- _ck, _mk, _lk, _params = keys_from_state(state, with_params=True)
101
+ _ck, _mk, _lk, _tags = keys_from_state(state, with_tags=True)
102
102
  try:
103
103
  _pipes = _get_pipes(
104
104
  _ck, _mk, _lk,
105
- params = _params,
106
- mrsm_instance = get_web_connector(state),
105
+ tags=(_tags or []),
106
+ mrsm_instance=get_web_connector(state),
107
107
  **kw
108
108
  )
109
109
  except Exception as e:
@@ -343,6 +343,7 @@ def accordion_items_from_pipe(
343
343
  items_titles['sql'] = '📃 SQL Query'
344
344
  items_titles.update({
345
345
  'recent-data': '🗃️ Recent Data',
346
+ 'query-data': '🔍 Query Data',
346
347
  'sync-data': '📝 Sync Documents',
347
348
  })
348
349
 
@@ -376,48 +377,48 @@ def accordion_items_from_pipe(
376
377
  html.Th(
377
378
  html.Span(
378
379
  "Key",
379
- id={'type': 'key-table-header', 'id': pipe_meta_str},
380
+ id={'type': 'key-table-header', 'index': pipe_meta_str},
380
381
  style={"textDecoration": "underline", "cursor": "pointer"},
381
382
  ),
382
383
  ),
383
384
  html.Th(
384
385
  html.Span(
385
386
  "Column",
386
- id={'type': 'column-table-header', 'id': pipe_meta_str},
387
+ id={'type': 'column-table-header', 'index': pipe_meta_str},
387
388
  style={"textDecoration": "underline", "cursor": "pointer"},
388
389
  ),
389
390
  ),
390
391
  html.Th(
391
392
  html.Span(
392
393
  "Index",
393
- id={'type': 'index-table-header', 'id': pipe_meta_str},
394
+ id={'type': 'index-table-header', 'index': pipe_meta_str},
394
395
  style={"textDecoration": "underline", "cursor": "pointer"},
395
396
  ),
396
397
  ),
397
398
  html.Th(
398
399
  html.Span(
399
400
  "Is Composite",
400
- id={'type': 'is-composite-table-header', 'id': pipe_meta_str},
401
+ id={'type': 'is-composite-table-header', 'index': pipe_meta_str},
401
402
  style={"textDecoration": "underline", "cursor": "pointer"},
402
403
  ),
403
404
  ),
404
405
  dbc.Tooltip(
405
406
  "Unique reference name for the index "
406
407
  "(e.g. `datetime` for the range axis)",
407
- target={'type': 'key-table-header', 'id': pipe_meta_str},
408
+ target={'type': 'key-table-header', 'index': pipe_meta_str},
408
409
  ),
409
410
  dbc.Tooltip(
410
411
  "The actual column (field name) in the target dataset.",
411
- target={'type': 'column-table-header', 'id': pipe_meta_str},
412
+ target={'type': 'column-table-header', 'index': pipe_meta_str},
412
413
  ),
413
414
  dbc.Tooltip(
414
415
  "The name of the index created on the given columns.",
415
- target={'type': 'index-table-header', 'id': pipe_meta_str},
416
+ target={'type': 'index-table-header', 'index': pipe_meta_str},
416
417
  ),
417
418
  dbc.Tooltip(
418
419
  "Whether the column is used in the composite primary key "
419
420
  "to determine updates.",
420
- target={'type': 'is-composite-table-header', 'id': pipe_meta_str},
421
+ target={'type': 'is-composite-table-header', 'index': pipe_meta_str},
421
422
  ),
422
423
  ]
423
424
  )
@@ -465,7 +466,7 @@ def accordion_items_from_pipe(
465
466
  overview_rows.append(
466
467
  html.Tr([
467
468
  html.Td("Indices" if len(indices_rows) != 1 else "Index"),
468
- html.Td(indices_table),
469
+ html.Td(html.Div(indices_table, style={'overflowX': 'auto'})),
469
470
  ])
470
471
  )
471
472
 
@@ -482,9 +483,14 @@ def accordion_items_from_pipe(
482
483
  ])
483
484
  )
484
485
 
485
- items_bodies['overview'] = dbc.Table(
486
- overview_header + [html.Tbody(overview_rows)],
487
- bordered=False, hover=True, striped=False,
486
+ items_bodies['overview'] = html.Div(
487
+ dbc.Table(
488
+ overview_header + [html.Tbody(overview_rows)],
489
+ bordered=False,
490
+ hover=True,
491
+ striped=False,
492
+ ),
493
+ style={'overflowX': 'auto'},
488
494
  )
489
495
 
490
496
  if 'stats' in active_items:
@@ -496,17 +502,13 @@ def accordion_items_from_pipe(
496
502
  (newest_time - oldest_time) if newest_time is not None and oldest_time is not None
497
503
  else None
498
504
  )
499
- rowcount = pipe.get_rowcount(debug=debug)
500
505
  except Exception:
501
506
  oldest_time = None
502
507
  newest_time = None
503
508
  interval = None
504
- rowcount = None
505
509
 
506
510
  stats_rows = []
507
- if rowcount is not None:
508
- stats_rows.append(html.Tr([html.Td("Row Count"), html.Td(f"{rowcount:,}")]))
509
- if interval is not None:
511
+ if interval is not None and not isinstance(interval, int):
510
512
  stats_rows.append(
511
513
  html.Tr([html.Td("Timespan"), html.Td(humanfriendly.format_timespan(interval))])
512
514
  )
@@ -515,7 +517,39 @@ def accordion_items_from_pipe(
515
517
  if newest_time is not None:
516
518
  stats_rows.append(html.Tr([html.Td("Newest time"), html.Td(str(newest_time))]))
517
519
 
518
- items_bodies['stats'] = dbc.Table(stats_header + [html.Tbody(stats_rows)], hover=True)
520
+ precision = pipe.precision
521
+ if precision:
522
+ stats_rows.append(
523
+ html.Tr([
524
+ html.Td("Precision"),
525
+ html.Td(str(precision.get('interval', 1)) + ' ' + str(precision.get('unit', 'unit')))
526
+ ])
527
+ )
528
+
529
+ stats_rows.append(
530
+ html.Tr([
531
+ html.Td("Row count"),
532
+ html.Td(
533
+ html.Div(
534
+ dbc.Button(
535
+ "Calculate",
536
+ color='link',
537
+ size='sm',
538
+ style={'text-decoration': 'none'},
539
+ id={'type': 'calculate-rowcount-button', 'index': pipe_meta_str},
540
+ )
541
+ if pipe.exists(debug=debug)
542
+ else '0'
543
+ ),
544
+ id={'type': 'calculate-rowcount-div', 'index': pipe_meta_str},
545
+ )
546
+ ])
547
+ )
548
+
549
+ items_bodies['stats'] = html.Div(
550
+ dbc.Table(stats_header + [html.Tbody(stats_rows)], hover=True),
551
+ style={'overflowX': 'auto'},
552
+ )
519
553
 
520
554
  if 'columns' in active_items:
521
555
  try:
@@ -535,9 +569,9 @@ def accordion_items_from_pipe(
535
569
  ]
536
570
  columns_body = [html.Tbody(columns_rows)]
537
571
  columns_table = dbc.Table(columns_header + columns_body, bordered=False, hover=True)
572
+ items_bodies['columns'] = html.Div(columns_table, style={'overflowX': 'auto'})
538
573
  except Exception:
539
- columns_table = html.P("Could not retrieve columns ― please try again.")
540
- items_bodies['columns'] = columns_table
574
+ items_bodies['columns'] = html.P("Could not retrieve columns ― please try again.")
541
575
 
542
576
  if 'parameters' in active_items:
543
577
  parameters_editor = dash_ace.DashAceEditor(
@@ -545,7 +579,7 @@ def accordion_items_from_pipe(
545
579
  mode='norm',
546
580
  tabSize=4,
547
581
  theme='twilight',
548
- id={'type': 'parameters-editor', 'index': json.dumps(pipe.meta)},
582
+ id={'type': 'parameters-editor', 'index': pipe_meta_str},
549
583
  width='100%',
550
584
  height='500px',
551
585
  readOnly=False,
@@ -557,19 +591,19 @@ def accordion_items_from_pipe(
557
591
  )
558
592
  update_parameters_button = dbc.Button(
559
593
  "Update",
560
- id={'type': 'update-parameters-button', 'index': json.dumps(pipe.meta)},
594
+ id={'type': 'update-parameters-button', 'index': pipe_meta_str},
561
595
  )
562
596
 
563
597
  as_yaml_button = dbc.Button(
564
598
  "YAML",
565
- id={'type': 'parameters-as-yaml-button', 'index': json.dumps(pipe.meta)},
599
+ id={'type': 'parameters-as-yaml-button', 'index': pipe_meta_str},
566
600
  color='link',
567
601
  size='sm',
568
602
  style={'text-decoration': 'none'},
569
603
  )
570
604
  as_json_button = dbc.Button(
571
605
  "JSON",
572
- id={'type': 'parameters-as-json-button', 'index': json.dumps(pipe.meta)},
606
+ id={'type': 'parameters-as-json-button', 'index': pipe_meta_str},
573
607
  color='link',
574
608
  size='sm',
575
609
  style={'text-decoration': 'none', 'margin-left': '10px'},
@@ -578,21 +612,28 @@ def accordion_items_from_pipe(
578
612
  parameters_editor,
579
613
  html.Br(),
580
614
  dbc.Row([
581
- dbc.Col(html.Span(
582
- (
583
- ([update_parameters_button] if authenticated else []) +
584
- [
585
- as_json_button,
586
- as_yaml_button,
587
- ]
588
- )
589
- ), width=4),
590
- dbc.Col([
591
- html.Div(
592
- id={'type': 'update-parameters-success-div', 'index': json.dumps(pipe.meta)}
593
- )
594
- ],
595
- width=True,
615
+ dbc.Col(
616
+ html.Span(
617
+ (
618
+ ([update_parameters_button] if authenticated else []) +
619
+ [
620
+ as_json_button,
621
+ as_yaml_button,
622
+ ]
623
+ )
624
+ ),
625
+ width=4,
626
+ ),
627
+ dbc.Col(
628
+ [
629
+ html.Div(
630
+ id={
631
+ 'type': 'update-parameters-success-div',
632
+ 'index': pipe_meta_str,
633
+ }
634
+ )
635
+ ],
636
+ width=True,
596
637
  )
597
638
  ]),
598
639
  ]
@@ -632,7 +673,7 @@ def accordion_items_from_pipe(
632
673
  mode='sql',
633
674
  tabSize=4,
634
675
  theme='twilight',
635
- id={'type': 'sql-editor', 'index': json.dumps(pipe.meta)},
676
+ id={'type': 'sql-editor', 'index': pipe_meta_str},
636
677
  width='100%',
637
678
  height='500px',
638
679
  readOnly=False,
@@ -644,7 +685,7 @@ def accordion_items_from_pipe(
644
685
  )
645
686
  update_sql_button = dbc.Button(
646
687
  "Update",
647
- id={'type': 'update-sql-button', 'index': json.dumps(pipe.meta)},
688
+ id={'type': 'update-sql-button', 'index': pipe_meta_str},
648
689
  )
649
690
  items_bodies['sql'] = html.Div([
650
691
  sql_editor,
@@ -653,7 +694,7 @@ def accordion_items_from_pipe(
653
694
  dbc.Col([update_sql_button], width=2),
654
695
  dbc.Col([
655
696
  html.Div(
656
- id={'type': 'update-sql-success-div', 'index': json.dumps(pipe.meta)}
697
+ id={'type': 'update-sql-success-div', 'index': pipe_meta_str}
657
698
  )
658
699
  ],
659
700
  width=True,
@@ -664,33 +705,84 @@ def accordion_items_from_pipe(
664
705
  if 'recent-data' in active_items:
665
706
  try:
666
707
  df = pipe.get_backtrack_data(backtrack_minutes=10, limit=10, debug=debug).astype(str)
667
- table = dbc.Table.from_dataframe(df, bordered=False, hover=True)
708
+ table = dbc.Table.from_dataframe(df, bordered=False, hover=True)
709
+ items_bodies['recent-data'] = html.Div(table, style={'overflowX': 'auto'})
668
710
  except Exception:
669
- table = html.P("Could not retrieve recent data.")
670
- items_bodies['recent-data'] = table
711
+ items_bodies['recent-data'] = html.P("Could not retrieve recent data.")
712
+
713
+ if 'query-data' in active_items:
714
+ query_editor = dash_ace.DashAceEditor(
715
+ value='{\n \n}',
716
+ mode='norm',
717
+ tabSize=4,
718
+ theme='twilight',
719
+ id={'type': 'query-editor', 'index': pipe_meta_str},
720
+ width='100%',
721
+ height='200px',
722
+ readOnly=False,
723
+ showGutter=True,
724
+ showPrintMargin=False,
725
+ highlightActiveLine=True,
726
+ wrapEnabled=True,
727
+ style={'min-height': '120px'},
728
+ )
729
+ query_data_button = dbc.Button(
730
+ "Query",
731
+ id={'type': 'query-data-button', 'index': pipe_meta_str},
732
+ )
733
+
734
+ begin_end_input_group = dbc.InputGroup(
735
+ [
736
+ dbc.Input(
737
+ id={'type': 'query-data-begin-input', 'index': pipe_meta_str},
738
+ placeholder="Begin",
739
+ ),
740
+ dbc.Input(
741
+ id={'type': 'query-data-end-input', 'index': pipe_meta_str},
742
+ placeholder="End",
743
+ ),
744
+ ],
745
+ size="sm",
746
+ )
747
+
748
+ limit_input = dbc.Input(
749
+ type="number",
750
+ min=0,
751
+ max=100,
752
+ value=10,
753
+ step=1,
754
+ placeholder="Limit",
755
+ id={'type': 'limit-input', 'index': pipe_meta_str},
756
+ )
757
+ query_result_div = html.Div(
758
+ id={'type': 'query-result-div', 'index': pipe_meta_str},
759
+ style={'overflowX': 'auto'},
760
+ )
761
+
762
+ items_bodies['query-data'] = html.Div([
763
+ query_editor,
764
+ html.Br(),
765
+ dbc.Row(
766
+ [
767
+ dbc.Col([query_data_button], lg=2, md=3, sm=4, xs=6, width=2),
768
+ dbc.Col([begin_end_input_group], lg=6, md=6, sm=4, width=6),
769
+ dbc.Col(html.Div([limit_input, dbc.FormText("Row Limit")]), lg=2, md=3, sm=4, xs=6, width=2),
770
+ ],
771
+ justify="between",
772
+ ),
773
+ dbc.Row([
774
+ dbc.Col([query_result_div], width=True),
775
+ ]),
776
+ ])
671
777
 
672
778
  if 'sync-data' in active_items:
673
- backtrack_df = pipe.get_backtrack_data(debug=debug, limit=1)
674
- try:
675
- json_text = to_json(
676
- backtrack_df,
677
- orient='records',
678
- date_format='iso',
679
- force_ascii=False,
680
- indent=4,
681
- date_unit='us',
682
- ) if backtrack_df is not None else '[]'
683
- except Exception as e:
684
- warn(e)
685
- json_text = '[]'
686
-
687
- json_text = json.dumps(json.loads(json_text), indent=4, separators=(',', ': '))
779
+ backtrack_text = get_backtrack_text(pipe)
688
780
  sync_editor = dash_ace.DashAceEditor(
689
- value = json_text,
781
+ value = backtrack_text,
690
782
  mode = 'norm',
691
783
  tabSize = 4,
692
784
  theme = 'twilight',
693
- id = {'type': 'sync-editor', 'index': json.dumps(pipe.meta)},
785
+ id = {'type': 'sync-editor', 'index': pipe_meta_str},
694
786
  width = '100%',
695
787
  height = '500px',
696
788
  readOnly = False,
@@ -700,16 +792,40 @@ def accordion_items_from_pipe(
700
792
  wrapEnabled = True,
701
793
  style = {'min-height': '120px'},
702
794
  )
795
+
796
+ sync_as_json_button = dbc.Button(
797
+ "JSON",
798
+ id={'type': 'sync-as-json-button', 'index': pipe_meta_str},
799
+ color='link',
800
+ size='sm',
801
+ style={'text-decoration': 'none', 'margin-left': '10px'},
802
+ )
803
+ sync_as_lines_button = dbc.Button(
804
+ "Lines",
805
+ id={'type': 'sync-as-lines-button', 'index': pipe_meta_str},
806
+ color='link',
807
+ size='sm',
808
+ style={'text-decoration': 'none', 'margin-left': '10px'},
809
+ )
810
+
703
811
  update_sync_button = dbc.Button(
704
812
  "Sync",
705
- id = {'type': 'update-sync-button', 'index': json.dumps(pipe.meta)},
813
+ id = {'type': 'update-sync-button', 'index': pipe_meta_str},
706
814
  )
707
- sync_success_div = html.Div(id={'type': 'sync-success-div', 'index': json.dumps(pipe.meta)})
815
+ sync_success_div = html.Div(id={'type': 'sync-success-div', 'index': pipe_meta_str})
708
816
  items_bodies['sync-data'] = html.Div([
709
817
  sync_editor,
710
818
  html.Br(),
711
819
  dbc.Row([
712
- dbc.Col([update_sync_button], width=1),
820
+ dbc.Col(html.Span(
821
+ (
822
+ ([update_sync_button] if authenticated else []) +
823
+ [
824
+ sync_as_json_button,
825
+ sync_as_lines_button,
826
+ ]
827
+ )
828
+ ), width=4),
713
829
  dbc.Col([sync_success_div], width=True),
714
830
  ]),
715
831
  ])
@@ -719,3 +835,237 @@ def accordion_items_from_pipe(
719
835
  for item_id, title in items_titles.items()
720
836
  ]
721
837
 
838
+
839
+ def get_backtrack_text(
840
+ pipe: mrsm.Pipe,
841
+ lines: bool = False,
842
+ limit: int = 5,
843
+ ) -> str:
844
+ """
845
+ Return the backtrack documents as text for the sync editor.
846
+ """
847
+ backtrack_df = pipe.get_backtrack_data(debug=debug, limit=limit)
848
+ if lines:
849
+ return to_simple_lines(backtrack_df)
850
+ try:
851
+ json_text = to_json(
852
+ backtrack_df,
853
+ orient='records',
854
+ date_format='iso',
855
+ force_ascii=False,
856
+ indent=4,
857
+ date_unit='us',
858
+ ) if backtrack_df is not None else '[]'
859
+ except Exception as e:
860
+ warn(e)
861
+ json_text = '[]'
862
+
863
+ return json.dumps(json.loads(json_text), indent=4, separators=(',', ': '))
864
+
865
+
866
+ def build_pipes_dropdown_keys_row(
867
+ connector_keys: List[str],
868
+ metric_keys: List[str],
869
+ location_keys: List[str],
870
+ tags: List[str],
871
+ pipes: List[mrsm.Pipe],
872
+ instance_connector: mrsm.connectors.InstanceConnector,
873
+ ) -> dbc.Row:
874
+ """
875
+ Return the dropdown keys row for the dedicated pipes page.
876
+ """
877
+ ck_alone = connector_keys and not any([str(x) for x in (tags + metric_keys + location_keys)])
878
+ mk_alone = metric_keys and not any([str(x) for x in (connector_keys + tags + location_keys)])
879
+ lk_alone = location_keys and not any([str(x) for x in (connector_keys + metric_keys + tags)])
880
+ all_keys = fetch_pipes_keys('registered', instance_connector)
881
+
882
+ ck_options_source = (
883
+ {keys_tuple[0] for keys_tuple in all_keys}
884
+ if ck_alone
885
+ else {p.connector_keys for p in pipes}
886
+ )
887
+ ck_options = sorted(ck_options_source.union(connector_keys))
888
+
889
+ mk_options_source = (
890
+ {keys_tuple[1] for keys_tuple in all_keys}
891
+ if mk_alone
892
+ else {p.metric_key for p in pipes}
893
+ )
894
+ mk_options = sorted(mk_options_source.union(metric_keys))
895
+
896
+ lk_options_source = (
897
+ {str(keys_tuple[2]) for keys_tuple in all_keys}
898
+ if lk_alone
899
+ else {str(p.location_key) for p in pipes}
900
+ )
901
+ lk_options = sorted(lk_options_source.union({str(lk) for lk in location_keys}))
902
+
903
+ return dbc.Row(
904
+ [
905
+ dbc.Col(
906
+ html.Div(
907
+ [
908
+ dcc.Dropdown(
909
+ id='pipes-connector-keys-dropdown',
910
+ options=ck_options,
911
+ value=[str(ck) for ck in connector_keys],
912
+ placeholder='Connectors',
913
+ multi=True,
914
+ ),
915
+ ],
916
+ className='dbc_dark',
917
+ ),
918
+ lg=4,
919
+ md=12,
920
+ sm=12,
921
+ ),
922
+ dbc.Col(
923
+ html.Div(
924
+ [
925
+ dcc.Dropdown(
926
+ id='pipes-metric-keys-dropdown',
927
+ options=mk_options,
928
+ value=[str(mk) for mk in metric_keys],
929
+ placeholder='Metrics',
930
+ multi=True,
931
+ ),
932
+ ],
933
+ className='dbc_dark'
934
+ ),
935
+ lg=4,
936
+ md=12,
937
+ sm=12,
938
+ ),
939
+ dbc.Col(
940
+ html.Div(
941
+ [
942
+ dcc.Dropdown(
943
+ id='pipes-location-keys-dropdown',
944
+ options=lk_options,
945
+ value=[str(lk) for lk in location_keys],
946
+ placeholder='Locations',
947
+ multi=True,
948
+ ),
949
+ ],
950
+ className='dbc_dark'
951
+ ),
952
+ lg=4,
953
+ md=12,
954
+ sm=12,
955
+ ),
956
+ ] ### end of filters row children
957
+ )
958
+
959
+
960
+ def build_pipes_tags_dropdown(
961
+ connector_keys: List[str],
962
+ metric_keys: List[str],
963
+ location_keys: List[str],
964
+ tags: List[str],
965
+ instance: str,
966
+ ) -> dbc.Row:
967
+ """
968
+ Build the tags dropdown for the dedicated pipes page.
969
+ """
970
+ _tags_alone = tags and not any([str(x) for x in (connector_keys + metric_keys + location_keys)])
971
+ _tags_pipes = mrsm.get_pipes(
972
+ connector_keys=connector_keys,
973
+ metric_keys=metric_keys,
974
+ location_keys=location_keys,
975
+ tags=tags,
976
+ instance=instance,
977
+ as_tags_dict=True,
978
+ )
979
+
980
+ _all_tags = list(
981
+ mrsm.get_pipes(
982
+ instance=instance,
983
+ as_tags_dict=True,
984
+ )
985
+ ) if _tags_alone else []
986
+
987
+ tags_options = [
988
+ str(tag)
989
+ for tag in (_all_tags if _tags_alone else _tags_pipes)
990
+ ]
991
+ if tags:
992
+ tags_options += [tag for tag in tags if tag not in tags_options]
993
+
994
+ return dbc.Row(
995
+ [
996
+ dbc.Col(
997
+ html.Div(
998
+ dcc.Dropdown(
999
+ id='pipes-tags-dropdown',
1000
+ options=tags_options,
1001
+ value=tags,
1002
+ placeholder='Tags',
1003
+ multi=True,
1004
+ searchable=True,
1005
+ ),
1006
+ className="dbc_dark",
1007
+ id="pipes-tags-dropdown-div",
1008
+ ),
1009
+ width=True,
1010
+ ),
1011
+ dbc.Col(
1012
+ dbc.Button(
1013
+ "Clear all",
1014
+ color='link',
1015
+ size='sm',
1016
+ style={'text-decoration': 'none'},
1017
+ id='pipes-clear-all-button',
1018
+ ),
1019
+ width='auto',
1020
+ ),
1021
+ ],
1022
+ className='g-0',
1023
+ align='center',
1024
+ )
1025
+
1026
+
1027
+ def build_pipes_navbar(instance_keys: Optional[str] = None, with_instance_select: bool = True):
1028
+ """
1029
+ Build the navbar from the selected instance keys.
1030
+ """
1031
+ instance_select = dbc.Select(
1032
+ id='instance-select',
1033
+ size='sm',
1034
+ value=instance_keys or str(get_api_connector()),
1035
+ options=[
1036
+ {'label': (i[:32] + '…') if len(i) > 32 else i, 'value': i}
1037
+ for i in get_connector_labels(*instance_types)
1038
+ ],
1039
+ class_name='dbc_dark custom-select custom-select-sm',
1040
+ )
1041
+ instance_select_div_style = {} if with_instance_select else {'visibility': 'hidden'}
1042
+ instance_select_div = html.Div(instance_select, style=instance_select_div_style)
1043
+ return html.Div(
1044
+ [
1045
+ pages_offcanvas,
1046
+ dbc.Navbar(
1047
+ dbc.Container(
1048
+ [
1049
+ logo_row,
1050
+ dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
1051
+ dbc.Collapse(
1052
+ dbc.Row(
1053
+ [
1054
+ dbc.Col(instance_select_div, width='auto'),
1055
+ dbc.Col(sign_out_button, width='auto'),
1056
+ ],
1057
+ className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",
1058
+ align='center',
1059
+ ),
1060
+ id='navbar-collapse',
1061
+ is_open=False,
1062
+ navbar=True,
1063
+ ),
1064
+ ]
1065
+ ),
1066
+ dark=True,
1067
+ color='dark'
1068
+ ),
1069
+ ],
1070
+ id='pages-navbar-div',
1071
+ )