meerschaum 2.9.5__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. meerschaum/__init__.py +5 -2
  2. meerschaum/_internal/__init__.py +1 -0
  3. meerschaum/_internal/arguments/_parse_arguments.py +4 -4
  4. meerschaum/_internal/arguments/_parser.py +33 -4
  5. meerschaum/_internal/cli/__init__.py +6 -0
  6. meerschaum/_internal/cli/daemons.py +103 -0
  7. meerschaum/_internal/cli/entry.py +220 -0
  8. meerschaum/_internal/cli/workers.py +435 -0
  9. meerschaum/_internal/docs/index.py +48 -2
  10. meerschaum/_internal/entry.py +50 -14
  11. meerschaum/_internal/shell/Shell.py +121 -29
  12. meerschaum/_internal/shell/__init__.py +4 -1
  13. meerschaum/_internal/static.py +359 -0
  14. meerschaum/_internal/term/TermPageHandler.py +1 -2
  15. meerschaum/_internal/term/__init__.py +40 -6
  16. meerschaum/_internal/term/tools.py +33 -8
  17. meerschaum/actions/__init__.py +6 -4
  18. meerschaum/actions/api.py +53 -13
  19. meerschaum/actions/attach.py +1 -0
  20. meerschaum/actions/bootstrap.py +8 -8
  21. meerschaum/actions/delete.py +4 -2
  22. meerschaum/actions/edit.py +171 -25
  23. meerschaum/actions/login.py +8 -8
  24. meerschaum/actions/register.py +143 -6
  25. meerschaum/actions/reload.py +22 -5
  26. meerschaum/actions/restart.py +14 -0
  27. meerschaum/actions/show.py +184 -31
  28. meerschaum/actions/start.py +166 -17
  29. meerschaum/actions/stop.py +38 -2
  30. meerschaum/actions/sync.py +7 -2
  31. meerschaum/actions/tag.py +9 -8
  32. meerschaum/actions/verify.py +5 -8
  33. meerschaum/api/__init__.py +45 -15
  34. meerschaum/api/_events.py +46 -4
  35. meerschaum/api/_oauth2.py +162 -9
  36. meerschaum/api/_tokens.py +102 -0
  37. meerschaum/api/dash/__init__.py +0 -3
  38. meerschaum/api/dash/callbacks/__init__.py +1 -0
  39. meerschaum/api/dash/callbacks/custom.py +4 -3
  40. meerschaum/api/dash/callbacks/dashboard.py +198 -118
  41. meerschaum/api/dash/callbacks/jobs.py +14 -7
  42. meerschaum/api/dash/callbacks/login.py +10 -1
  43. meerschaum/api/dash/callbacks/pipes.py +194 -14
  44. meerschaum/api/dash/callbacks/plugins.py +0 -1
  45. meerschaum/api/dash/callbacks/register.py +10 -3
  46. meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
  47. meerschaum/api/dash/callbacks/tokens.py +389 -0
  48. meerschaum/api/dash/components.py +36 -15
  49. meerschaum/api/dash/jobs.py +1 -1
  50. meerschaum/api/dash/keys.py +35 -93
  51. meerschaum/api/dash/pages/__init__.py +2 -1
  52. meerschaum/api/dash/pages/dashboard.py +1 -20
  53. meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
  54. meerschaum/api/dash/pages/login.py +2 -2
  55. meerschaum/api/dash/pages/pipes.py +16 -5
  56. meerschaum/api/dash/pages/settings/password_reset.py +1 -1
  57. meerschaum/api/dash/pages/tokens.py +53 -0
  58. meerschaum/api/dash/pipes.py +382 -95
  59. meerschaum/api/dash/sessions.py +12 -0
  60. meerschaum/api/dash/tokens.py +603 -0
  61. meerschaum/api/dash/websockets.py +1 -1
  62. meerschaum/api/dash/webterm.py +18 -6
  63. meerschaum/api/models/__init__.py +23 -3
  64. meerschaum/api/models/_actions.py +22 -0
  65. meerschaum/api/models/_pipes.py +91 -7
  66. meerschaum/api/models/_tokens.py +81 -0
  67. meerschaum/api/resources/static/js/terminado.js +3 -0
  68. meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
  69. meerschaum/api/resources/templates/termpage.html +13 -0
  70. meerschaum/api/routes/__init__.py +1 -0
  71. meerschaum/api/routes/_actions.py +3 -4
  72. meerschaum/api/routes/_connectors.py +3 -7
  73. meerschaum/api/routes/_jobs.py +26 -35
  74. meerschaum/api/routes/_login.py +120 -15
  75. meerschaum/api/routes/_misc.py +5 -10
  76. meerschaum/api/routes/_pipes.py +178 -143
  77. meerschaum/api/routes/_plugins.py +38 -28
  78. meerschaum/api/routes/_tokens.py +236 -0
  79. meerschaum/api/routes/_users.py +47 -35
  80. meerschaum/api/routes/_version.py +3 -3
  81. meerschaum/api/routes/_webterm.py +3 -3
  82. meerschaum/config/__init__.py +100 -30
  83. meerschaum/config/_default.py +132 -64
  84. meerschaum/config/_edit.py +38 -32
  85. meerschaum/config/_formatting.py +2 -0
  86. meerschaum/config/_patch.py +10 -8
  87. meerschaum/config/_paths.py +133 -13
  88. meerschaum/config/_read_config.py +87 -36
  89. meerschaum/config/_sync.py +6 -3
  90. meerschaum/config/_version.py +1 -1
  91. meerschaum/config/environment.py +262 -0
  92. meerschaum/config/stack/__init__.py +37 -15
  93. meerschaum/config/static.py +18 -0
  94. meerschaum/connectors/_Connector.py +11 -6
  95. meerschaum/connectors/__init__.py +41 -22
  96. meerschaum/connectors/api/_APIConnector.py +34 -6
  97. meerschaum/connectors/api/_actions.py +2 -2
  98. meerschaum/connectors/api/_jobs.py +12 -1
  99. meerschaum/connectors/api/_login.py +33 -7
  100. meerschaum/connectors/api/_misc.py +2 -2
  101. meerschaum/connectors/api/_pipes.py +23 -32
  102. meerschaum/connectors/api/_plugins.py +2 -2
  103. meerschaum/connectors/api/_request.py +1 -1
  104. meerschaum/connectors/api/_tokens.py +146 -0
  105. meerschaum/connectors/api/_users.py +70 -58
  106. meerschaum/connectors/instance/_InstanceConnector.py +83 -0
  107. meerschaum/connectors/instance/__init__.py +10 -0
  108. meerschaum/connectors/instance/_pipes.py +442 -0
  109. meerschaum/connectors/instance/_plugins.py +159 -0
  110. meerschaum/connectors/instance/_tokens.py +317 -0
  111. meerschaum/connectors/instance/_users.py +188 -0
  112. meerschaum/connectors/parse.py +5 -2
  113. meerschaum/connectors/sql/_SQLConnector.py +22 -5
  114. meerschaum/connectors/sql/_cli.py +12 -11
  115. meerschaum/connectors/sql/_create_engine.py +12 -168
  116. meerschaum/connectors/sql/_fetch.py +2 -18
  117. meerschaum/connectors/sql/_pipes.py +295 -278
  118. meerschaum/connectors/sql/_plugins.py +29 -0
  119. meerschaum/connectors/sql/_sql.py +46 -21
  120. meerschaum/connectors/sql/_users.py +36 -2
  121. meerschaum/connectors/sql/tables/__init__.py +254 -122
  122. meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
  123. meerschaum/connectors/valkey/_pipes.py +60 -31
  124. meerschaum/connectors/valkey/_plugins.py +2 -26
  125. meerschaum/core/Pipe/__init__.py +115 -85
  126. meerschaum/core/Pipe/_attributes.py +425 -124
  127. meerschaum/core/Pipe/_bootstrap.py +54 -24
  128. meerschaum/core/Pipe/_cache.py +555 -0
  129. meerschaum/core/Pipe/_clear.py +0 -11
  130. meerschaum/core/Pipe/_data.py +96 -68
  131. meerschaum/core/Pipe/_deduplicate.py +0 -13
  132. meerschaum/core/Pipe/_delete.py +12 -21
  133. meerschaum/core/Pipe/_drop.py +11 -23
  134. meerschaum/core/Pipe/_dtypes.py +49 -19
  135. meerschaum/core/Pipe/_edit.py +14 -4
  136. meerschaum/core/Pipe/_fetch.py +1 -1
  137. meerschaum/core/Pipe/_index.py +8 -14
  138. meerschaum/core/Pipe/_show.py +5 -5
  139. meerschaum/core/Pipe/_sync.py +123 -204
  140. meerschaum/core/Pipe/_verify.py +4 -4
  141. meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
  142. meerschaum/core/Plugin/__init__.py +1 -1
  143. meerschaum/core/Token/_Token.py +220 -0
  144. meerschaum/core/Token/__init__.py +12 -0
  145. meerschaum/core/User/_User.py +35 -10
  146. meerschaum/core/User/__init__.py +9 -1
  147. meerschaum/core/__init__.py +1 -0
  148. meerschaum/jobs/_Executor.py +88 -4
  149. meerschaum/jobs/_Job.py +149 -38
  150. meerschaum/jobs/__init__.py +3 -2
  151. meerschaum/jobs/systemd.py +8 -3
  152. meerschaum/models/__init__.py +35 -0
  153. meerschaum/models/pipes.py +247 -0
  154. meerschaum/models/tokens.py +38 -0
  155. meerschaum/models/users.py +26 -0
  156. meerschaum/plugins/__init__.py +301 -88
  157. meerschaum/plugins/bootstrap.py +510 -4
  158. meerschaum/utils/_get_pipes.py +97 -30
  159. meerschaum/utils/daemon/Daemon.py +199 -43
  160. meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
  161. meerschaum/utils/daemon/RotatingFile.py +63 -36
  162. meerschaum/utils/daemon/StdinFile.py +53 -13
  163. meerschaum/utils/daemon/__init__.py +47 -6
  164. meerschaum/utils/daemon/_names.py +6 -3
  165. meerschaum/utils/dataframe.py +479 -81
  166. meerschaum/utils/debug.py +49 -19
  167. meerschaum/utils/dtypes/__init__.py +476 -34
  168. meerschaum/utils/dtypes/sql.py +369 -29
  169. meerschaum/utils/formatting/__init__.py +5 -2
  170. meerschaum/utils/formatting/_jobs.py +1 -1
  171. meerschaum/utils/formatting/_pipes.py +52 -50
  172. meerschaum/utils/formatting/_pprint.py +1 -0
  173. meerschaum/utils/formatting/_shell.py +44 -18
  174. meerschaum/utils/misc.py +268 -186
  175. meerschaum/utils/packages/__init__.py +25 -40
  176. meerschaum/utils/packages/_packages.py +42 -34
  177. meerschaum/utils/pipes.py +213 -0
  178. meerschaum/utils/process.py +2 -2
  179. meerschaum/utils/prompt.py +175 -144
  180. meerschaum/utils/schedule.py +2 -1
  181. meerschaum/utils/sql.py +134 -47
  182. meerschaum/utils/threading.py +42 -0
  183. meerschaum/utils/typing.py +1 -4
  184. meerschaum/utils/venv/_Venv.py +2 -2
  185. meerschaum/utils/venv/__init__.py +7 -7
  186. meerschaum/utils/warnings.py +19 -13
  187. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
  188. meerschaum-3.0.0.dist-info/RECORD +289 -0
  189. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
  190. meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
  191. meerschaum/api/models/_interfaces.py +0 -15
  192. meerschaum/api/models/_locations.py +0 -15
  193. meerschaum/api/models/_metrics.py +0 -15
  194. meerschaum/config/_environment.py +0 -145
  195. meerschaum/config/static/__init__.py +0 -186
  196. meerschaum-2.9.5.dist-info/RECORD +0 -263
  197. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
  198. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
  199. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
  200. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
@@ -12,18 +12,26 @@ import shlex
12
12
  from textwrap import dedent
13
13
  from urllib.parse import urlencode
14
14
 
15
+ from meerschaum.utils import fetch_pipes_keys
15
16
  from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
16
- from meerschaum.utils.misc import string_to_dict
17
+ from meerschaum.utils.misc import get_connector_labels
18
+ from meerschaum.connectors import instance_types
17
19
  from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
18
20
  from meerschaum.utils.sql import get_pd_type
19
21
  from meerschaum.utils.yaml import yaml
20
22
  from meerschaum.utils.warnings import warn
21
- from meerschaum.utils.dataframe import to_json
23
+ from meerschaum.utils.dataframe import to_json, to_simple_lines
22
24
  from meerschaum.connectors.sql._fetch import get_pipe_query
23
- from meerschaum.api import CHECK_UPDATE
25
+ from meerschaum.api import CHECK_UPDATE, get_api_connector
24
26
  from meerschaum.api.dash import debug, _get_pipes
25
27
  from meerschaum.api.dash.connectors import get_web_connector
26
- from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
28
+ from meerschaum.api.dash.components import (
29
+ alert_from_success_tuple,
30
+ build_cards_grid,
31
+ sign_out_button,
32
+ logo_row,
33
+ pages_offcanvas,
34
+ )
27
35
  from meerschaum.api.dash.sessions import is_session_authenticated
28
36
  from meerschaum.config import get_config
29
37
  import meerschaum as mrsm
@@ -58,28 +66,20 @@ def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, N
58
66
 
59
67
  def keys_from_state(
60
68
  state: Dict[str, Any],
61
- with_params: bool = False
69
+ with_tags: bool = False,
62
70
  ) -> Union[
63
71
  Tuple[List[str], List[str], List[str]],
64
- Tuple[List[str], List[str], List[str], str],
72
+ Tuple[List[str], List[str], List[str], List[str]],
65
73
  ]:
66
74
  """
67
75
  Read the current state and return the selected keys lists.
68
76
  """
69
77
  _filters = {
70
- 'ck' : state.get(f"connector-keys-{state['pipes-filter-tabs.active_tab']}.value", None),
71
- 'mk' : state.get(f"metric-keys-{state['pipes-filter-tabs.active_tab']}.value", None),
72
- 'lk' : state.get(f"location-keys-{state['pipes-filter-tabs.active_tab']}.value", None),
78
+ 'ck': state.get("connector-keys-dropdown.value", None),
79
+ 'mk': state.get("metric-keys-dropdown.value", None),
80
+ 'lk': state.get("location-keys-dropdown.value", None),
81
+ 'tags': state.get("tags-dropdown.value", None),
73
82
  }
74
- if state['pipes-filter-tabs.active_tab'] == 'input':
75
- try:
76
- # params = string_to_dict(state['params-textarea.value'])
77
- params = string_to_dict(state['search-parameters-editor.value'])
78
- except Exception:
79
- params = None
80
- else:
81
- params = None
82
-
83
83
  for k in _filters:
84
84
  _filters[k] = [] if _filters[k] is None else _filters[k]
85
85
  if not isinstance(_filters[k], list):
@@ -89,8 +89,8 @@ def keys_from_state(
89
89
  print(e)
90
90
  _filters[k] = []
91
91
  keys = [_filters['ck'], _filters['mk'], _filters['lk']]
92
- if with_params:
93
- keys.append(params)
92
+ if with_tags:
93
+ keys.append(_filters['tags'])
94
94
  return tuple(keys)
95
95
 
96
96
 
@@ -98,12 +98,12 @@ def pipes_from_state(
98
98
  state: Dict[str, Any],
99
99
  **kw
100
100
  ):
101
- _ck, _mk, _lk, _params = keys_from_state(state, with_params=True)
101
+ _ck, _mk, _lk, _tags = keys_from_state(state, with_tags=True)
102
102
  try:
103
103
  _pipes = _get_pipes(
104
104
  _ck, _mk, _lk,
105
- params = _params,
106
- mrsm_instance = get_web_connector(state),
105
+ tags=(_tags or []),
106
+ mrsm_instance=get_web_connector(state),
107
107
  **kw
108
108
  )
109
109
  except Exception as e:
@@ -377,48 +377,48 @@ def accordion_items_from_pipe(
377
377
  html.Th(
378
378
  html.Span(
379
379
  "Key",
380
- id={'type': 'key-table-header', 'id': pipe_meta_str},
380
+ id={'type': 'key-table-header', 'index': pipe_meta_str},
381
381
  style={"textDecoration": "underline", "cursor": "pointer"},
382
382
  ),
383
383
  ),
384
384
  html.Th(
385
385
  html.Span(
386
386
  "Column",
387
- id={'type': 'column-table-header', 'id': pipe_meta_str},
387
+ id={'type': 'column-table-header', 'index': pipe_meta_str},
388
388
  style={"textDecoration": "underline", "cursor": "pointer"},
389
389
  ),
390
390
  ),
391
391
  html.Th(
392
392
  html.Span(
393
393
  "Index",
394
- id={'type': 'index-table-header', 'id': pipe_meta_str},
394
+ id={'type': 'index-table-header', 'index': pipe_meta_str},
395
395
  style={"textDecoration": "underline", "cursor": "pointer"},
396
396
  ),
397
397
  ),
398
398
  html.Th(
399
399
  html.Span(
400
400
  "Is Composite",
401
- id={'type': 'is-composite-table-header', 'id': pipe_meta_str},
401
+ id={'type': 'is-composite-table-header', 'index': pipe_meta_str},
402
402
  style={"textDecoration": "underline", "cursor": "pointer"},
403
403
  ),
404
404
  ),
405
405
  dbc.Tooltip(
406
406
  "Unique reference name for the index "
407
407
  "(e.g. `datetime` for the range axis)",
408
- target={'type': 'key-table-header', 'id': pipe_meta_str},
408
+ target={'type': 'key-table-header', 'index': pipe_meta_str},
409
409
  ),
410
410
  dbc.Tooltip(
411
411
  "The actual column (field name) in the target dataset.",
412
- target={'type': 'column-table-header', 'id': pipe_meta_str},
412
+ target={'type': 'column-table-header', 'index': pipe_meta_str},
413
413
  ),
414
414
  dbc.Tooltip(
415
415
  "The name of the index created on the given columns.",
416
- target={'type': 'index-table-header', 'id': pipe_meta_str},
416
+ target={'type': 'index-table-header', 'index': pipe_meta_str},
417
417
  ),
418
418
  dbc.Tooltip(
419
419
  "Whether the column is used in the composite primary key "
420
420
  "to determine updates.",
421
- target={'type': 'is-composite-table-header', 'id': pipe_meta_str},
421
+ target={'type': 'is-composite-table-header', 'index': pipe_meta_str},
422
422
  ),
423
423
  ]
424
424
  )
@@ -466,7 +466,7 @@ def accordion_items_from_pipe(
466
466
  overview_rows.append(
467
467
  html.Tr([
468
468
  html.Td("Indices" if len(indices_rows) != 1 else "Index"),
469
- html.Td(indices_table),
469
+ html.Td(html.Div(indices_table, style={'overflowX': 'auto'})),
470
470
  ])
471
471
  )
472
472
 
@@ -483,9 +483,14 @@ def accordion_items_from_pipe(
483
483
  ])
484
484
  )
485
485
 
486
- items_bodies['overview'] = dbc.Table(
487
- overview_header + [html.Tbody(overview_rows)],
488
- bordered=False, hover=True, striped=False,
486
+ items_bodies['overview'] = html.Div(
487
+ dbc.Table(
488
+ overview_header + [html.Tbody(overview_rows)],
489
+ bordered=False,
490
+ hover=True,
491
+ striped=False,
492
+ ),
493
+ style={'overflowX': 'auto'},
489
494
  )
490
495
 
491
496
  if 'stats' in active_items:
@@ -497,17 +502,13 @@ def accordion_items_from_pipe(
497
502
  (newest_time - oldest_time) if newest_time is not None and oldest_time is not None
498
503
  else None
499
504
  )
500
- rowcount = pipe.get_rowcount(debug=debug)
501
505
  except Exception:
502
506
  oldest_time = None
503
507
  newest_time = None
504
508
  interval = None
505
- rowcount = None
506
509
 
507
510
  stats_rows = []
508
- if rowcount is not None:
509
- stats_rows.append(html.Tr([html.Td("Row Count"), html.Td(f"{rowcount:,}")]))
510
- if interval is not None:
511
+ if interval is not None and not isinstance(interval, int):
511
512
  stats_rows.append(
512
513
  html.Tr([html.Td("Timespan"), html.Td(humanfriendly.format_timespan(interval))])
513
514
  )
@@ -516,7 +517,39 @@ def accordion_items_from_pipe(
516
517
  if newest_time is not None:
517
518
  stats_rows.append(html.Tr([html.Td("Newest time"), html.Td(str(newest_time))]))
518
519
 
519
- items_bodies['stats'] = dbc.Table(stats_header + [html.Tbody(stats_rows)], hover=True)
520
+ precision = pipe.precision
521
+ if precision:
522
+ stats_rows.append(
523
+ html.Tr([
524
+ html.Td("Precision"),
525
+ html.Td(str(precision.get('interval', 1)) + ' ' + str(precision.get('unit', 'unit')))
526
+ ])
527
+ )
528
+
529
+ stats_rows.append(
530
+ html.Tr([
531
+ html.Td("Row count"),
532
+ html.Td(
533
+ html.Div(
534
+ dbc.Button(
535
+ "Calculate",
536
+ color='link',
537
+ size='sm',
538
+ style={'text-decoration': 'none'},
539
+ id={'type': 'calculate-rowcount-button', 'index': pipe_meta_str},
540
+ )
541
+ if pipe.exists(debug=debug)
542
+ else '0'
543
+ ),
544
+ id={'type': 'calculate-rowcount-div', 'index': pipe_meta_str},
545
+ )
546
+ ])
547
+ )
548
+
549
+ items_bodies['stats'] = html.Div(
550
+ dbc.Table(stats_header + [html.Tbody(stats_rows)], hover=True),
551
+ style={'overflowX': 'auto'},
552
+ )
520
553
 
521
554
  if 'columns' in active_items:
522
555
  try:
@@ -536,9 +569,9 @@ def accordion_items_from_pipe(
536
569
  ]
537
570
  columns_body = [html.Tbody(columns_rows)]
538
571
  columns_table = dbc.Table(columns_header + columns_body, bordered=False, hover=True)
572
+ items_bodies['columns'] = html.Div(columns_table, style={'overflowX': 'auto'})
539
573
  except Exception:
540
- columns_table = html.P("Could not retrieve columns ― please try again.")
541
- items_bodies['columns'] = columns_table
574
+ items_bodies['columns'] = html.P("Could not retrieve columns ― please try again.")
542
575
 
543
576
  if 'parameters' in active_items:
544
577
  parameters_editor = dash_ace.DashAceEditor(
@@ -546,7 +579,7 @@ def accordion_items_from_pipe(
546
579
  mode='norm',
547
580
  tabSize=4,
548
581
  theme='twilight',
549
- id={'type': 'parameters-editor', 'index': json.dumps(pipe.meta)},
582
+ id={'type': 'parameters-editor', 'index': pipe_meta_str},
550
583
  width='100%',
551
584
  height='500px',
552
585
  readOnly=False,
@@ -558,19 +591,19 @@ def accordion_items_from_pipe(
558
591
  )
559
592
  update_parameters_button = dbc.Button(
560
593
  "Update",
561
- id={'type': 'update-parameters-button', 'index': json.dumps(pipe.meta)},
594
+ id={'type': 'update-parameters-button', 'index': pipe_meta_str},
562
595
  )
563
596
 
564
597
  as_yaml_button = dbc.Button(
565
598
  "YAML",
566
- id={'type': 'parameters-as-yaml-button', 'index': json.dumps(pipe.meta)},
599
+ id={'type': 'parameters-as-yaml-button', 'index': pipe_meta_str},
567
600
  color='link',
568
601
  size='sm',
569
602
  style={'text-decoration': 'none'},
570
603
  )
571
604
  as_json_button = dbc.Button(
572
605
  "JSON",
573
- id={'type': 'parameters-as-json-button', 'index': json.dumps(pipe.meta)},
606
+ id={'type': 'parameters-as-json-button', 'index': pipe_meta_str},
574
607
  color='link',
575
608
  size='sm',
576
609
  style={'text-decoration': 'none', 'margin-left': '10px'},
@@ -579,21 +612,28 @@ def accordion_items_from_pipe(
579
612
  parameters_editor,
580
613
  html.Br(),
581
614
  dbc.Row([
582
- dbc.Col(html.Span(
583
- (
584
- ([update_parameters_button] if authenticated else []) +
585
- [
586
- as_json_button,
587
- as_yaml_button,
588
- ]
589
- )
590
- ), width=4),
591
- dbc.Col([
592
- html.Div(
593
- id={'type': 'update-parameters-success-div', 'index': json.dumps(pipe.meta)}
594
- )
595
- ],
596
- width=True,
615
+ dbc.Col(
616
+ html.Span(
617
+ (
618
+ ([update_parameters_button] if authenticated else []) +
619
+ [
620
+ as_json_button,
621
+ as_yaml_button,
622
+ ]
623
+ )
624
+ ),
625
+ width=4,
626
+ ),
627
+ dbc.Col(
628
+ [
629
+ html.Div(
630
+ id={
631
+ 'type': 'update-parameters-success-div',
632
+ 'index': pipe_meta_str,
633
+ }
634
+ )
635
+ ],
636
+ width=True,
597
637
  )
598
638
  ]),
599
639
  ]
@@ -633,7 +673,7 @@ def accordion_items_from_pipe(
633
673
  mode='sql',
634
674
  tabSize=4,
635
675
  theme='twilight',
636
- id={'type': 'sql-editor', 'index': json.dumps(pipe.meta)},
676
+ id={'type': 'sql-editor', 'index': pipe_meta_str},
637
677
  width='100%',
638
678
  height='500px',
639
679
  readOnly=False,
@@ -645,7 +685,7 @@ def accordion_items_from_pipe(
645
685
  )
646
686
  update_sql_button = dbc.Button(
647
687
  "Update",
648
- id={'type': 'update-sql-button', 'index': json.dumps(pipe.meta)},
688
+ id={'type': 'update-sql-button', 'index': pipe_meta_str},
649
689
  )
650
690
  items_bodies['sql'] = html.Div([
651
691
  sql_editor,
@@ -654,7 +694,7 @@ def accordion_items_from_pipe(
654
694
  dbc.Col([update_sql_button], width=2),
655
695
  dbc.Col([
656
696
  html.Div(
657
- id={'type': 'update-sql-success-div', 'index': json.dumps(pipe.meta)}
697
+ id={'type': 'update-sql-success-div', 'index': pipe_meta_str}
658
698
  )
659
699
  ],
660
700
  width=True,
@@ -665,10 +705,10 @@ def accordion_items_from_pipe(
665
705
  if 'recent-data' in active_items:
666
706
  try:
667
707
  df = pipe.get_backtrack_data(backtrack_minutes=10, limit=10, debug=debug).astype(str)
668
- table = dbc.Table.from_dataframe(df, bordered=False, hover=True)
708
+ table = dbc.Table.from_dataframe(df, bordered=False, hover=True)
709
+ items_bodies['recent-data'] = html.Div(table, style={'overflowX': 'auto'})
669
710
  except Exception:
670
- table = html.P("Could not retrieve recent data.")
671
- items_bodies['recent-data'] = table
711
+ items_bodies['recent-data'] = html.P("Could not retrieve recent data.")
672
712
 
673
713
  if 'query-data' in active_items:
674
714
  query_editor = dash_ace.DashAceEditor(
@@ -676,7 +716,7 @@ def accordion_items_from_pipe(
676
716
  mode='norm',
677
717
  tabSize=4,
678
718
  theme='twilight',
679
- id={'type': 'query-editor', 'index': json.dumps(pipe.meta)},
719
+ id={'type': 'query-editor', 'index': pipe_meta_str},
680
720
  width='100%',
681
721
  height='200px',
682
722
  readOnly=False,
@@ -688,17 +728,17 @@ def accordion_items_from_pipe(
688
728
  )
689
729
  query_data_button = dbc.Button(
690
730
  "Query",
691
- id={'type': 'query-data-button', 'index': json.dumps(pipe.meta)},
731
+ id={'type': 'query-data-button', 'index': pipe_meta_str},
692
732
  )
693
733
 
694
734
  begin_end_input_group = dbc.InputGroup(
695
735
  [
696
736
  dbc.Input(
697
- id={'type': 'query-data-begin-input', 'index': json.dumps(pipe.meta)},
737
+ id={'type': 'query-data-begin-input', 'index': pipe_meta_str},
698
738
  placeholder="Begin",
699
739
  ),
700
740
  dbc.Input(
701
- id={'type': 'query-data-end-input', 'index': json.dumps(pipe.meta)},
741
+ id={'type': 'query-data-end-input', 'index': pipe_meta_str},
702
742
  placeholder="End",
703
743
  ),
704
744
  ],
@@ -712,9 +752,12 @@ def accordion_items_from_pipe(
712
752
  value=10,
713
753
  step=1,
714
754
  placeholder="Limit",
715
- id={'type': 'limit-input', 'index': json.dumps(pipe.meta)},
755
+ id={'type': 'limit-input', 'index': pipe_meta_str},
756
+ )
757
+ query_result_div = html.Div(
758
+ id={'type': 'query-result-div', 'index': pipe_meta_str},
759
+ style={'overflowX': 'auto'},
716
760
  )
717
- query_result_div = html.Div(id={'type': 'query-result-div', 'index': json.dumps(pipe.meta)})
718
761
 
719
762
  items_bodies['query-data'] = html.Div([
720
763
  query_editor,
@@ -722,7 +765,7 @@ def accordion_items_from_pipe(
722
765
  dbc.Row(
723
766
  [
724
767
  dbc.Col([query_data_button], lg=2, md=3, sm=4, xs=6, width=2),
725
- dbc.Col([begin_end_input_group], lg=3, md=3, sm=4, width=4),
768
+ dbc.Col([begin_end_input_group], lg=6, md=6, sm=4, width=6),
726
769
  dbc.Col(html.Div([limit_input, dbc.FormText("Row Limit")]), lg=2, md=3, sm=4, xs=6, width=2),
727
770
  ],
728
771
  justify="between",
@@ -733,27 +776,13 @@ def accordion_items_from_pipe(
733
776
  ])
734
777
 
735
778
  if 'sync-data' in active_items:
736
- backtrack_df = pipe.get_backtrack_data(debug=debug, limit=1)
737
- try:
738
- json_text = to_json(
739
- backtrack_df,
740
- orient='records',
741
- date_format='iso',
742
- force_ascii=False,
743
- indent=4,
744
- date_unit='us',
745
- ) if backtrack_df is not None else '[]'
746
- except Exception as e:
747
- warn(e)
748
- json_text = '[]'
749
-
750
- json_text = json.dumps(json.loads(json_text), indent=4, separators=(',', ': '))
779
+ backtrack_text = get_backtrack_text(pipe)
751
780
  sync_editor = dash_ace.DashAceEditor(
752
- value = json_text,
781
+ value = backtrack_text,
753
782
  mode = 'norm',
754
783
  tabSize = 4,
755
784
  theme = 'twilight',
756
- id = {'type': 'sync-editor', 'index': json.dumps(pipe.meta)},
785
+ id = {'type': 'sync-editor', 'index': pipe_meta_str},
757
786
  width = '100%',
758
787
  height = '500px',
759
788
  readOnly = False,
@@ -763,16 +792,40 @@ def accordion_items_from_pipe(
763
792
  wrapEnabled = True,
764
793
  style = {'min-height': '120px'},
765
794
  )
795
+
796
+ sync_as_json_button = dbc.Button(
797
+ "JSON",
798
+ id={'type': 'sync-as-json-button', 'index': pipe_meta_str},
799
+ color='link',
800
+ size='sm',
801
+ style={'text-decoration': 'none', 'margin-left': '10px'},
802
+ )
803
+ sync_as_lines_button = dbc.Button(
804
+ "Lines",
805
+ id={'type': 'sync-as-lines-button', 'index': pipe_meta_str},
806
+ color='link',
807
+ size='sm',
808
+ style={'text-decoration': 'none', 'margin-left': '10px'},
809
+ )
810
+
766
811
  update_sync_button = dbc.Button(
767
812
  "Sync",
768
- id = {'type': 'update-sync-button', 'index': json.dumps(pipe.meta)},
813
+ id = {'type': 'update-sync-button', 'index': pipe_meta_str},
769
814
  )
770
- sync_success_div = html.Div(id={'type': 'sync-success-div', 'index': json.dumps(pipe.meta)})
815
+ sync_success_div = html.Div(id={'type': 'sync-success-div', 'index': pipe_meta_str})
771
816
  items_bodies['sync-data'] = html.Div([
772
817
  sync_editor,
773
818
  html.Br(),
774
819
  dbc.Row([
775
- dbc.Col([update_sync_button], width=1),
820
+ dbc.Col(html.Span(
821
+ (
822
+ ([update_sync_button] if authenticated else []) +
823
+ [
824
+ sync_as_json_button,
825
+ sync_as_lines_button,
826
+ ]
827
+ )
828
+ ), width=4),
776
829
  dbc.Col([sync_success_div], width=True),
777
830
  ]),
778
831
  ])
@@ -782,3 +835,237 @@ def accordion_items_from_pipe(
782
835
  for item_id, title in items_titles.items()
783
836
  ]
784
837
 
838
+
839
+ def get_backtrack_text(
840
+ pipe: mrsm.Pipe,
841
+ lines: bool = False,
842
+ limit: int = 5,
843
+ ) -> str:
844
+ """
845
+ Return the backtrack documents as text for the sync editor.
846
+ """
847
+ backtrack_df = pipe.get_backtrack_data(debug=debug, limit=limit)
848
+ if lines:
849
+ return to_simple_lines(backtrack_df)
850
+ try:
851
+ json_text = to_json(
852
+ backtrack_df,
853
+ orient='records',
854
+ date_format='iso',
855
+ force_ascii=False,
856
+ indent=4,
857
+ date_unit='us',
858
+ ) if backtrack_df is not None else '[]'
859
+ except Exception as e:
860
+ warn(e)
861
+ json_text = '[]'
862
+
863
+ return json.dumps(json.loads(json_text), indent=4, separators=(',', ': '))
864
+
865
+
866
+ def build_pipes_dropdown_keys_row(
867
+ connector_keys: List[str],
868
+ metric_keys: List[str],
869
+ location_keys: List[str],
870
+ tags: List[str],
871
+ pipes: List[mrsm.Pipe],
872
+ instance_connector: mrsm.connectors.InstanceConnector,
873
+ ) -> dbc.Row:
874
+ """
875
+ Return the dropdown keys row for the dedicated pipes page.
876
+ """
877
+ ck_alone = connector_keys and not any([str(x) for x in (tags + metric_keys + location_keys)])
878
+ mk_alone = metric_keys and not any([str(x) for x in (connector_keys + tags + location_keys)])
879
+ lk_alone = location_keys and not any([str(x) for x in (connector_keys + metric_keys + tags)])
880
+ all_keys = fetch_pipes_keys('registered', instance_connector)
881
+
882
+ ck_options_source = (
883
+ {keys_tuple[0] for keys_tuple in all_keys}
884
+ if ck_alone
885
+ else {p.connector_keys for p in pipes}
886
+ )
887
+ ck_options = sorted(ck_options_source.union(connector_keys))
888
+
889
+ mk_options_source = (
890
+ {keys_tuple[1] for keys_tuple in all_keys}
891
+ if mk_alone
892
+ else {p.metric_key for p in pipes}
893
+ )
894
+ mk_options = sorted(mk_options_source.union(metric_keys))
895
+
896
+ lk_options_source = (
897
+ {str(keys_tuple[2]) for keys_tuple in all_keys}
898
+ if lk_alone
899
+ else {str(p.location_key) for p in pipes}
900
+ )
901
+ lk_options = sorted(lk_options_source.union({str(lk) for lk in location_keys}))
902
+
903
+ return dbc.Row(
904
+ [
905
+ dbc.Col(
906
+ html.Div(
907
+ [
908
+ dcc.Dropdown(
909
+ id='pipes-connector-keys-dropdown',
910
+ options=ck_options,
911
+ value=[str(ck) for ck in connector_keys],
912
+ placeholder='Connectors',
913
+ multi=True,
914
+ ),
915
+ ],
916
+ className='dbc_dark',
917
+ ),
918
+ lg=4,
919
+ md=12,
920
+ sm=12,
921
+ ),
922
+ dbc.Col(
923
+ html.Div(
924
+ [
925
+ dcc.Dropdown(
926
+ id='pipes-metric-keys-dropdown',
927
+ options=mk_options,
928
+ value=[str(mk) for mk in metric_keys],
929
+ placeholder='Metrics',
930
+ multi=True,
931
+ ),
932
+ ],
933
+ className='dbc_dark'
934
+ ),
935
+ lg=4,
936
+ md=12,
937
+ sm=12,
938
+ ),
939
+ dbc.Col(
940
+ html.Div(
941
+ [
942
+ dcc.Dropdown(
943
+ id='pipes-location-keys-dropdown',
944
+ options=lk_options,
945
+ value=[str(lk) for lk in location_keys],
946
+ placeholder='Locations',
947
+ multi=True,
948
+ ),
949
+ ],
950
+ className='dbc_dark'
951
+ ),
952
+ lg=4,
953
+ md=12,
954
+ sm=12,
955
+ ),
956
+ ] ### end of filters row children
957
+ )
958
+
959
+
960
+ def build_pipes_tags_dropdown(
961
+ connector_keys: List[str],
962
+ metric_keys: List[str],
963
+ location_keys: List[str],
964
+ tags: List[str],
965
+ instance: str,
966
+ ) -> dbc.Row:
967
+ """
968
+ Build the tags dropdown for the dedicated pipes page.
969
+ """
970
+ _tags_alone = tags and not any([str(x) for x in (connector_keys + metric_keys + location_keys)])
971
+ _tags_pipes = mrsm.get_pipes(
972
+ connector_keys=connector_keys,
973
+ metric_keys=metric_keys,
974
+ location_keys=location_keys,
975
+ tags=tags,
976
+ instance=instance,
977
+ as_tags_dict=True,
978
+ )
979
+
980
+ _all_tags = list(
981
+ mrsm.get_pipes(
982
+ instance=instance,
983
+ as_tags_dict=True,
984
+ )
985
+ ) if _tags_alone else []
986
+
987
+ tags_options = [
988
+ str(tag)
989
+ for tag in (_all_tags if _tags_alone else _tags_pipes)
990
+ ]
991
+ if tags:
992
+ tags_options += [tag for tag in tags if tag not in tags_options]
993
+
994
+ return dbc.Row(
995
+ [
996
+ dbc.Col(
997
+ html.Div(
998
+ dcc.Dropdown(
999
+ id='pipes-tags-dropdown',
1000
+ options=tags_options,
1001
+ value=tags,
1002
+ placeholder='Tags',
1003
+ multi=True,
1004
+ searchable=True,
1005
+ ),
1006
+ className="dbc_dark",
1007
+ id="pipes-tags-dropdown-div",
1008
+ ),
1009
+ width=True,
1010
+ ),
1011
+ dbc.Col(
1012
+ dbc.Button(
1013
+ "Clear all",
1014
+ color='link',
1015
+ size='sm',
1016
+ style={'text-decoration': 'none'},
1017
+ id='pipes-clear-all-button',
1018
+ ),
1019
+ width='auto',
1020
+ ),
1021
+ ],
1022
+ className='g-0',
1023
+ align='center',
1024
+ )
1025
+
1026
+
1027
+ def build_pipes_navbar(instance_keys: Optional[str] = None, with_instance_select: bool = True):
1028
+ """
1029
+ Build the navbar from the selected instance keys.
1030
+ """
1031
+ instance_select = dbc.Select(
1032
+ id='instance-select',
1033
+ size='sm',
1034
+ value=instance_keys or str(get_api_connector()),
1035
+ options=[
1036
+ {'label': (i[:32] + '…') if len(i) > 32 else i, 'value': i}
1037
+ for i in get_connector_labels(*instance_types)
1038
+ ],
1039
+ class_name='dbc_dark custom-select custom-select-sm',
1040
+ )
1041
+ instance_select_div_style = {} if with_instance_select else {'visibility': 'hidden'}
1042
+ instance_select_div = html.Div(instance_select, style=instance_select_div_style)
1043
+ return html.Div(
1044
+ [
1045
+ pages_offcanvas,
1046
+ dbc.Navbar(
1047
+ dbc.Container(
1048
+ [
1049
+ logo_row,
1050
+ dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
1051
+ dbc.Collapse(
1052
+ dbc.Row(
1053
+ [
1054
+ dbc.Col(instance_select_div, width='auto'),
1055
+ dbc.Col(sign_out_button, width='auto'),
1056
+ ],
1057
+ className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",
1058
+ align='center',
1059
+ ),
1060
+ id='navbar-collapse',
1061
+ is_open=False,
1062
+ navbar=True,
1063
+ ),
1064
+ ]
1065
+ ),
1066
+ dark=True,
1067
+ color='dark'
1068
+ ),
1069
+ ],
1070
+ id='pages-navbar-div',
1071
+ )