meerschaum 2.2.0rc1__py3-none-any.whl → 2.2.0rc2__py3-none-any.whl

meerschaum/__main__.py

@@ -4,7 +4,7 @@
 # vim:fenc=utf-8
 
 """
-Copyright 2021 Bennett Meares
+Copyright 2024 Bennett Meares
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

meerschaum/actions/show.py

@@ -578,7 +578,7 @@ def _show_logs(
         `show logs myjob myotherjob`
     """
     import os, pathlib, random, asyncio
-    from datetime import datetime
+    from datetime import datetime, timezone
     from meerschaum.utils.packages import attempt_import, import_rich
     from meerschaum.utils.daemon import get_filtered_daemons, Daemon
     from meerschaum.utils.warnings import warn, info
@@ -589,81 +589,106 @@ def _show_logs(
     if not ANSI:
         info = print
     colors = get_config('jobs', 'logs', 'colors')
+    timestamp_format = get_config('jobs', 'logs', 'timestamp_format')
+    follow_timestamp_format = get_config('jobs', 'logs', 'follow_timestamp_format')
     daemons = get_filtered_daemons(action)
-
-    def _build_buffer_spaces(daemons) -> Dict[str, str]:
-        _max_len_id = max([len(d.daemon_id) for d in daemons]) if daemons else 0
-        _buffer_len = max(get_config('jobs', 'logs', 'min_buffer_len'), _max_len_id + 2)
+    now = datetime.now(timezone.utc)
+    now_str = now.strftime(timestamp_format)
+    now_follow_str = now.strftime(follow_timestamp_format)
+
+    def build_buffer_spaces(daemons) -> Dict[str, str]:
+        max_len_id = max(len(d.daemon_id) for d in daemons) if daemons else 0
+        buffer_len = max(
+            get_config('jobs', 'logs', 'min_buffer_len'),
+            max_len_id 
+        )
         return {
-            d.daemon_id: ''.join([' ' for i in range(_buffer_len - len(d.daemon_id))])
+            d.daemon_id: ''.join([' ' for i in range(buffer_len - len(d.daemon_id))])
             for d in daemons
         }
 
-    def _build_job_colors(daemons, _old_job_colors = None) -> Dict[str, str]:
+    def build_job_colors(daemons, _old_job_colors = None) -> Dict[str, str]:
         return {d.daemon_id: colors[i % len(colors)] for i, d in enumerate(daemons)}
 
-    _buffer_spaces = _build_buffer_spaces(daemons)
-    _job_colors = _build_job_colors(daemons)
+    buffer_spaces = build_buffer_spaces(daemons)
+    job_colors = build_job_colors(daemons)
 
-    def _get_buffer_spaces(daemon_id):
-        nonlocal _buffer_spaces, daemons
-        if daemon_id not in _buffer_spaces:
+    def get_buffer_spaces(daemon_id):
+        nonlocal buffer_spaces, daemons
+        if daemon_id not in buffer_spaces:
             d = Daemon(daemon_id=daemon_id)
             if d not in daemons:
                 daemons = get_filtered_daemons(action)
-            _buffer_spaces = _build_buffer_spaces(daemons)
-        return _buffer_spaces[daemon_id]
+            buffer_spaces = build_buffer_spaces(daemons)
+        return buffer_spaces[daemon_id] or ' '
 
-    def _get_job_colors(daemon_id):
-        nonlocal _job_colors, daemons
-        if daemon_id not in _job_colors:
+    def get_job_colors(daemon_id):
+        nonlocal job_colors, daemons
+        if daemon_id not in job_colors:
             d = Daemon(daemon_id=daemon_id)
             if d not in daemons:
                 daemons = get_filtered_daemons(action)
-            _job_colors = _build_job_colors(daemons)
-        return _job_colors[daemon_id]
+            job_colors = build_job_colors(daemons)
+        return job_colors[daemon_id]
 
-    def _follow_pretty_print():
+    def follow_pretty_print():
         watchfiles = attempt_import('watchfiles')
         rich = import_rich()
         rich_text = attempt_import('rich.text')
-        _watch_daemon_ids = {d.daemon_id: d for d in daemons}
+        watch_daemon_ids = {d.daemon_id: d for d in daemons}
         info("Watching log files...")
 
-        def _print_job_line(daemon, line):
-            date_prefix_str = line[:len('YYYY-MM-DD HH:mm')]
+        previous_line_timestamp = None
+        def print_job_line(daemon, line):
+            nonlocal previous_line_timestamp
+            date_prefix_str = line[:len(now_str)]
             try:
-                line_timestamp = datetime.fromisoformat(date_prefix_str)
+                line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
+                previous_line_timestamp = line_timestamp
             except Exception as e:
                 line_timestamp = None
             if line_timestamp:
-                line = line[len('YYYY-MM-DD HH:mm | '):]
-            if len(line) == 0:
+                line = line[(len(now_str) + 3):]
+            else:
+                line_timestamp = previous_line_timestamp
+
+            if len(line) == 0 or line == '\n':
                 return
+
             text = rich_text.Text(daemon.daemon_id)
-            text.append(
-                _get_buffer_spaces(daemon.daemon_id) + '| '
-                + (line[:-1] if line[-1] == '\n' else line)
+            line_prefix = (
+                get_buffer_spaces(daemon.daemon_id)
+                + (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
+                + ' | '
             )
+            text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
             if ANSI:
                 text.stylize(
-                    _get_job_colors(daemon.daemon_id),
+                    get_job_colors(daemon.daemon_id),
                     0,
-                    len(daemon.daemon_id) + len(_get_buffer_spaces(daemon.daemon_id)) + 1
+                    len(daemon.daemon_id) + len(line_prefix)
                 )
             get_console().print(text)
 
 
-        def _print_log_lines(daemon):
+        def print_log_lines(daemon):
             for line in daemon.readlines():
-                _print_job_line(daemon, line)
+                print_job_line(daemon, line)
 
-        def _seek_back_offset(d) -> bool:
+        def seek_back_offset(d) -> bool:
             if d.log_offset_path.exists():
                 d.log_offset_path.unlink()
 
             latest_subfile_path = d.rotating_log.get_latest_subfile_path()
             latest_subfile_index = d.rotating_log.get_latest_subfile_index()
+
+            ### Sometimes the latest file is empty.
+            if os.stat(latest_subfile_path).st_size == 0 and latest_subfile_index > 0:
+                latest_subfile_index -= 1
+                latest_subfile_path = d.rotating_log.get_subfile_path_from_index(
+                    latest_subfile_index
+                )
+
             with open(latest_subfile_path, 'r', encoding='utf-8') as f:
                 latest_lines = f.readlines()
 
@@ -683,11 +708,11 @@ def _show_logs(
             for d in daemons
         }
         for d in daemons:
-            _seek_back_offset(d)
-            _print_log_lines(d)
+            seek_back_offset(d)
+            print_log_lines(d)
 
         _quit = False
-        def _watch_logs():
+        def watch_logs():
             for changes in watchfiles.watch(LOGS_RESOURCES_PATH):
                 if _quit:
                     return
@@ -699,7 +724,7 @@ def _show_logs(
                     if not file_path.exists():
                         continue
                     daemon_id = file_path.name.split('.log')[0]
-                    if daemon_id not in _watch_daemon_ids and action:
+                    if daemon_id not in watch_daemon_ids and action:
                         continue
                     try:
                         daemon = Daemon(daemon_id=daemon_id)
@@ -708,21 +733,21 @@ def _show_logs(
                         warn(f"Seeing new logs for non-existent job '{daemon_id}'.", stack=False)
 
                     if daemon is not None:
-                        _print_log_lines(daemon)
+                        print_log_lines(daemon)
 
         try:
-            _watch_logs()
+            watch_logs()
         except KeyboardInterrupt as ki:
             _quit = True
 
-    def _print_nopretty_log_text():
+    def print_nopretty_log_text():
         for d in daemons:
             log_text = d.log_text
             print(d.daemon_id)
             print(log_text)
 
-    _print_log_text = _follow_pretty_print if not nopretty else _print_nopretty_log_text
-    _print_log_text()
+    print_log_text = follow_pretty_print if not nopretty else print_nopretty_log_text
+    print_log_text()
 
     return True, "Success"
 

meerschaum/api/dash/callbacks/dashboard.py

@@ -440,18 +440,13 @@ def update_flags(input_flags_dropdown_values, n_clicks, input_flags_texts):
             className = 'input-text',
         )
 
+    remove_index = trigger_dict['index'] if trigger_type == 'input-flags-remove-button' else None
     rows = [
         build_row(i, val, val_text)
         for i, (val, val_text) in enumerate(zip(input_flags_dropdown_values, input_flags_texts))
+        if i != remove_index
     ]
 
-    if trigger_type == 'input-flags-remove-button':
-        remove_index = trigger_dict['index']
-        try:
-            del rows[remove_index]
-        except IndexError:
-            pass
-
     if not rows or input_flags_dropdown_values[-1]:
         rows.append(build_row(len(rows), None, None))
 

meerschaum/api/dash/pipes.py

@@ -24,6 +24,7 @@ from meerschaum.api.dash import (
 from meerschaum.api.dash.connectors import get_web_connector
 from meerschaum.api.dash.components import alert_from_success_tuple
 from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.config import get_config
 import meerschaum as mrsm
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 dash_ace = attempt_import('dash_ace', lazy=False, check_update=CHECK_UPDATE)
@@ -110,12 +111,16 @@ def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None):
     session_id = (session_data or {}).get('session-id', None)
     authenticated = is_session_authenticated(str(session_id))
 
-    _pipes = pipes_from_state(*keys, as_list=True)
-    alerts = [alert_from_success_tuple(_pipes)]
-    if not isinstance(_pipes, list):
-        _pipes = []
-    for p in _pipes:
-        meta_str = json.dumps(p.meta)
+    pipes = pipes_from_state(*keys, as_list=True)
+    alerts = [alert_from_success_tuple(pipes)]
+    if not isinstance(pipes, list):
+        pipes = []
+
+    max_num_pipes_cards = get_config('dash', 'max_num_pipes_cards')
+    overflow_pipes = pipes[max_num_pipes_cards:]
+
+    for pipe in pipes[:max_num_pipes_cards]:
+        meta_str = json.dumps(pipe.meta)
         footer_children = dbc.Row(
             [
                 dbc.Col(
@@ -188,13 +193,13 @@ def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None):
         )
         card_body_children = [
             html.H5(
-                html.B(str(p)),
+                html.B(str(pipe)),
                 className = 'card-title',
                 style = {'font-family': ['monospace']}
             ),
             html.Div(
                 dbc.Accordion(
-                    accordion_items_from_pipe(p, authenticated=authenticated),
+                    accordion_items_from_pipe(pipe, authenticated=authenticated),
                     flush = True,
                     start_collapsed = True,
                     id = {'type': 'pipe-accordion', 'index': meta_str},
@@ -203,11 +208,30 @@ def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None):
 
         ]
         cards.append(
-            dbc.Card(children=[
+            dbc.Card([
                 dbc.CardBody(children=card_body_children),
                 dbc.CardFooter(children=footer_children),
             ])
         )
+
+    if overflow_pipes:
+        cards.append(
+            dbc.Card([
+                dbc.CardBody(
+                    html.Ul(
+                        [
+                            html.Li(html.H5(
+                                html.B(str(pipe)),
+                                className = 'card-title',
+                                style = {'font-family': ['monospace']}
+                            ))
+                            for pipe in overflow_pipes
+                        ]
+                    )
+                )
+            ])
+        )
+
     return cards, alerts
 
 

meerschaum/api/dash/plugins.py

@@ -32,17 +32,22 @@ def get_plugins_cards(
         desc = get_api_connector().get_plugin_attributes(plugin).get(
             'description', 'No description provided.'
         )
-        desc_textarea_kw = dict(
-            value=desc, readOnly=True, debounce=False, className='plugin-description',
-            draggable=False, wrap='overflow',
-            id={'type': 'description-textarea', 'index': plugin_name},
-        )
+        desc_textarea_kw = {
+            'value': desc,
+            'readOnly': True,
+            'debounce': False,
+            'className': 'plugin-description',
+            'draggable': False,
+            'wrap': 'overflow',
+            'placeholder': "Edit the plugin's description",
+            'id': {'type': 'description-textarea', 'index': plugin_name},
+        }
 
         card_body_children = [html.H4(plugin_name)]
 
         if is_plugin_owner(plugin_name, session_data):
             desc_textarea_kw['readOnly'] = False
-        card_body_children += [dbc.Textarea(**desc_textarea_kw)]
+        card_body_children.append(dbc.Textarea(**desc_textarea_kw))
         if not desc_textarea_kw['readOnly']:
             card_body_children += [
                 dbc.Button(
@@ -53,12 +58,23 @@ def get_plugins_cards(
                 ),
                 html.Div(id={'type': 'edit-alert-div', 'index': plugin_name}),
             ]
-        _plugin_username = get_api_connector().get_plugin_username(plugin, debug=debug)
+        plugin_username = get_api_connector().get_plugin_username(plugin, debug=debug)
+        plugin_version = get_api_connector().get_plugin_version(plugin, debug=debug) or ' '
         card_children = [
-            dbc.CardHeader([html.A('👤 ' + str(_plugin_username), href='#')]),
+            dbc.CardHeader(
+                [
+                    dbc.Row(
+                        [
+                            dbc.Col(html.A('👤 ' + str(plugin_username), href='#')),
+                            dbc.Col(html.Pre(str(plugin_version), style={'text-align': 'right'})),
+                        ],
+                        justify = 'between',
+                    ),
+                ],
+            ),
             dbc.CardBody(card_body_children),
             dbc.CardFooter([
-                html.A('⬇️ Download source', href=(endpoints['plugins'] + '/' + plugin_name))
+                html.A('⬇️ Download', href=(endpoints['plugins'] + '/' + plugin_name))
             ]),
         ]
         cards.append(

meerschaum/api/resources/templates/termpage.html

@@ -70,6 +70,9 @@ window.addEventListener(
         if (!fl){ continue; }
         fl_val = event.data['input_flags_texts'][index];
         if (!fl_val){ continue; }
+        if (fl_val.includes(' ')){
+          fl_val = "'" + fl_val + "'";
+        }
         flags_str += " " + fl + " " + fl_val;
     }
 

meerschaum/api/routes/_login.py

@@ -39,10 +39,11 @@ def login(
         else (data.username, data.password)
     ) if not no_auth else ('no-auth', 'no-auth')
 
-    from meerschaum.core.User._User import get_pwd_context
+    from meerschaum.core.User._User import verify_password
     user = User(username, password)
-    correct_password = no_auth or get_pwd_context().verify(
-        password, get_api_connector().get_user_password_hash(user, debug=debug)
+    correct_password = no_auth or verify_password(
+        password,
+        get_api_connector().get_user_password_hash(user, debug=debug)
     )
     if not correct_password:
         raise InvalidCredentialsException
@@ -51,7 +52,7 @@ def login(
     expires_delta = timedelta(minutes=expires_minutes)
     expires_dt = datetime.now(timezone.utc).replace(tzinfo=None) + expires_delta
     access_token = manager.create_access_token(
-        data = dict(sub=username),
+        data = {'sub': username},
         expires = expires_delta
     )
     return {

meerschaum/api/routes/_plugins.py

@@ -90,18 +90,21 @@ def register_plugin(
             pass
 
     plugin = Plugin(name, version=version, attributes=attributes)
+    if curr_user is None:
+        return (
+            False,
+            "Cannot register a plugin without logging in (are you running with `--insecure`)?"
+        )
+
     if curr_user is not None:
         plugin_user_id = get_api_connector().get_plugin_user_id(plugin)
         curr_user_id = get_api_connector().get_user_id(curr_user) if curr_user is not None else -1
         if plugin_user_id is not None and plugin_user_id != curr_user_id:
             return False, f"User '{curr_user.username}' cannot edit plugin '{plugin}'."
         plugin.user_id = curr_user_id
-    else:
-        plugin.user_id = -1
 
     success, msg = get_api_connector().register_plugin(plugin, make_archive=False, debug=debug)
 
-    ### TODO delete and install new version of plugin on success
     if success:
         archive_path = plugin.archive_path
         temp_archive_path = pathlib.Path(str(archive_path) + '.tmp')

meerschaum/config/_dash.py

@@ -0,0 +1,11 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Define default configuration for the Dash application.
+"""
+
+default_dash_config = {
+    'max_num_pipes_cards': 20,
+}

meerschaum/config/_default.py

@@ -44,6 +44,7 @@ default_meerschaum_config = {
             },
             'local': {
                 'host': 'localhost',
+                'port': 8000,
             },
             'mrsm': {
                 'host': 'api.mrsm.io',
@@ -151,7 +152,6 @@ default_config['pipes'] = default_pipes_config
 default_config['plugins'] = default_plugins_config
 from meerschaum.config._jobs import default_jobs_config
 default_config['jobs'] = default_jobs_config
-#  default_config['experimental'] = default_experimental_config
 ### add configs from other packages
 try:
     import meerschaum.config.stack
@@ -160,6 +160,8 @@ except ImportError as e:
 finally:
     from meerschaum.config.stack import default_stack_config
 default_config['stack'] = default_stack_config
+from meerschaum.config._dash import default_dash_config
+default_config['dash'] = default_dash_config
 
 default_header_comment = """
 #####################################################################

meerschaum/config/_jobs.py

@@ -9,13 +9,19 @@ Default configuration for jobs.
 default_jobs_config = {
     'timeout_seconds': 8,
     'check_timeout_interval_seconds': 0.1,
-    'logs' : {
+    'terminal': {
+        'lines': 40,
+        'columns': 70,
+    },
+    'logs': {
         'num_files_to_keep': 5,
         'max_file_size': 100_000,
         'lines_to_show': 30,
-        'refresh_files_seconds': 5.0,
-        'min_buffer_len': 15,
-        'colors' : [
+        'refresh_files_seconds': 5,
+        'min_buffer_len': 10,
+        'timestamp_format': '%Y-%m-%d %H:%M',
+        'follow_timestamp_format': '%H:%M',
+        'colors': [
             'cyan',
             'magenta',
             'orange3',

meerschaum/config/_paths.py

@@ -153,6 +153,7 @@ paths = {
 
     'DAEMON_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'jobs'),
     'LOGS_RESOURCES_PATH'            : ('{ROOT_DIR_PATH}', 'logs'),
+    'DAEMON_ERROR_LOG_PATH'          : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
 }
 
 def set_root(root: Union[Path, str]):

meerschaum/config/_sync.py

@@ -52,7 +52,7 @@ def sync_yaml_configs(
         if not path.exists():
             return "", {}
         header_comment = ""
-        with open(path, 'r') as f:
+        with open(path, 'r', encoding='utf-8') as f:
             if _yaml is not None:
                 config = yaml.load(f)
             else:
@@ -84,7 +84,7 @@ def sync_yaml_configs(
     new_path = sub_path
 
     ### write changes
-    with open(new_path, 'w+') as f:
+    with open(new_path, 'w+', encoding='utf-8') as f:
         f.write(new_header)
         f.write(new_config_text)
     if permissions is not None:
@@ -133,4 +133,3 @@ def sync_files(keys: Optional[List[str]] = None):
     for k in keys:
         if k in key_functions:
             key_functions[k]()
-

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.2.0rc1"
+__version__ = "2.2.0rc2"

meerschaum/config/stack/__init__.py

@@ -33,7 +33,7 @@ api_host = "api"
 
 env_dict = {
     'COMPOSE_PROJECT_NAME' : 'mrsm',
-    'TIMESCALEDB_VERSION' : 'latest-pg15-oss',
+    'TIMESCALEDB_VERSION' : 'latest-pg16-oss',
     'POSTGRES_USER' : f'{db_user}',
     'POSTGRES_PASSWORD' : f'{db_pass}',
     'POSTGRES_DB' : f'{db_base}',
@@ -232,11 +232,11 @@ NECESSARY_FILES = [STACK_COMPOSE_PATH, GRAFANA_DATASOURCE_PATH, GRAFANA_DASHBOAR
 def get_necessary_files():
     from meerschaum.config import get_config
     return {
-        STACK_COMPOSE_PATH : (
+        STACK_COMPOSE_PATH: (
             get_config('stack', STACK_COMPOSE_FILENAME, substitute=True), compose_header
         ),
-        GRAFANA_DATASOURCE_PATH : get_config('stack', 'grafana', 'datasource', substitute=True),
-        GRAFANA_DASHBOARD_PATH : get_config('stack', 'grafana', 'dashboard', substitute=True),
+        GRAFANA_DATASOURCE_PATH: get_config('stack', 'grafana', 'datasource', substitute=True),
+        GRAFANA_DASHBOARD_PATH: get_config('stack', 'grafana', 'dashboard', substitute=True),
     }
 
 
@@ -250,8 +250,8 @@ def write_stack(
     return sync_files(['stack'])
    
 def edit_stack(
-        action : Optional[List[str]] = None,
-        debug : bool = False,
+        action: Optional[List[str]] = None,
+        debug: bool = False,
         **kw
     ):
     """Open docker-compose.yaml or .env for editing."""

meerschaum/config/stack/grafana/__init__.py

@@ -17,7 +17,7 @@ default_datasource = {
             'type': 'postgres',
             'jsonData': {
                 'sslmode': 'disable',
-                'postgresVersion': 1400,
+                'postgresVersion': 1500,
                 'timescaledb': True,
             },
             'user': db_user,

meerschaum/config/static/__init__.py

@@ -103,11 +103,13 @@ STATIC_CONFIG: Dict[str, Any] = {
     },
     'users': {
         'password_hash': {
+            'algorithm_name': 'sha256',
+            'salt_bytes': 16,
             'schemes': [
                 'pbkdf2_sha256',
             ],
             'default': 'pbkdf2_sha256',
-            'pbkdf2_sha256__default_rounds': 30000,
+            'pbkdf2_sha256__default_rounds': 3_000_000,
         },
         'min_username_length': 1,
         'max_username_length': 26,

meerschaum/connectors/sql/_plugins.py

@@ -108,9 +108,7 @@ def get_plugin_version(
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
     sqlalchemy = attempt_import('sqlalchemy')
-
     query = sqlalchemy.select(plugins_tbl.c.version).where(plugins_tbl.c.plugin_name == plugin.name)
-
     return self.value(query, debug=debug)
 
 def get_plugin_user_id(