meerschaum 2.5.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

meerschaum/_internal/arguments/_parser.py

@@ -90,8 +90,13 @@ def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
     except Exception as e:
         dt = None
     if dt is None:
-        from meerschaum.utils.warnings import warn, error
+        from meerschaum.utils.warnings import error
         error(f"'{dt_str}' is not a valid datetime format.", stack=False)
+
+    if isinstance(dt, datetime):
+        from meerschaum.utils.dtypes import coerce_timezone
+        dt = coerce_timezone(dt)
+
     return dt
 
 

meerschaum/_internal/entry.py

@@ -13,6 +13,7 @@ import os
 import sys
 import pathlib
 
+import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict, Callable, Any
 from meerschaum.config.static import STATIC_CONFIG as _STATIC_CONFIG
 
@@ -229,7 +230,7 @@ def entry_with_args(
         _ = get_shell(**kw).cmdloop()
         return True, "Success"
 
-    skip_schedule = False
+    _skip_schedule = False
 
     executor_keys = kw.get('executor_keys', None)
     if executor_keys is None:
@@ -246,7 +247,7 @@ def entry_with_args(
             api_label = executor_keys.split(':')[-1]
             kw['action'].insert(0, 'api')
             kw['action'].insert(1, api_label)
-            skip_schedule = True
+            _skip_schedule = True
 
     ### If the `--daemon` flag is present, prepend 'start job'.
     if kw.get('daemon', False) and kw['action'][0] != 'stack':
@@ -271,7 +272,7 @@ def entry_with_args(
         and kw['action'][0] == 'start'
         and kw['action'][1] in ('job', 'jobs')
     ):
-        skip_schedule = True
+        _skip_schedule = True
 
     kw['action'] = remove_leading_action(kw['action'], _actions=_actions)
 
@@ -279,10 +280,11 @@ def entry_with_args(
         _do_action_wrapper,
         action_function,
         plugin_name,
+        _skip_schedule=_skip_schedule,
         **kw
     )
 
-    if kw.get('schedule', None) and not skip_schedule:
+    if kw.get('schedule', None) and not _skip_schedule:
         from meerschaum.utils.schedule import schedule_function
         from meerschaum.utils.misc import interval_str
         import time
@@ -304,7 +306,12 @@ def entry_with_args(
     return result
 
 
-def _do_action_wrapper(action_function, plugin_name, **kw):
+def _do_action_wrapper(
+    action_function,
+    plugin_name,
+    _skip_schedule: bool = False,
+    **kw
+):
     from meerschaum.plugins import Plugin
     from meerschaum.utils.venv import Venv
     from meerschaum.utils.misc import filter_keywords
@@ -328,6 +335,10 @@ def _do_action_wrapper(action_function, plugin_name, **kw):
             )
         except KeyboardInterrupt:
             result = False, f"Cancelled action `{action_name.lstrip()}`."
+
+    if kw.get('schedule', None) and not _skip_schedule:
+        mrsm.pprint(result)
+
     return result
 
 _shells = []

meerschaum/actions/edit.py

@@ -65,24 +65,24 @@ def _complete_edit(
     from meerschaum._internal.shell import default_action_completer
     return default_action_completer(action=(['edit'] + action), **kw)
 
-def _edit_config(action : Optional[List[str]] = None, **kw : Any) -> SuccessTuple:
+
+def _edit_config(action: Optional[List[str]] = None, **kw : Any) -> SuccessTuple:
     """
     Edit Meerschaum configuration files.
-    
+
     Specify a specific configuration key to edit.
     Defaults to editing `meerschaum` configuration (connectors, instance, etc.).
-    
+
     Examples:
         ```
         ### Edit the main 'meerschaum' configuration.
         edit config
-    
+
         ### Edit 'system' configuration.
         edit config system
-    
+
         ### Create a new configuration file called 'myconfig'.
         edit config myconfig
-    
         ```
     """
     from meerschaum.config._edit import edit_config

meerschaum/actions/sql.py

@@ -14,6 +14,7 @@ exec_methods = {
     'exec',
 }
 
+
 def sql(
     action: Optional[List[str]] = None,
     gui: bool = False,
@@ -22,40 +23,40 @@ def sql(
     **kw: Any
 ):
     """Execute a SQL query or launch an interactive CLI. All positional arguments are optional.
-    
+
     Usage:
         `sql {label} {method} {query / table}`
-    
+
     Options:
         - `sql {label}`
             Launch an interactive CLI. If {label} is omitted, use 'main'.
-    
+
         - `sql {label} read [query / table]`
             Read a table or query as a pandas DataFrame and print the result.
-    
+
         - `sql {label} exec [query]`
             Execute a query and print the success status.
-    
+
     Examples:
         - `sql`
             Open an interactive CLI for `sql:main`.
-    
+
         - `sql local`
             Open an interactive CLI for `sql:local`.
-    
+
         - `sql table`
             Read from `table` on `sql:main`
               (translates to `SELECT * FROM table`).
-    
+
         - `sql local table`
             Read from `table` on `sql:local`.
-    
+
         - `sql local read table`
             Read from `table` on `sql:local`.
-    
+
         - `sql "SELECT * FROM table WHERE id = 1"`
             Execute the above query on `sql:main` and print the results.
-    
+
         - `sql local exec "INSERT INTO table (id) VALUES (1)"
             Execute the above query on `sql:local`.
     """

meerschaum/api/dash/pages/login.py

@@ -19,13 +19,13 @@ allow_user_registration = permissions_config['registration']['users']
 
 registration_div = html.Div(
     id = 'registration-div',
-    style = {'height' : '100%'},
-    children = (
+    style={'height' : '100%'},
+    children=(
         [
             dcc.Link(
                 'No account? Create one here.',
-                href = (dash_endpoint + '/register'),
-                refresh = False
+                href=(dash_endpoint + '/register'),
+                refresh=False
             ),
         ] if allow_user_registration
         else [
@@ -36,12 +36,12 @@ registration_div = html.Div(
             dbc.Button(
                 'More information.',
                 id = 'show-user-registration-disabled-button',
-                color = 'link',
-                size = 'sm',
+                color='link',
+                size='sm',
             ),
             dbc.Collapse(
-                id = 'user-registration-disabled-collapse',
-                children = [
+                id='user-registration-disabled-collapse',
+                children=[
                     dcc.Markdown(
                         "For example, to register user `newuser` on instance `sql:main`:"
                     ),
@@ -60,10 +60,10 @@ registration_div = html.Div(
                     html.Pre(
                         html.Code(
                             json.dumps({
-                                'api' : {
-                                    'permissions' : {
-                                        'registration' : {
-                                            'users' : True,
+                                'api': {
+                                    'permissions': {
+                                        'registration': {
+                                            'users': True,
                                         },
                                     }
                                 }
@@ -112,11 +112,11 @@ layout = dbc.Container([
                 dbc.Row([
                     dbc.Col([
                         html.Button(
-                            children = 'Login',
-                            n_clicks = 0,
-                            type = 'submit',
-                            id = 'login-button',
-                            className = 'btn btn-primary btn-lg'
+                            children='Login',
+                            n_clicks=0,
+                            type='submit',
+                            id='login-button',
+                            className='btn btn-primary btn-lg'
                         ),
                     ]),
                 ]),

meerschaum/api/dash/pipes.py

@@ -33,6 +33,11 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
 humanfriendly = attempt_import('humanfriendly', check_update=CHECK_UPDATE)
 pd = import_pandas()
 
+INDEX_PREFIX_EMOJI: Dict[str, str] = {
+    'datetime': '🕓',
+    'primary': '🔑',
+}
+
 
 def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, None]:
     """
@@ -346,6 +351,8 @@ def accordion_items_from_pipe(
     for item in skip_items:
         _ = items_titles.pop(item, None)
 
+    pipe_meta_str = json.dumps(pipe.meta, sort_keys=True)
+
     ### Only generate items if they're in the `active_items` list.
     items_bodies = {}
     if 'overview' in active_items:
@@ -358,22 +365,107 @@ def accordion_items_from_pipe(
             html.Tr([html.Td("Instance"), html.Td(html.Pre(f"{pipe.instance_keys}"))]),
             html.Tr([html.Td("Target Table"), html.Td(html.Pre(f"{pipe.target}"))]),
         ]
-        columns = pipe.columns.copy()
-        if columns:
-            datetime_index = columns.pop('datetime', None)
-            columns_items = []
-            if datetime_index:
-                columns_items.append(html.Li(f"{datetime_index} (datetime)"))
-            columns_items.extend([
-                html.Li(f"{col}")
-                for col_key, col in columns.items()
-            ])
+
+        indices_header = [
+            html.Thead(
+                html.Tr(
+                    [
+                        html.Th(
+                            html.Span(
+                                "Key",
+                                id={'type': 'key-table-header', 'id': pipe_meta_str},
+                                style={"textDecoration": "underline", "cursor": "pointer"},
+                            ),
+                        ),
+                        html.Th(
+                            html.Span(
+                                "Column",
+                                id={'type': 'column-table-header', 'id': pipe_meta_str},
+                                style={"textDecoration": "underline", "cursor": "pointer"},
+                            ),
+                        ),
+                        html.Th(
+                            html.Span(
+                                "Index",
+                                id={'type': 'index-table-header', 'id': pipe_meta_str},
+                                style={"textDecoration": "underline", "cursor": "pointer"},
+                            ),
+                        ),
+                        html.Th(
+                            html.Span(
+                                "Is Composite",
+                                id={'type': 'is-composite-table-header', 'id': pipe_meta_str},
+                                style={"textDecoration": "underline", "cursor": "pointer"},
+                            ),
+                        ),
+                        dbc.Tooltip(
+                            "Unique reference name for the index "
+                            "(e.g. `datetime` for the range axis)",
+                            target={'type': 'key-table-header', 'id': pipe_meta_str},
+                        ),
+                        dbc.Tooltip(
+                            "The actual column (field name) in the target dataset.",
+                            target={'type': 'column-table-header', 'id': pipe_meta_str},
+                        ),
+                        dbc.Tooltip(
+                            "The name of the index created on the given columns.",
+                            target={'type': 'index-table-header', 'id': pipe_meta_str},
+                        ),
+                        dbc.Tooltip(
+                            "Whether the column is used in the composite primary key "
+                            "to determine updates.",
+                            target={'type': 'is-composite-table-header', 'id': pipe_meta_str},
+                        ),
+                    ]
+                )
+            )
+        ]
+
+        indices = pipe.indices
+        columns = pipe.columns
+        index_column_names = pipe.get_indices()
+        indices_rows = []
+        for ix_key, ix_cols in indices.items():
+            col = columns.get(ix_key, None)
+            ix_name = index_column_names.get(ix_key, None)
+            if not col and not ix_cols:
+                continue
+            if not isinstance(ix_cols, (list, tuple)):
+                ix_cols = [ix_cols]
+            if col:
+                col_item = html.Pre(col)
+            elif len(ix_cols) == 1:
+                col_item = html.Pre(ix_cols[0])
+            else:
+                col_item = html.Pre(html.Ul([html.Li(_c) for _c in ix_cols]))
+            is_composite_item = "✅" if col else ""
+            ix_key_item = html.Pre(
+                INDEX_PREFIX_EMOJI.get(ix_key, '')
+                + (' ' if ix_key in INDEX_PREFIX_EMOJI else '')
+                + ix_key
+            )
+            indices_rows.append(
+                html.Tr([
+                    html.Td(ix_key_item),
+                    html.Td(col_item),
+                    html.Td(html.Pre(ix_name or '')),
+                    html.Td(is_composite_item),
+                ])
+            )
+        indices_table = dbc.Table(
+            indices_header + [html.Tbody(indices_rows)],
+            bordered=True,
+            hover=False,
+            striped=True,
+        )
+        if indices_rows:
             overview_rows.append(
                 html.Tr([
-                    html.Td("Indices" if len(columns_items) != 1 else "Index"),
-                    html.Td(html.Pre(html.Ul(columns_items))),
+                    html.Td("Indices" if len(indices_rows) != 1 else "Index"),
+                    html.Td(indices_table),
                 ])
             )
+
         tags = pipe.tags
         if tags:
             tags_items = html.Ul([
@@ -420,7 +512,6 @@ def accordion_items_from_pipe(
         if newest_time is not None:
             stats_rows.append(html.Tr([html.Td("Newest time"), html.Td(str(newest_time))]))
 
-
         items_bodies['stats'] = dbc.Table(stats_header + [html.Tbody(stats_rows)], hover=True)
 
     if 'columns' in active_items:

meerschaum/api/routes/_pipes.py

@@ -558,7 +558,7 @@ def get_pipe_id(
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
-) -> int:
+) -> Union[int, str]:
     """
     Get a Pipe's ID.
     """
@@ -573,36 +573,36 @@ def get_pipe_id(
     tags = ['Pipes']
 )
 def get_pipe_attributes(
-        connector_keys : str,
-        metric_key : str,
-        location_key : str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Dict[str, Any]:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Any]:
     """Get a Pipe's attributes."""
     return get_pipe(connector_keys, metric_key, location_key).attributes
 
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists', tags=['Pipes'])
 def get_pipe_exists(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> bool:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
     """Determine whether a Pipe exists."""
     return get_pipe(connector_keys, metric_key, location_key).exists()
 
 
 @app.post(endpoints['metadata'], tags=['Pipes'])
 def create_metadata(
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> bool:
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
     """Create Pipe metadata tables"""
     from meerschaum.connectors.sql.tables import get_tables
     try:
@@ -614,16 +614,16 @@ def create_metadata(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount', tags=['Pipes'])
 def get_pipe_rowcount(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        begin: Union[str, int, None] = None,
-        end: Union[str, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> int:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> int:
     """
     Return a pipe's rowcount.
     """
@@ -632,10 +632,10 @@ def get_pipe_rowcount(
     if is_int(end):
         end = int(end)
     return get_pipe(connector_keys, metric_key, location_key).get_rowcount(
-        begin = begin,
-        end = end,
-        params = params,
-        debug = debug
+        begin=begin,
+        end=end,
+        params=params,
+        debug=debug,
     )
 
 
@@ -644,13 +644,13 @@ def get_pipe_rowcount(
     tags=['Pipes']
 )
 def get_pipe_columns_types(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Dict[str, str]:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, str]:
     """
     Return a dictionary of column names and types.
 
@@ -663,3 +663,21 @@ def get_pipe_columns_types(
     ```
     """
     return get_pipe(connector_keys, metric_key, location_key).dtypes
+
+
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/indices',
+    tags=['Pipes']
+)
+def get_pipe_columns_indices(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, List[Dict[str, str]]]:
+    """
+    Return a dictionary of column names and related indices.
+    """
+    return get_pipe(connector_keys, metric_key, location_key).get_columns_indices()

meerschaum/api/routes/_webterm.py

@@ -13,6 +13,7 @@ from meerschaum.utils.packages import attempt_import
 from meerschaum.api.dash.sessions import is_session_authenticated
 fastapi, fastapi_responses = attempt_import('fastapi', 'fastapi.responses')
 import starlette
+httpcore = attempt_import('httpcore')
 httpx = attempt_import('httpx')
 websockets = attempt_import('websockets')
 Request = fastapi.Request

meerschaum/config/_edit.py

@@ -7,44 +7,71 @@ Functions for editing the configuration file
 """
 
 from __future__ import annotations
-import sys
 import pathlib
 from meerschaum.utils.typing import Optional, Any, SuccessTuple, Mapping, Dict, List, Union
 
+
 def edit_config(
-        keys : Optional[List[str]] = None,
-        params : Optional[Dict[str, Any]] = None,
-        debug : bool = False,
-        **kw : Any
-    ) -> SuccessTuple:
+    keys: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """Edit the configuration files."""
     from meerschaum.config import get_config, config
-    from meerschaum.config._read_config import get_keyfile_path
+    from meerschaum.config._read_config import get_keyfile_path, read_config
     from meerschaum.config._paths import CONFIG_DIR_PATH
     from meerschaum.utils.packages import reload_meerschaum
     from meerschaum.utils.misc import edit_file
-    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.warnings import warn, dprint
+    from meerschaum.utils.prompt import prompt
 
     if keys is None:
         keys = []
 
-    for k in keys:
-        ### If defined in default, create the config file.
-        if isinstance(config, dict) and k in config:
-            del config[k]
-        get_config(k, write_missing=True, warn=False)
-        edit_file(get_keyfile_path(k, create_new=True))
+    def _edit_key(key: str):
+        while True:
+            ### If defined in default, create the config file.
+            key_config = config.pop(key, None)
+            keyfile_path = get_keyfile_path(key, create_new=True)
+            get_config(key, write_missing=True, warn=False)
+
+            edit_file(get_keyfile_path(key, create_new=True))
+
+            ### TODO: verify that the file is valid. Retry if not.
+            try:
+                new_key_config = read_config(
+                    CONFIG_DIR_PATH,
+                    [key],
+                    write_missing=False,
+                    raise_parsing_errors=True,
+                )
+            except Exception:
+                if key_config:
+                    config[key] = key_config
+                warn(f"Could not parse key '{key}'.", stack=False)
+                _ = prompt(f"Press [Enter] to edit '{keyfile_path}', [CTRL+C] to exit.")
+                continue
+
+            if new_key_config:
+                break
+
+    try:
+        for k in keys:
+            _edit_key(k)
+    except KeyboardInterrupt:
+        return False, f""
 
     reload_meerschaum(debug=debug)
     return (True, "Success")
 
 
 def write_config(
-        config_dict: Optional[Dict[str, Any]] = None,
-        directory: Union[str, pathlib.Path, None] = None,
-        debug: bool = False,
-        **kw : Any
-    ) -> bool:
+    config_dict: Optional[Dict[str, Any]] = None,
+    directory: Union[str, pathlib.Path, None] = None,
+    debug: bool = False,
+    **kw: Any
+) -> bool:
     """Write YAML and JSON files to the configuration directory.
 
     Parameters