meerschaum 2.5.1__py3-none-any.whl → 2.6.0__py3-none-any.whl

meerschaum/_internal/arguments/_parser.py

@@ -90,8 +90,13 @@ def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
     except Exception as e:
         dt = None
     if dt is None:
-        from meerschaum.utils.warnings import warn, error
+        from meerschaum.utils.warnings import error
         error(f"'{dt_str}' is not a valid datetime format.", stack=False)
+
+    if isinstance(dt, datetime):
+        from meerschaum.utils.dtypes import coerce_timezone
+        dt = coerce_timezone(dt)
+
     return dt
 
 

meerschaum/actions/edit.py

@@ -65,24 +65,24 @@ def _complete_edit(
     from meerschaum._internal.shell import default_action_completer
     return default_action_completer(action=(['edit'] + action), **kw)
 
-def _edit_config(action : Optional[List[str]] = None, **kw : Any) -> SuccessTuple:
+
+def _edit_config(action: Optional[List[str]] = None, **kw : Any) -> SuccessTuple:
     """
     Edit Meerschaum configuration files.
-    
+
     Specify a specific configuration key to edit.
     Defaults to editing `meerschaum` configuration (connectors, instance, etc.).
-    
+
     Examples:
         ```
         ### Edit the main 'meerschaum' configuration.
         edit config
-    
+
         ### Edit 'system' configuration.
         edit config system
-    
+
         ### Create a new configuration file called 'myconfig'.
         edit config myconfig
-    
         ```
     """
     from meerschaum.config._edit import edit_config

meerschaum/actions/sql.py

@@ -14,6 +14,7 @@ exec_methods = {
     'exec',
 }
 
+
 def sql(
     action: Optional[List[str]] = None,
     gui: bool = False,
@@ -22,40 +23,40 @@ def sql(
     **kw: Any
 ):
     """Execute a SQL query or launch an interactive CLI. All positional arguments are optional.
-    
+
     Usage:
         `sql {label} {method} {query / table}`
-    
+
     Options:
         - `sql {label}`
             Launch an interactive CLI. If {label} is omitted, use 'main'.
-    
+
         - `sql {label} read [query / table]`
             Read a table or query as a pandas DataFrame and print the result.
-    
+
         - `sql {label} exec [query]`
             Execute a query and print the success status.
-    
+
     Examples:
         - `sql`
             Open an interactive CLI for `sql:main`.
-    
+
         - `sql local`
             Open an interactive CLI for `sql:local`.
-    
+
         - `sql table`
             Read from `table` on `sql:main`
               (translates to `SELECT * FROM table`).
-    
+
         - `sql local table`
             Read from `table` on `sql:local`.
-    
+
         - `sql local read table`
             Read from `table` on `sql:local`.
-    
+
         - `sql "SELECT * FROM table WHERE id = 1"`
             Execute the above query on `sql:main` and print the results.
-    
+
         - `sql local exec "INSERT INTO table (id) VALUES (1)"
             Execute the above query on `sql:local`.
     """

meerschaum/api/dash/pages/login.py

@@ -19,13 +19,13 @@ allow_user_registration = permissions_config['registration']['users']
 
 registration_div = html.Div(
     id = 'registration-div',
-    style = {'height' : '100%'},
-    children = (
+    style={'height' : '100%'},
+    children=(
         [
             dcc.Link(
                 'No account? Create one here.',
-                href = (dash_endpoint + '/register'),
-                refresh = False
+                href=(dash_endpoint + '/register'),
+                refresh=False
             ),
         ] if allow_user_registration
         else [
@@ -36,12 +36,12 @@ registration_div = html.Div(
             dbc.Button(
                 'More information.',
                 id = 'show-user-registration-disabled-button',
-                color = 'link',
-                size = 'sm',
+                color='link',
+                size='sm',
             ),
             dbc.Collapse(
-                id = 'user-registration-disabled-collapse',
-                children = [
+                id='user-registration-disabled-collapse',
+                children=[
                     dcc.Markdown(
                         "For example, to register user `newuser` on instance `sql:main`:"
                     ),
@@ -60,10 +60,10 @@ registration_div = html.Div(
                     html.Pre(
                         html.Code(
                             json.dumps({
-                                'api' : {
-                                    'permissions' : {
-                                        'registration' : {
-                                            'users' : True,
+                                'api': {
+                                    'permissions': {
+                                        'registration': {
+                                            'users': True,
                                         },
                                     }
                                 }
@@ -112,11 +112,11 @@ layout = dbc.Container([
                 dbc.Row([
                     dbc.Col([
                         html.Button(
-                            children = 'Login',
-                            n_clicks = 0,
-                            type = 'submit',
-                            id = 'login-button',
-                            className = 'btn btn-primary btn-lg'
+                            children='Login',
+                            n_clicks=0,
+                            type='submit',
+                            id='login-button',
+                            className='btn btn-primary btn-lg'
                         ),
                     ]),
                 ]),

meerschaum/api/dash/pipes.py

@@ -33,6 +33,11 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
 humanfriendly = attempt_import('humanfriendly', check_update=CHECK_UPDATE)
 pd = import_pandas()
 
+INDEX_PREFIX_EMOJI: Dict[str, str] = {
+    'datetime': '🕓',
+    'primary': '🔑',
+}
+
 
 def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, None]:
     """
@@ -420,9 +425,9 @@ def accordion_items_from_pipe(
         columns = pipe.columns
         index_column_names = pipe.get_indices()
         indices_rows = []
-        for ix_key, ix_name in index_column_names.items():
+        for ix_key, ix_cols in indices.items():
             col = columns.get(ix_key, None)
-            ix_cols = indices.get(ix_key, None)
+            ix_name = index_column_names.get(ix_key, None)
             if not col and not ix_cols:
                 continue
             if not isinstance(ix_cols, (list, tuple)):
@@ -434,12 +439,16 @@ def accordion_items_from_pipe(
             else:
                 col_item = html.Pre(html.Ul([html.Li(_c) for _c in ix_cols]))
             is_composite_item = "✅" if col else ""
-            ix_key_item = html.Pre(ix_key) if ix_key != 'datetime' else html.Pre(f"🕓 {ix_key}")
+            ix_key_item = html.Pre(
+                INDEX_PREFIX_EMOJI.get(ix_key, '')
+                + (' ' if ix_key in INDEX_PREFIX_EMOJI else '')
+                + ix_key
+            )
             indices_rows.append(
                 html.Tr([
                     html.Td(ix_key_item),
                     html.Td(col_item),
-                    html.Td(html.Pre(ix_name)),
+                    html.Td(html.Pre(ix_name or '')),
                     html.Td(is_composite_item),
                 ])
             )

meerschaum/api/routes/_pipes.py

@@ -558,7 +558,7 @@ def get_pipe_id(
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
-) -> int:
+) -> Union[int, str]:
     """
     Get a Pipe's ID.
     """
@@ -573,36 +573,36 @@ def get_pipe_id(
     tags = ['Pipes']
 )
 def get_pipe_attributes(
-        connector_keys : str,
-        metric_key : str,
-        location_key : str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Dict[str, Any]:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Any]:
     """Get a Pipe's attributes."""
     return get_pipe(connector_keys, metric_key, location_key).attributes
 
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists', tags=['Pipes'])
 def get_pipe_exists(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> bool:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
     """Determine whether a Pipe exists."""
     return get_pipe(connector_keys, metric_key, location_key).exists()
 
 
 @app.post(endpoints['metadata'], tags=['Pipes'])
 def create_metadata(
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> bool:
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
     """Create Pipe metadata tables"""
     from meerschaum.connectors.sql.tables import get_tables
     try:
@@ -614,16 +614,16 @@ def create_metadata(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount', tags=['Pipes'])
 def get_pipe_rowcount(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        begin: Union[str, int, None] = None,
-        end: Union[str, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> int:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> int:
     """
     Return a pipe's rowcount.
     """
@@ -632,10 +632,10 @@ def get_pipe_rowcount(
     if is_int(end):
         end = int(end)
     return get_pipe(connector_keys, metric_key, location_key).get_rowcount(
-        begin = begin,
-        end = end,
-        params = params,
-        debug = debug
+        begin=begin,
+        end=end,
+        params=params,
+        debug=debug,
     )
 
 
@@ -644,13 +644,13 @@ def get_pipe_rowcount(
     tags=['Pipes']
 )
 def get_pipe_columns_types(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Dict[str, str]:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, str]:
     """
     Return a dictionary of column names and types.
 
@@ -663,3 +663,21 @@ def get_pipe_columns_types(
     ```
     """
     return get_pipe(connector_keys, metric_key, location_key).dtypes
+
+
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/indices',
+    tags=['Pipes']
+)
+def get_pipe_columns_indices(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, List[Dict[str, str]]]:
+    """
+    Return a dictionary of column names and related indices.
+    """
+    return get_pipe(connector_keys, metric_key, location_key).get_columns_indices()

meerschaum/config/_edit.py

@@ -7,44 +7,71 @@ Functions for editing the configuration file
 """
 
 from __future__ import annotations
-import sys
 import pathlib
 from meerschaum.utils.typing import Optional, Any, SuccessTuple, Mapping, Dict, List, Union
 
+
 def edit_config(
-        keys : Optional[List[str]] = None,
-        params : Optional[Dict[str, Any]] = None,
-        debug : bool = False,
-        **kw : Any
-    ) -> SuccessTuple:
+    keys: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """Edit the configuration files."""
     from meerschaum.config import get_config, config
-    from meerschaum.config._read_config import get_keyfile_path
+    from meerschaum.config._read_config import get_keyfile_path, read_config
     from meerschaum.config._paths import CONFIG_DIR_PATH
     from meerschaum.utils.packages import reload_meerschaum
     from meerschaum.utils.misc import edit_file
-    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.warnings import warn, dprint
+    from meerschaum.utils.prompt import prompt
 
     if keys is None:
         keys = []
 
-    for k in keys:
-        ### If defined in default, create the config file.
-        if isinstance(config, dict) and k in config:
-            del config[k]
-        get_config(k, write_missing=True, warn=False)
-        edit_file(get_keyfile_path(k, create_new=True))
+    def _edit_key(key: str):
+        while True:
+            ### If defined in default, create the config file.
+            key_config = config.pop(key, None)
+            keyfile_path = get_keyfile_path(key, create_new=True)
+            get_config(key, write_missing=True, warn=False)
+
+            edit_file(get_keyfile_path(key, create_new=True))
+
+            ### TODO: verify that the file is valid. Retry if not.
+            try:
+                new_key_config = read_config(
+                    CONFIG_DIR_PATH,
+                    [key],
+                    write_missing=False,
+                    raise_parsing_errors=True,
+                )
+            except Exception:
+                if key_config:
+                    config[key] = key_config
+                warn(f"Could not parse key '{key}'.", stack=False)
+                _ = prompt(f"Press [Enter] to edit '{keyfile_path}', [CTRL+C] to exit.")
+                continue
+
+            if new_key_config:
+                break
+
+    try:
+        for k in keys:
+            _edit_key(k)
+    except KeyboardInterrupt:
+        return False, f""
 
     reload_meerschaum(debug=debug)
     return (True, "Success")
 
 
 def write_config(
-        config_dict: Optional[Dict[str, Any]] = None,
-        directory: Union[str, pathlib.Path, None] = None,
-        debug: bool = False,
-        **kw : Any
-    ) -> bool:
+    config_dict: Optional[Dict[str, Any]] = None,
+    directory: Union[str, pathlib.Path, None] = None,
+    debug: bool = False,
+    **kw: Any
+) -> bool:
     """Write YAML and JSON files to the configuration directory.
 
     Parameters

meerschaum/config/_read_config.py

@@ -6,22 +6,26 @@ Import the config yaml file
 """
 
 from __future__ import annotations
+import pathlib
+
 from meerschaum.utils.typing import Optional, Dict, Any, List, Tuple, Union
 from meerschaum.config import get_config
 
+
 def read_config(
-    directory: Optional[str] = None,
+    directory: Union[pathlib.Path, str, None] = None,
     keys: Optional[List[str]] = None,
-    write_missing : bool = True,
-    substitute : bool = True,
-    with_filenames : bool = False,
+    write_missing: bool = True,
+    substitute: bool = True,
+    with_filenames: bool = False,
+    raise_parsing_errors: bool = False,
 ) -> Union[Dict[str, Any], Tuple[Dict[str, Any], List[str]]]:
     """
     Read the configuration directory.
 
     Parameters
     ----------
-    directory: Optional[str], default None
+    directory: Union[pathlib.Path, str, None], default None
         The directory with configuration files (.json and .yaml).
 
     keys: Optional[List[str]], default None
@@ -36,7 +40,10 @@ def read_config(
 
     with_filename: bool, default False
         If `True`, return a tuple of the configuration dictionary with a list of read filenames.
-        
+
+    raise_parsing_errors: bool, default False
+        If `True`, re-raise parsing exceptions.
+
     Examples
     --------
     >>> read_config(keys=['meerschaum'], with_filename=True)
@@ -63,9 +70,9 @@ def read_config(
 
     default_filetype = STATIC_CONFIG['config']['default_filetype']
     filetype_loaders = {
-        'yml' : yaml.load,
-        'yaml' : yaml.load,
-        'json' : json.load,
+        'yml': yaml.load,
+        'yaml': yaml.load,
+        'json': json.load,
     }
 
     ### Construct filekeys (files to parse).
@@ -167,6 +174,8 @@ def read_config(
                         _config_key = filetype_loaders[_type](f)
                     except Exception as e:
                         print(f"Error processing file: {filepath}")
+                        if raise_parsing_errors:
+                            raise e
                         import traceback
                         traceback.print_exc()
                         _config_key = {}
@@ -184,6 +193,8 @@ def read_config(
                     config[symlinks_key][key] = _single_key_config[symlinks_key][key]
                 break
             except Exception as e:
+                if raise_parsing_errors:
+                    raise e
                 print(f"Unable to parse {filename}!")
                 import traceback
                 traceback.print_exc()

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.5.1"
+__version__ = "2.6.0"

meerschaum/config/stack/__init__.py

@@ -39,7 +39,7 @@ valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
 
 env_dict = {
     'COMPOSE_PROJECT_NAME': 'mrsm',
-    'TIMESCALEDB_VERSION': 'latest-pg16-oss',
+    'TIMESCALEDB_VERSION': 'latest-pg16',
     'POSTGRES_USER': db_user,
     'POSTGRES_PASSWORD': db_pass,
     'POSTGRES_DB': db_base,

meerschaum/config/static/__init__.py

@@ -148,6 +148,7 @@ STATIC_CONFIG: Dict[str, Any] = {
             'min_ratio_columns_changed_for_full_astype': 0.5,
         },
         'exists_timeout_seconds': 5.0,
+        'static_schema_cache_seconds': 60.0,
     },
     'jobs': {
         'check_restart_seconds': 1.0,

meerschaum/connectors/api/_APIConnector.py

@@ -58,6 +58,7 @@ class APIConnector(Connector):
         drop_pipe,
         clear_pipe,
         get_pipe_columns_types,
+        get_pipe_columns_indices,
     )
     from ._fetch import fetch
     from ._plugins import (

meerschaum/connectors/api/_pipes.py

@@ -669,11 +669,11 @@ def clear_pipe(
     kw.pop('force', None)
     return self.do_action_legacy(
         ['clear', 'pipes'],
-        connector_keys = pipe.connector_keys,
-        metric_keys = pipe.metric_key,
-        location_keys = pipe.location_key,
-        force = True,
-        debug = debug,
+        connector_keys=pipe.connector_keys,
+        metric_keys=pipe.metric_key,
+        location_keys=pipe.location_key,
+        force=True,
+        debug=debug,
         **kw
     )
 
@@ -690,7 +690,7 @@ def get_pipe_columns_types(
     ----------
     pipe: meerschaum.Pipe
         The pipe whose columns to be queried.
-        
+
     Returns
     -------
     A dictionary mapping column names to their database types.
@@ -707,11 +707,42 @@ def get_pipe_columns_types(
     r_url = pipe_r_url(pipe) + '/columns/types'
     response = self.get(
         r_url,
-        debug = debug
+        debug=debug
+    )
+    j = response.json()
+    if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
+        warn(j['detail'])
+        return None
+    if not isinstance(j, dict):
+        warn(response.text)
+        return None
+    return j
+
+
+def get_pipe_columns_indices(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Union[Dict[str, str], None]:
+    """
+    Fetch the index information for a pipe.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose columns to be queried.
+
+    Returns
+    -------
+    A dictionary mapping column names to a list of associated index information.
+    """
+    r_url = pipe_r_url(pipe) + '/columns/indices'
+    response = self.get(
+        r_url,
+        debug=debug
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
-        from meerschaum.utils.warnings import warn
         warn(j['detail'])
         return None
     if not isinstance(j, dict):

meerschaum/connectors/sql/_SQLConnector.py

@@ -67,6 +67,8 @@ class SQLConnector(Connector):
         get_pipe_columns_types,
         get_to_sql_dtype,
         get_pipe_schema,
+        create_pipe_table_from_df,
+        get_pipe_columns_indices,
     )
     from ._plugins import (
         register_plugin,
@@ -141,9 +143,9 @@ class SQLConnector(Connector):
             if uri.startswith('postgres') and not uri.startswith('postgresql'):
                 uri = uri.replace('postgres', 'postgresql', 1)
             if uri.startswith('postgresql') and not uri.startswith('postgresql+'):
-                uri = uri.replace('postgresql://', 'postgresql+psycopg', 1)
+                uri = uri.replace('postgresql://', 'postgresql+psycopg://', 1)
             if uri.startswith('timescaledb://'):
-                uri = uri.replace('timescaledb://', 'postgresql://', 1)
+                uri = uri.replace('timescaledb://', 'postgresql+psycopg://', 1)
                 flavor = 'timescaledb'
             kw['uri'] = uri
             from_uri_params = self.from_uri(kw['uri'], as_dict=True)
@@ -155,7 +157,6 @@ class SQLConnector(Connector):
             if flavor:
                 kw['flavor'] = flavor
 
-
         ### set __dict__ in base class
         super().__init__(
             'sql',