meerschaum 2.5.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

meerschaum/config/_read_config.py

@@ -6,22 +6,26 @@ Import the config yaml file
 """
 
 from __future__ import annotations
+import pathlib
+
 from meerschaum.utils.typing import Optional, Dict, Any, List, Tuple, Union
 from meerschaum.config import get_config
 
+
 def read_config(
-    directory: Optional[str] = None,
+    directory: Union[pathlib.Path, str, None] = None,
     keys: Optional[List[str]] = None,
-    write_missing : bool = True,
-    substitute : bool = True,
-    with_filenames : bool = False,
+    write_missing: bool = True,
+    substitute: bool = True,
+    with_filenames: bool = False,
+    raise_parsing_errors: bool = False,
 ) -> Union[Dict[str, Any], Tuple[Dict[str, Any], List[str]]]:
     """
     Read the configuration directory.
 
     Parameters
     ----------
-    directory: Optional[str], default None
+    directory: Union[pathlib.Path, str, None], default None
         The directory with configuration files (.json and .yaml).
 
     keys: Optional[List[str]], default None
@@ -36,7 +40,10 @@ def read_config(
 
     with_filename: bool, default False
         If `True`, return a tuple of the configuration dictionary with a list of read filenames.
-        
+
+    raise_parsing_errors: bool, default False
+        If `True`, re-raise parsing exceptions.
+
     Examples
     --------
     >>> read_config(keys=['meerschaum'], with_filename=True)
@@ -63,9 +70,9 @@ def read_config(
 
     default_filetype = STATIC_CONFIG['config']['default_filetype']
     filetype_loaders = {
-        'yml' : yaml.load,
-        'yaml' : yaml.load,
-        'json' : json.load,
+        'yml': yaml.load,
+        'yaml': yaml.load,
+        'json': json.load,
     }
 
     ### Construct filekeys (files to parse).
@@ -167,6 +174,8 @@ def read_config(
                         _config_key = filetype_loaders[_type](f)
                     except Exception as e:
                         print(f"Error processing file: {filepath}")
+                        if raise_parsing_errors:
+                            raise e
                         import traceback
                         traceback.print_exc()
                         _config_key = {}
@@ -184,6 +193,8 @@ def read_config(
                     config[symlinks_key][key] = _single_key_config[symlinks_key][key]
                 break
             except Exception as e:
+                if raise_parsing_errors:
+                    raise e
                 print(f"Unable to parse {filename}!")
                 import traceback
                 traceback.print_exc()

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.5.0"
+__version__ = "2.6.0"

meerschaum/config/stack/__init__.py

@@ -39,7 +39,7 @@ valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
 
 env_dict = {
     'COMPOSE_PROJECT_NAME': 'mrsm',
-    'TIMESCALEDB_VERSION': 'latest-pg16-oss',
+    'TIMESCALEDB_VERSION': 'latest-pg16',
     'POSTGRES_USER': db_user,
     'POSTGRES_PASSWORD': db_pass,
     'POSTGRES_DB': db_base,

meerschaum/config/static/__init__.py

@@ -148,6 +148,7 @@ STATIC_CONFIG: Dict[str, Any] = {
             'min_ratio_columns_changed_for_full_astype': 0.5,
         },
         'exists_timeout_seconds': 5.0,
+        'static_schema_cache_seconds': 60.0,
     },
     'jobs': {
         'check_restart_seconds': 1.0,

meerschaum/connectors/api/_APIConnector.py

@@ -58,6 +58,7 @@ class APIConnector(Connector):
         drop_pipe,
         clear_pipe,
         get_pipe_columns_types,
+        get_pipe_columns_indices,
     )
     from ._fetch import fetch
     from ._plugins import (

meerschaum/connectors/api/_pipes.py

@@ -669,11 +669,11 @@ def clear_pipe(
     kw.pop('force', None)
     return self.do_action_legacy(
         ['clear', 'pipes'],
-        connector_keys = pipe.connector_keys,
-        metric_keys = pipe.metric_key,
-        location_keys = pipe.location_key,
-        force = True,
-        debug = debug,
+        connector_keys=pipe.connector_keys,
+        metric_keys=pipe.metric_key,
+        location_keys=pipe.location_key,
+        force=True,
+        debug=debug,
         **kw
     )
 
@@ -690,7 +690,7 @@ def get_pipe_columns_types(
     ----------
     pipe: meerschaum.Pipe
         The pipe whose columns to be queried.
-        
+
     Returns
     -------
     A dictionary mapping column names to their database types.
@@ -707,11 +707,42 @@ def get_pipe_columns_types(
     r_url = pipe_r_url(pipe) + '/columns/types'
     response = self.get(
         r_url,
-        debug = debug
+        debug=debug
+    )
+    j = response.json()
+    if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
+        warn(j['detail'])
+        return None
+    if not isinstance(j, dict):
+        warn(response.text)
+        return None
+    return j
+
+
+def get_pipe_columns_indices(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Union[Dict[str, str], None]:
+    """
+    Fetch the index information for a pipe.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose columns to be queried.
+
+    Returns
+    -------
+    A dictionary mapping column names to a list of associated index information.
+    """
+    r_url = pipe_r_url(pipe) + '/columns/indices'
+    response = self.get(
+        r_url,
+        debug=debug
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
-        from meerschaum.utils.warnings import warn
         warn(j['detail'])
         return None
     if not isinstance(j, dict):

meerschaum/connectors/sql/_SQLConnector.py

@@ -67,6 +67,8 @@ class SQLConnector(Connector):
         get_pipe_columns_types,
         get_to_sql_dtype,
         get_pipe_schema,
+        create_pipe_table_from_df,
+        get_pipe_columns_indices,
     )
     from ._plugins import (
         register_plugin,
@@ -141,9 +143,9 @@ class SQLConnector(Connector):
             if uri.startswith('postgres') and not uri.startswith('postgresql'):
                 uri = uri.replace('postgres', 'postgresql', 1)
             if uri.startswith('postgresql') and not uri.startswith('postgresql+'):
-                uri = uri.replace('postgresql://', 'postgresql+psycopg', 1)
+                uri = uri.replace('postgresql://', 'postgresql+psycopg://', 1)
             if uri.startswith('timescaledb://'):
-                uri = uri.replace('timescaledb://', 'postgresql://', 1)
+                uri = uri.replace('timescaledb://', 'postgresql+psycopg://', 1)
                 flavor = 'timescaledb'
             kw['uri'] = uri
             from_uri_params = self.from_uri(kw['uri'], as_dict=True)
@@ -155,7 +157,6 @@ class SQLConnector(Connector):
             if flavor:
                 kw['flavor'] = flavor
 
-
         ### set __dict__ in base class
         super().__init__(
             'sql',