meerschaum 2.5.1__py3-none-any.whl → 2.6.0.dev1__py3-none-any.whl

meerschaum/_internal/arguments/_parser.py

@@ -90,8 +90,13 @@ def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
     except Exception as e:
         dt = None
     if dt is None:
-        from meerschaum.utils.warnings import warn, error
+        from meerschaum.utils.warnings import error
         error(f"'{dt_str}' is not a valid datetime format.", stack=False)
+
+    if isinstance(dt, datetime):
+        from meerschaum.utils.dtypes import coerce_timezone
+        dt = coerce_timezone(dt)
+
     return dt
 
 

meerschaum/actions/edit.py

@@ -65,24 +65,24 @@ def _complete_edit(
     from meerschaum._internal.shell import default_action_completer
     return default_action_completer(action=(['edit'] + action), **kw)
 
-def _edit_config(action : Optional[List[str]] = None, **kw : Any) -> SuccessTuple:
+
+def _edit_config(action: Optional[List[str]] = None, **kw : Any) -> SuccessTuple:
     """
     Edit Meerschaum configuration files.
-    
+
     Specify a specific configuration key to edit.
     Defaults to editing `meerschaum` configuration (connectors, instance, etc.).
-    
+
     Examples:
         ```
         ### Edit the main 'meerschaum' configuration.
         edit config
-    
+
         ### Edit 'system' configuration.
         edit config system
-    
+
         ### Create a new configuration file called 'myconfig'.
         edit config myconfig
-    
         ```
     """
     from meerschaum.config._edit import edit_config

meerschaum/actions/sql.py

@@ -14,6 +14,7 @@ exec_methods = {
     'exec',
 }
 
+
 def sql(
     action: Optional[List[str]] = None,
     gui: bool = False,
@@ -22,40 +23,40 @@ def sql(
     **kw: Any
 ):
     """Execute a SQL query or launch an interactive CLI. All positional arguments are optional.
-    
+
     Usage:
         `sql {label} {method} {query / table}`
-    
+
     Options:
         - `sql {label}`
             Launch an interactive CLI. If {label} is omitted, use 'main'.
-    
+
         - `sql {label} read [query / table]`
             Read a table or query as a pandas DataFrame and print the result.
-    
+
         - `sql {label} exec [query]`
             Execute a query and print the success status.
-    
+
     Examples:
         - `sql`
             Open an interactive CLI for `sql:main`.
-    
+
         - `sql local`
             Open an interactive CLI for `sql:local`.
-    
+
         - `sql table`
             Read from `table` on `sql:main`
               (translates to `SELECT * FROM table`).
-    
+
         - `sql local table`
             Read from `table` on `sql:local`.
-    
+
         - `sql local read table`
             Read from `table` on `sql:local`.
-    
+
         - `sql "SELECT * FROM table WHERE id = 1"`
             Execute the above query on `sql:main` and print the results.
-    
+
         - `sql local exec "INSERT INTO table (id) VALUES (1)"
             Execute the above query on `sql:local`.
     """

meerschaum/config/_edit.py

@@ -7,44 +7,71 @@ Functions for editing the configuration file
 """
 
 from __future__ import annotations
-import sys
 import pathlib
 from meerschaum.utils.typing import Optional, Any, SuccessTuple, Mapping, Dict, List, Union
 
+
 def edit_config(
-        keys : Optional[List[str]] = None,
-        params : Optional[Dict[str, Any]] = None,
-        debug : bool = False,
-        **kw : Any
-    ) -> SuccessTuple:
+    keys: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """Edit the configuration files."""
     from meerschaum.config import get_config, config
-    from meerschaum.config._read_config import get_keyfile_path
+    from meerschaum.config._read_config import get_keyfile_path, read_config
     from meerschaum.config._paths import CONFIG_DIR_PATH
     from meerschaum.utils.packages import reload_meerschaum
     from meerschaum.utils.misc import edit_file
-    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.warnings import warn, dprint
+    from meerschaum.utils.prompt import prompt
 
     if keys is None:
         keys = []
 
-    for k in keys:
-        ### If defined in default, create the config file.
-        if isinstance(config, dict) and k in config:
-            del config[k]
-        get_config(k, write_missing=True, warn=False)
-        edit_file(get_keyfile_path(k, create_new=True))
+    def _edit_key(key: str):
+        while True:
+            ### If defined in default, create the config file.
+            key_config = config.pop(key, None)
+            keyfile_path = get_keyfile_path(key, create_new=True)
+            get_config(key, write_missing=True, warn=False)
+
+            edit_file(get_keyfile_path(key, create_new=True))
+
+            ### TODO: verify that the file is valid. Retry if not.
+            try:
+                new_key_config = read_config(
+                    CONFIG_DIR_PATH,
+                    [key],
+                    write_missing=False,
+                    raise_parsing_errors=True,
+                )
+            except Exception:
+                if key_config:
+                    config[key] = key_config
+                warn(f"Could not parse key '{key}'.", stack=False)
+                _ = prompt(f"Press [Enter] to edit '{keyfile_path}', [CTRL+C] to exit.")
+                continue
+
+            if new_key_config:
+                break
+
+    try:
+        for k in keys:
+            _edit_key(k)
+    except KeyboardInterrupt:
+        return False, f""
 
     reload_meerschaum(debug=debug)
     return (True, "Success")
 
 
 def write_config(
-        config_dict: Optional[Dict[str, Any]] = None,
-        directory: Union[str, pathlib.Path, None] = None,
-        debug: bool = False,
-        **kw : Any
-    ) -> bool:
+    config_dict: Optional[Dict[str, Any]] = None,
+    directory: Union[str, pathlib.Path, None] = None,
+    debug: bool = False,
+    **kw: Any
+) -> bool:
     """Write YAML and JSON files to the configuration directory.
 
     Parameters

meerschaum/config/_read_config.py

@@ -6,22 +6,26 @@ Import the config yaml file
 """
 
 from __future__ import annotations
+import pathlib
+
 from meerschaum.utils.typing import Optional, Dict, Any, List, Tuple, Union
 from meerschaum.config import get_config
 
+
 def read_config(
-    directory: Optional[str] = None,
+    directory: Union[pathlib.Path, str, None] = None,
     keys: Optional[List[str]] = None,
-    write_missing : bool = True,
-    substitute : bool = True,
-    with_filenames : bool = False,
+    write_missing: bool = True,
+    substitute: bool = True,
+    with_filenames: bool = False,
+    raise_parsing_errors: bool = False,
 ) -> Union[Dict[str, Any], Tuple[Dict[str, Any], List[str]]]:
     """
     Read the configuration directory.
 
     Parameters
     ----------
-    directory: Optional[str], default None
+    directory: Union[pathlib.Path, str, None], default None
         The directory with configuration files (.json and .yaml).
 
     keys: Optional[List[str]], default None
@@ -36,7 +40,10 @@ def read_config(
 
     with_filename: bool, default False
         If `True`, return a tuple of the configuration dictionary with a list of read filenames.
-        
+
+    raise_parsing_errors: bool, default False
+        If `True`, re-raise parsing exceptions.
+
     Examples
     --------
     >>> read_config(keys=['meerschaum'], with_filename=True)
@@ -63,9 +70,9 @@ def read_config(
 
     default_filetype = STATIC_CONFIG['config']['default_filetype']
     filetype_loaders = {
-        'yml' : yaml.load,
-        'yaml' : yaml.load,
-        'json' : json.load,
+        'yml': yaml.load,
+        'yaml': yaml.load,
+        'json': json.load,
     }
 
     ### Construct filekeys (files to parse).
@@ -167,6 +174,8 @@ def read_config(
                         _config_key = filetype_loaders[_type](f)
                     except Exception as e:
                         print(f"Error processing file: {filepath}")
+                        if raise_parsing_errors:
+                            raise e
                         import traceback
                         traceback.print_exc()
                         _config_key = {}
@@ -184,6 +193,8 @@ def read_config(
                     config[symlinks_key][key] = _single_key_config[symlinks_key][key]
                 break
             except Exception as e:
+                if raise_parsing_errors:
+                    raise e
                 print(f"Unable to parse {filename}!")
                 import traceback
                 traceback.print_exc()

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.5.1"
+__version__ = "2.6.0.dev1"

meerschaum/config/stack/__init__.py

@@ -39,7 +39,7 @@ valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
 
 env_dict = {
     'COMPOSE_PROJECT_NAME': 'mrsm',
-    'TIMESCALEDB_VERSION': 'latest-pg16-oss',
+    'TIMESCALEDB_VERSION': 'latest-pg16',
     'POSTGRES_USER': db_user,
     'POSTGRES_PASSWORD': db_pass,
     'POSTGRES_DB': db_base,

meerschaum/connectors/sql/_pipes.py

@@ -404,7 +404,7 @@ def get_create_index_queries(
     indices = pipe.indices
 
     _datetime = pipe.get_columns('datetime', error=False)
-    _datetime_type = pipe.dtypes.get(_datetime, 'datetime64[ns]')
+    _datetime_type = pipe.dtypes.get(_datetime, 'datetime64[ns, UTC]')
     _datetime_name = (
         sql_item_name(_datetime, self.flavor, None)
         if _datetime is not None else None
@@ -738,7 +738,7 @@ def get_pipe_data(
                 dt_type = dtypes.get(_dt, 'object').lower()
                 if 'datetime' not in dt_type:
                     if 'int' not in dt_type:
-                        dtypes[_dt] = 'datetime64[ns]'
+                        dtypes[_dt] = 'datetime64[ns, UTC]'
     existing_cols = pipe.get_columns_types(debug=debug)
     select_columns = (
         [
@@ -1197,7 +1197,12 @@ def sync_pipe(
     A `SuccessTuple` of success (`bool`) and message (`str`).
     """
     from meerschaum.utils.packages import import_pandas
-    from meerschaum.utils.sql import get_update_queries, sql_item_name, json_flavors, update_queries
+    from meerschaum.utils.sql import (
+        get_update_queries,
+        sql_item_name,
+        update_queries,
+        get_create_table_queries,
+    )
     from meerschaum.utils.misc import generate_password
     from meerschaum.utils.dataframe import get_json_cols, get_numeric_cols
     from meerschaum.utils.dtypes import are_dtypes_equal
@@ -1232,7 +1237,6 @@ def sync_pipe(
 
     ### if table does not exist, create it with indices
     is_new = False
-    add_cols_query = None
     if not pipe.exists(debug=debug):
         check_existing = False
         is_new = True
@@ -1252,9 +1256,7 @@ def sync_pipe(
 
     ### NOTE: Oracle SQL < 23c (2023) and SQLite does not support booleans,
     ### so infer bools and persist them to `dtypes`.
-    ### MSSQL supports `BIT` for booleans, but we coerce bools to int for MSSQL
-    ### to avoid merge issues.
-    if self.flavor in ('oracle', 'sqlite', 'mssql', 'mysql', 'mariadb'):
+    if self.flavor in ('oracle', 'sqlite', 'mysql', 'mariadb'):
         pipe_dtypes = pipe.dtypes
         new_bool_cols = {
             col: 'bool[pyarrow]'
@@ -1309,7 +1311,60 @@ def sync_pipe(
         'schema': self.get_pipe_schema(pipe),
     })
 
+    primary_key = pipe.columns.get('primary', None)
+    new_dtypes = {
+        **{
+            col: str(typ)
+            for col, typ in unseen_df.dtypes.items()
+        },
+        **{
+            col: 'int'
+            for col_ix, col in pipe.columns.items()
+            if col_ix != 'primary'
+        },
+        **pipe.dtypes
+    } if is_new else {}
+    autoincrement = (
+        pipe.parameters.get('autoincrement', False)
+        or (is_new and primary_key and primary_key not in new_dtypes)
+    )
+    if autoincrement and autoincrement not in pipe.parameters:
+        pipe.parameters['autoincrement'] = autoincrement
+        edit_success, edit_msg = pipe.edit(debug=debug)
+        if not edit_success:
+            return edit_success, edit_msg
+
+    if autoincrement and primary_key and primary_key not in df.columns:
+        if unseen_df is not None and primary_key in unseen_df.columns:
+            del unseen_df[primary_key]
+        if update_df is not None and primary_key in update_df.columns:
+            del update_df[primary_key]
+        if delta_df is not None and primary_key in delta_df.columns:
+            del delta_df[primary_key]
+
+    if is_new:
+        if autoincrement:
+            _ = new_dtypes.pop(primary_key, None)
+
+        ### TODO: see if this can be removed
+        if 'datetime' in pipe.columns and self.flavor == 'timescaledb':
+            primary_key = None
+
+        create_table_queries = get_create_table_queries(
+            new_dtypes,
+            pipe.target,
+            self.flavor,
+            schema=self.get_pipe_schema(pipe),
+            primary_key=primary_key,
+        )
+        create_success = all(
+            self.exec_queries(create_table_queries, break_on_error=True, rollback=True, debug=debug)
+        )
+        if not create_success:
+            warn(f"Failed to create '{pipe.target}'. Continuing...")
+
     stats = self.to_sql(unseen_df, **unseen_kw)
+
     if is_new:
         if not self.create_indices(pipe, debug=debug):
             warn(f"Failed to create indices for {pipe}. Continuing...")
@@ -1358,7 +1413,7 @@ def sync_pipe(
         ]
         update_queries = get_update_queries(
             pipe.target,
-            temp_target, 
+            temp_target,
             self,
             join_cols,
             upsert=upsert,
@@ -1960,7 +2015,7 @@ def get_sync_time(
     table = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
 
     dt_col = pipe.columns.get('datetime', None)
-    dt_type = pipe.dtypes.get(dt_col, 'datetime64[ns]')
+    dt_type = pipe.dtypes.get(dt_col, 'datetime64[ns, UTC]')
     if not dt_col:
         _dt = pipe.guess_datetime()
         dt = sql_item_name(_dt, self.flavor, None) if _dt else None
@@ -2366,7 +2421,7 @@ def get_pipe_columns_types(
     ----------
     pipe: mrsm.Pipe:
         The pipe to get the columns for.
-        
+
     Returns
     -------
     A dictionary of columns names (`str`) and types (`str`).
@@ -2381,17 +2436,18 @@ def get_pipe_columns_types(
     }
     >>> 
     """
+    from meerschaum.utils.sql import get_table_cols_types
     if not pipe.exists(debug=debug):
         return {}
 
-    if self.flavor == 'duckdb':
-        from meerschaum.utils.sql import get_table_cols_types
-        return get_table_cols_types(
-            pipe.target,
-            self,
-            flavor=self.flavor,
-            schema=self.get_pipe_schema(pipe),
-        )
+    #  if self.flavor not in ('oracle', 'mysql', 'mariadb'):
+    return get_table_cols_types(
+        pipe.target,
+        self,
+        flavor=self.flavor,
+        schema=self.get_pipe_schema(pipe),
+        debug=debug,
+    )
 
     table_columns = {}
     try:
@@ -2823,11 +2879,11 @@ def get_alter_columns_queries(
 
 
 def get_to_sql_dtype(
-        self,
-        pipe: 'mrsm.Pipe',
-        df: 'pd.DataFrame',
-        update_dtypes: bool = True,
-    ) -> Dict[str, 'sqlalchemy.sql.visitors.TraversibleType']:
+    self,
+    pipe: 'mrsm.Pipe',
+    df: 'pd.DataFrame',
+    update_dtypes: bool = True,
+) -> Dict[str, 'sqlalchemy.sql.visitors.TraversibleType']:
     """
     Given a pipe and DataFrame, return the `dtype` dictionary for `to_sql()`.
 
@@ -2947,7 +3003,7 @@ def deduplicate_pipe(
     duplicates_cte_name = sql_item_name('dups', self.flavor, None)
     duplicate_row_number_name = sql_item_name('dup_row_num', self.flavor, None)
     previous_row_number_name = sql_item_name('prev_row_num', self.flavor, None)
-    
+
     index_list_str = (
         sql_item_name(dt_col, self.flavor, None)
         if dt_col

meerschaum/connectors/sql/_sql.py

@@ -17,8 +17,8 @@ from meerschaum.utils.warnings import warn
 ### database flavors that can use bulk insert
 _bulk_flavors = {'postgresql', 'timescaledb', 'citus'}
 ### flavors that do not support chunks
-_disallow_chunks_flavors = ['duckdb']
-_max_chunks_flavors = {'sqlite': 1000,}
+_disallow_chunks_flavors = []
+_max_chunks_flavors = {'sqlite': 1000}
 SKIP_READ_TRANSACTION_FLAVORS: list[str] = ['mssql']
 
 
@@ -123,7 +123,8 @@ def read(
     if chunks is not None and chunks <= 0:
         return []
     from meerschaum.utils.sql import sql_item_name, truncate_item_name
-    from meerschaum.utils.dtypes.sql import NUMERIC_PRECISION_FLAVORS
+    from meerschaum.utils.dtypes import are_dtypes_equal, coerce_timezone
+    from meerschaum.utils.dtypes.sql import NUMERIC_PRECISION_FLAVORS, TIMEZONE_NAIVE_FLAVORS
     from meerschaum.utils.packages import attempt_import, import_pandas
     from meerschaum.utils.pool import get_pool
     from meerschaum.utils.dataframe import chunksize_to_npartitions, get_numeric_cols
@@ -139,6 +140,16 @@ def read(
     if is_dask:
         chunksize = None
     schema = schema or self.schema
+    utc_dt_cols = [
+        col
+        for col, typ in dtype.items()
+        if are_dtypes_equal(typ, 'datetime') and 'utc' in typ.lower()
+    ] if dtype else []
+
+    if dtype and utc_dt_cols and self.flavor in TIMEZONE_NAIVE_FLAVORS:
+        dtype = dtype.copy()
+        for col in utc_dt_cols:
+            dtype[col] = 'datetime64[ns]'
 
     pool = get_pool(workers=workers)
     sqlalchemy = attempt_import("sqlalchemy")
@@ -162,7 +173,6 @@ def read(
                 )
             chunksize = _max_chunks_flavors[self.flavor]
 
-    ### NOTE: A bug in duckdb_engine does not allow for chunks.
     if chunksize is not None and self.flavor in _disallow_chunks_flavors:
         chunksize = None
 
@@ -206,6 +216,9 @@ def read(
     chunk_list = []
     chunk_hook_results = []
     def _process_chunk(_chunk, _retry_on_failure: bool = True):
+        if self.flavor in TIMEZONE_NAIVE_FLAVORS:
+            for col in utc_dt_cols:
+                _chunk[col] = coerce_timezone(_chunk[col], strip_timezone=False)
         if not as_hook_results:
             chunk_list.append(_chunk)
         if chunk_hook is None:
@@ -765,7 +778,7 @@ def to_sql(
         DROP_IF_EXISTS_FLAVORS,
     )
     from meerschaum.utils.dataframe import get_json_cols, get_numeric_cols, get_uuid_cols
-    from meerschaum.utils.dtypes import are_dtypes_equal, quantize_decimal
+    from meerschaum.utils.dtypes import are_dtypes_equal, quantize_decimal, coerce_timezone
     from meerschaum.utils.dtypes.sql import (
         NUMERIC_PRECISION_FLAVORS,
         PD_TO_SQLALCHEMY_DTYPES_FLAVORS,
@@ -848,7 +861,6 @@ def to_sql(
             if not success:
                 warn(f"Unable to drop {name}")
 
-
         ### Enforce NVARCHAR(2000) as text instead of CLOB.
         dtype = to_sql_kw.get('dtype', {})
         for col, typ in df.dtypes.items():
@@ -858,11 +870,18 @@ def to_sql(
                 dtype[col] = sqlalchemy.types.INTEGER
         to_sql_kw['dtype'] = dtype
     elif self.flavor == 'mssql':
+        pass
+        ### TODO clean this up
+        #  dtype = to_sql_kw.get('dtype', {})
+        #  for col, typ in df.dtypes.items():
+            #  if are_dtypes_equal(str(typ), 'bool'):
+                #  dtype[col] = sqlalchemy.types.INTEGER
+        #  to_sql_kw['dtype'] = dtype
+    elif self.flavor == 'duckdb':
         dtype = to_sql_kw.get('dtype', {})
-        for col, typ in df.dtypes.items():
-            if are_dtypes_equal(str(typ), 'bool'):
-                dtype[col] = sqlalchemy.types.INTEGER
-        to_sql_kw['dtype'] = dtype
+        dt_cols = [col for col, typ in df.dtypes.items() if are_dtypes_equal(str(typ), 'datetime')]
+        for col in dt_cols:
+            df[col] = coerce_timezone(df[col], strip_utc=False)
 
     ### Check for JSON columns.
     if self.flavor not in json_flavors:

meerschaum/connectors/valkey/_pipes.py

@@ -706,7 +706,7 @@ def get_sync_time(
     """
     from meerschaum.utils.dtypes import are_dtypes_equal
     dt_col = pipe.columns.get('datetime', None)
-    dt_typ = pipe.dtypes.get(dt_col, 'datetime64[ns]')
+    dt_typ = pipe.dtypes.get(dt_col, 'datetime64[ns, UTC]')
     if not dt_col:
         return None
 

meerschaum/core/Pipe/__init__.py

@@ -153,6 +153,7 @@ class Pipe:
         dtypes: Optional[Dict[str, str]] = None,
         instance: Optional[Union[str, InstanceConnector]] = None,
         temporary: bool = False,
+        upsert: Optional[bool] = None,
         mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
         cache: bool = False,
         debug: bool = False,
@@ -201,6 +202,9 @@ class Pipe:
         instance: Optional[Union[str, InstanceConnector]], default None
             Alias for `mrsm_instance`. If `mrsm_instance` is supplied, this value is ignored.
 
+        upsert: Optional[bool], default None
+            If `True`, set `upsert` to `True` in the parameters.
+
         temporary: bool, default False
             If `True`, prevent instance tables (pipes, users, plugins) from being created.
 
@@ -268,7 +272,7 @@ class Pipe:
             or indexes
             or self._attributes.get('parameters', {}).get('indices', None)
             or self._attributes.get('parameters', {}).get('indexes', None)
-        ) or columns
+        )
         if isinstance(indices, dict):
             indices_key = (
                 'indexes'
@@ -292,6 +296,9 @@ class Pipe:
         elif dtypes is not None:
             warn(f"The provided dtypes are of invalid type '{type(dtypes)}'.")
 
+        if isinstance(upsert, bool):
+            self._attributes['parameters']['upsert'] = upsert
+
         ### NOTE: The parameters dictionary is {} by default.
         ###       A Pipe may be registered without parameters, then edited,
         ###       or a Pipe may be registered with parameters set in-memory first.
@@ -308,7 +315,6 @@ class Pipe:
 
         self._cache = cache and get_config('system', 'experimental', 'cache')
 
-
     @property
     def meta(self):
         """
@@ -321,7 +327,6 @@ class Pipe:
             'instance': self.instance_keys,
         }
 
-
     def keys(self) -> List[str]:
         """
         Return the ordered keys for this pipe.
@@ -332,7 +337,6 @@ class Pipe:
             if key != 'instance'
         }
 
-
     @property
     def instance_connector(self) -> Union[InstanceConnector, None]:
         """
@@ -369,7 +373,6 @@ class Pipe:
                 return None
         return self._connector
 
-
     @property
     def cache_connector(self) -> Union[meerschaum.connectors.sql.SQLConnector, None]:
         """
@@ -391,7 +394,6 @@ class Pipe:
 
         return self._cache_connector
 
-
     @property
     def cache_pipe(self) -> Union['meerschaum.Pipe', None]:
         """
@@ -433,11 +435,9 @@ class Pipe:
 
         return self._cache_pipe
 
-
     def __str__(self, ansi: bool=False):
         return pipe_repr(self, ansi=ansi)
 
-
     def __eq__(self, other):
         try:
             return (
@@ -489,7 +489,6 @@ class Pipe:
         """
         self.__init__(**_state)
 
-
     def __getitem__(self, key: str) -> Any:
         """
         Index the pipe's attributes.