meerschaum 2.6.0.dev1__py3-none-any.whl → 2.6.1__py3-none-any.whl

meerschaum/connectors/sql/_sql.py

@@ -17,7 +17,7 @@ from meerschaum.utils.warnings import warn
 ### database flavors that can use bulk insert
 _bulk_flavors = {'postgresql', 'timescaledb', 'citus'}
 ### flavors that do not support chunks
-_disallow_chunks_flavors = []
+_disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000}
 SKIP_READ_TRANSACTION_FLAVORS: list[str] = ['mssql']
 
@@ -134,7 +134,7 @@ def read(
     pd = import_pandas()
     dd = None
     is_dask = 'dask' in pd.__name__
-    pd = attempt_import('pandas')
+    pandas = attempt_import('pandas')
     is_dask = dd is not None
     npartitions = chunksize_to_npartitions(chunksize)
     if is_dask:
@@ -498,6 +498,8 @@ def exec(
     commit: Optional[bool] = None,
     close: Optional[bool] = None,
     with_connection: bool = False,
+    _connection=None,
+    _transaction=None,
     **kw: Any
 ) -> Union[
         sqlalchemy.engine.result.resultProxy,
@@ -508,7 +510,7 @@ def exec(
 ]:
     """
     Execute SQL code and return the `sqlalchemy` result, e.g. when calling stored procedures.
-    
+
     If inserting data, please use bind variables to avoid SQL injection!
 
     Parameters
@@ -565,15 +567,24 @@ def exec(
     if not hasattr(query, 'compile'):
         query = sqlalchemy.text(query)
 
-    connection = self.get_connection()
+    connection = _connection if _connection is not None else self.get_connection()
 
     try:
-        transaction = connection.begin() if _commit else None
-    except sqlalchemy.exc.InvalidRequestError:
+        transaction = (
+            _transaction
+            if _transaction is not None else (
+                connection.begin()
+                if _commit
+                else None
+            )
+        )
+    except sqlalchemy.exc.InvalidRequestError as e:
+        if _connection is not None or _transaction is not None:
+            raise e
         connection = self.get_connection(rebuild=True)
         transaction = connection.begin()
 
-    if transaction is not None and not transaction.is_active:
+    if transaction is not None and not transaction.is_active and _transaction is not None:
         connection = self.get_connection(rebuild=True)
         transaction = connection.begin() if _commit else None
 
@@ -708,6 +719,8 @@ def to_sql(
     debug: bool = False,
     as_tuple: bool = False,
     as_dict: bool = False,
+    _connection=None,
+    _transaction=None,
     **kw
 ) -> Union[bool, SuccessTuple]:
     """
@@ -782,6 +795,7 @@ def to_sql(
     from meerschaum.utils.dtypes.sql import (
         NUMERIC_PRECISION_FLAVORS,
         PD_TO_SQLALCHEMY_DTYPES_FLAVORS,
+        get_db_type_from_pd_type,
     )
     from meerschaum.connectors.sql._create_engine import flavor_configs
     from meerschaum.utils.packages import attempt_import, import_pandas
@@ -849,6 +863,8 @@ def to_sql(
         to_sql_kw.update({
             'parallel': True,
         })
+    elif _connection is not None:
+        to_sql_kw['con'] = _connection
 
     if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
     if self.flavor == 'oracle':
@@ -869,19 +885,24 @@ def to_sql(
             elif are_dtypes_equal(str(typ), 'int'):
                 dtype[col] = sqlalchemy.types.INTEGER
         to_sql_kw['dtype'] = dtype
-    elif self.flavor == 'mssql':
-        pass
-        ### TODO clean this up
-        #  dtype = to_sql_kw.get('dtype', {})
-        #  for col, typ in df.dtypes.items():
-            #  if are_dtypes_equal(str(typ), 'bool'):
-                #  dtype[col] = sqlalchemy.types.INTEGER
-        #  to_sql_kw['dtype'] = dtype
     elif self.flavor == 'duckdb':
         dtype = to_sql_kw.get('dtype', {})
         dt_cols = [col for col, typ in df.dtypes.items() if are_dtypes_equal(str(typ), 'datetime')]
         for col in dt_cols:
             df[col] = coerce_timezone(df[col], strip_utc=False)
+    elif self.flavor == 'mssql':
+        dtype = to_sql_kw.get('dtype', {})
+        dt_cols = [col for col, typ in df.dtypes.items() if are_dtypes_equal(str(typ), 'datetime')]
+        new_dtype = {}
+        for col in dt_cols:
+            if col in dtype:
+                continue
+            dt_typ = get_db_type_from_pd_type(str(df.dtypes[col]), self.flavor, as_sqlalchemy=True)
+            if col not in dtype:
+                new_dtype[col] = dt_typ
+
+        dtype.update(new_dtype)
+        to_sql_kw['dtype'] = dtype
 
     ### Check for JSON columns.
     if self.flavor not in json_flavors:
@@ -916,7 +937,7 @@ def to_sql(
 
     try:
         with warnings.catch_warnings():
-            warnings.filterwarnings('ignore', 'case sensitivity issues')
+            warnings.filterwarnings('ignore')
             df.to_sql(**to_sql_kw)
         success = True
     except Exception as e:

meerschaum/connectors/valkey/_ValkeyConnector.py

@@ -408,6 +408,7 @@ class ValkeyConnector(Connector):
         -------
         A list of dictionaries, where all keys and values are strings.
         """
+        from meerschaum.utils.dtypes import coerce_timezone
         table_name = self.quote_table(table)
         datetime_column_key = self.get_datetime_column_key(table)
         datetime_column = self.get(datetime_column_key)
@@ -424,10 +425,10 @@ class ValkeyConnector(Connector):
         dateutil_parser = mrsm.attempt_import('dateutil.parser')
 
         if isinstance(begin, str):
-            begin = dateutil_parser.parse(begin)
+            begin = coerce_timezone(dateutil_parser.parse(begin))
 
         if isinstance(end, str):
-            end = dateutil_parser.parse(end)
+            end = coerce_timezone(dateutil_parser.parse(end))
 
         begin_ts = (
             (

meerschaum/connectors/valkey/_pipes.py

@@ -409,6 +409,7 @@ def get_pipe_data(
         return None
 
     from meerschaum.utils.dataframe import query_df, parse_df_datetimes
+    from meerschaum.utils.dtypes import are_dtypes_equal
 
     valkey_dtypes = pipe.parameters.get('valkey', {}).get('dtypes', {})
     dt_col = pipe.columns.get('datetime', None)
@@ -442,13 +443,14 @@ def get_pipe_data(
     ignore_dt_cols = [
         col
         for col, dtype in pipe.dtypes.items()
-        if 'datetime' not in str(dtype)
+        if not are_dtypes_equal(str(dtype), 'datetime')
     ]
 
     df = parse_df_datetimes(
         docs,
         ignore_cols=ignore_dt_cols,
         chunksize=kwargs.get('chunksize', None),
+        strip_timezone=(pipe.tzinfo is None),
         debug=debug,
     )
     for col, typ in valkey_dtypes.items():
@@ -501,6 +503,7 @@ def sync_pipe(
     -------
     A `SuccessTuple` indicating success.
     """
+    from meerschaum.utils.dtypes import are_dtypes_equal
     dt_col = pipe.columns.get('datetime', None)
     indices = [col for col in pipe.columns.values() if col]
     table_name = self.quote_table(pipe.target)
@@ -508,6 +511,7 @@ def sync_pipe(
     if is_dask:
         df = df.compute()
     upsert = pipe.parameters.get('upsert', False)
+    static = pipe.parameters.get('static', False)
 
     def _serialize_indices_docs(_docs):
         return [
@@ -526,7 +530,11 @@ def sync_pipe(
 
     valkey_dtypes = pipe.parameters.get('valkey', {}).get('dtypes', {})
     new_dtypes = {
-        str(key): str(val)
+        str(key): (
+            str(val)
+            if not are_dtypes_equal(str(val), 'datetime')
+            else 'datetime64[ns, UTC]'
+        )
         for key, val in df.dtypes.items()
         if str(key) not in valkey_dtypes
     }
@@ -539,7 +547,7 @@ def sync_pipe(
                 new_dtypes[col] = 'string'
                 df[col] = df[col].astype('string')
 
-    if new_dtypes:
+    if new_dtypes and not static:
         valkey_dtypes.update(new_dtypes)
         if 'valkey' not in pipe.parameters:
             pipe.parameters['valkey'] = {}
@@ -625,7 +633,7 @@ def get_pipe_columns_types(
 
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     return {
-        col: get_db_type_from_pd_type(typ)
+        col: get_db_type_from_pd_type(typ, flavor='postgresql')
         for col, typ in pipe.parameters.get('valkey', {}).get('dtypes', {}).items()
     }
 
@@ -733,7 +741,7 @@ def get_sync_time(
         return (
             int(dt_val)
             if are_dtypes_equal(dt_typ, 'int')
-            else dateutil_parser.parse(str(dt_val)).replace(tzinfo=None)
+            else dateutil_parser.parse(str(dt_val))
         )
     except Exception as e:
         warn(f"Failed to parse sync time for {pipe}:\n{e}")

meerschaum/core/Pipe/__init__.py

@@ -92,6 +92,7 @@ class Pipe:
         _get_data_as_iterator,
         get_chunk_interval,
         get_chunk_bounds,
+        parse_date_bounds,
     )
     from ._register import register
     from ._attributes import (
@@ -101,8 +102,13 @@ class Pipe:
         indices,
         indexes,
         dtypes,
+        autoincrement,
+        upsert,
+        static,
+        tzinfo,
         get_columns,
         get_columns_types,
+        get_columns_indices,
         get_indices,
         tags,
         get_id,
@@ -154,6 +160,8 @@ class Pipe:
         instance: Optional[Union[str, InstanceConnector]] = None,
         temporary: bool = False,
         upsert: Optional[bool] = None,
+        autoincrement: Optional[bool] = None,
+        static: Optional[bool] = None,
         mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
         cache: bool = False,
         debug: bool = False,
@@ -205,6 +213,12 @@ class Pipe:
         upsert: Optional[bool], default None
             If `True`, set `upsert` to `True` in the parameters.
 
+        autoincrement: Optional[bool], default None
+            If `True`, set `autoincrement` in the parameters.
+
+        static: Optional[bool], default None
+            If `True`, set `static` in the parameters.
+
         temporary: bool, default False
             If `True`, prevent instance tables (pipes, users, plugins) from being created.
 
@@ -299,6 +313,12 @@ class Pipe:
         if isinstance(upsert, bool):
             self._attributes['parameters']['upsert'] = upsert
 
+        if isinstance(autoincrement, bool):
+            self._attributes['parameters']['autoincrement'] = autoincrement
+
+        if isinstance(static, bool):
+            self._attributes['parameters']['static'] = static
+
         ### NOTE: The parameters dictionary is {} by default.
         ###       A Pipe may be registered without parameters, then edited,
         ###       or a Pipe may be registered with parameters set in-memory first.

meerschaum/core/Pipe/_attributes.py

@@ -7,9 +7,14 @@ Fetch and manipulate Pipes' attributes
 """
 
 from __future__ import annotations
+
+from datetime import timezone
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import Tuple, Dict, SuccessTuple, Any, Union, Optional, List
 from meerschaum.utils.warnings import warn
 
+
 @property
 def attributes(self) -> Dict[str, Any]:
     """
@@ -50,6 +55,13 @@ def parameters(self) -> Optional[Dict[str, Any]]:
     """
     if 'parameters' not in self.attributes:
         self.attributes['parameters'] = {}
+    _parameters = self.attributes['parameters']
+    dt_col = _parameters.get('columns', {}).get('datetime', None)
+    dt_typ = _parameters.get('dtypes', {}).get(dt_col, None) if dt_col else None
+    if dt_col and not dt_typ:
+        if 'dtypes' not in _parameters:
+            self.attributes['parameters']['dtypes'] = {}
+        self.attributes['parameters']['dtypes'][dt_col] = 'datetime'
     return self.attributes['parameters']
 
 
@@ -84,7 +96,7 @@ def columns(self, _columns: Union[Dict[str, str], List[str]]) -> None:
     """
     if isinstance(_columns, (list, tuple)):
         _columns = {col: col for col in _columns}
-    if not isinstance(columns, dict):
+    if not isinstance(_columns, dict):
         warn(f"{self}.columns must be a dictionary, received {type(_columns)}.")
         return
     self.parameters['columns'] = _columns
@@ -108,7 +120,7 @@ def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
     if not isinstance(_indices, dict):
         _indices = {}
         self.parameters[indices_key] = _indices
-    unique_cols = (
+    unique_cols = list(set((
         [dt_col]
         if dt_col
         else []
@@ -116,7 +128,7 @@ def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
         col
         for col_ix, col in _columns.items()
         if col_ix != 'datetime'
-    ]
+    ]))
     return {
         **({'unique': unique_cols} if len(unique_cols) > 1 else {}),
         **_columns,
@@ -203,6 +215,80 @@ def dtypes(self, _dtypes: Dict[str, Any]) -> None:
     self.parameters['dtypes'] = _dtypes
 
 
+@property
+def upsert(self) -> bool:
+    """
+    Return whether `upsert` is set for the pipe.
+    """
+    if 'upsert' not in self.parameters:
+        self.parameters['upsert'] = False
+    return self.parameters['upsert']
+
+
+@upsert.setter
+def upsert(self, _upsert: bool) -> None:
+    """
+    Set the `upsert` parameter for the pipe.
+    """
+    self.parameters['upsert'] = _upsert
+
+
+@property
+def static(self) -> bool:
+    """
+    Return whether `static` is set for the pipe.
+    """
+    if 'static' not in self.parameters:
+        self.parameters['static'] = False
+    return self.parameters['static']
+
+
+@static.setter
+def static(self, _static: bool) -> None:
+    """
+    Set the `static` parameter for the pipe.
+    """
+    self.parameters['static'] = _static
+
+
+@property
+def autoincrement(self) -> bool:
+    """
+    Return the `autoincrement` parameter for the pipe.
+    """
+    if 'autoincrement' not in self.parameters:
+        self.parameters['autoincrement'] = False
+
+    return self.parameters['autoincrement']
+
+
+@autoincrement.setter
+def autoincrement(self, _autoincrement: bool) -> None:
+    """
+    Set the `autoincrement` parameter for the pipe.
+    """
+    self.parameters['autoincrement'] = _autoincrement
+
+
+@property
+def tzinfo(self) -> Union[None, timezone]:
+    """
+    Return `timezone.utc` if the pipe is timezone-aware.
+    """
+    dt_col = self.columns.get('datetime', None)
+    if not dt_col:
+        return None
+
+    dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
+    if 'utc' in dt_typ.lower() or dt_typ == 'datetime':
+        return timezone.utc
+
+    if dt_typ == 'datetime64[ns]':
+        return None
+
+    return None
+
+
 def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
     """
     Check if the requested columns are defined.
@@ -248,12 +334,19 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
     return tuple(col_names)
 
 
-def get_columns_types(self, debug: bool = False) -> Union[Dict[str, str], None]:
+def get_columns_types(
+    self,
+    refresh: bool = False,
+    debug: bool = False,
+) -> Union[Dict[str, str], None]:
     """
     Get a dictionary of a pipe's column names and their types.
 
     Parameters
     ----------
+    refresh: bool, default False
+        If `True`, invalidate the cache and fetch directly from the instance connector.
+
     debug: bool, default False:
         Verbosity toggle.
 
@@ -265,17 +358,91 @@ def get_columns_types(self, debug: bool = False) -> Union[Dict[str, str], None]:
     --------
     >>> pipe.get_columns_types()
     {
-      'dt': 'TIMESTAMP WITHOUT TIMEZONE',
+      'dt': 'TIMESTAMP WITH TIMEZONE',
       'id': 'BIGINT',
       'val': 'DOUBLE PRECISION',
     }
     >>>
     """
-    from meerschaum.utils.venv import Venv
+    import time
     from meerschaum.connectors import get_connector_plugin
+    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum.utils.warnings import dprint
 
-    with Venv(get_connector_plugin(self.instance_connector)):
-        return self.instance_connector.get_pipe_columns_types(self, debug=debug)
+    now = time.perf_counter()
+    cache_seconds = STATIC_CONFIG['pipes']['static_schema_cache_seconds']
+    static = self.parameters.get('static', False)
+    if not static:
+        refresh = True
+    if refresh:
+        _ = self.__dict__.pop('_columns_types_timestamp', None)
+        _ = self.__dict__.pop('_columns_types', None)
+    _columns_types = self.__dict__.get('_columns_types', None)
+    if _columns_types:
+        columns_types_timestamp = self.__dict__.get('_columns_types_timestamp', None)
+        if columns_types_timestamp is not None:
+            delta = now - columns_types_timestamp
+            if delta < cache_seconds:
+                if debug:
+                    dprint(
+                        f"Returning cached `columns_types` for {self} "
+                        f"({round(delta, 2)} seconds old)."
+                    )
+                return _columns_types
+
+    with mrsm.Venv(get_connector_plugin(self.instance_connector)):
+        _columns_types = (
+            self.instance_connector.get_pipe_columns_types(self, debug=debug)
+            if hasattr(self.instance_connector, 'get_pipe_columns_types')
+            else None
+        )
+
+    self.__dict__['_columns_types'] = _columns_types
+    self.__dict__['_columns_types_timestamp'] = now
+    return _columns_types or {}
+
+
+def get_columns_indices(
+    self,
+    debug: bool = False,
+    refresh: bool = False,
+) -> Dict[str, List[Dict[str, str]]]:
+    """
+    Return a dictionary mapping columns to index information.
+    """
+    import time
+    from meerschaum.connectors import get_connector_plugin
+    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum.utils.warnings import dprint
+
+    now = time.perf_counter()
+    exists_timeout_seconds = STATIC_CONFIG['pipes']['exists_timeout_seconds']
+    if refresh:
+        _ = self.__dict__.pop('_columns_indices_timestamp', None)
+        _ = self.__dict__.pop('_columns_indices', None)
+    _columns_indices = self.__dict__.get('_columns_indices', None)
+    if _columns_indices:
+        columns_indices_timestamp = self.__dict__.get('_columns_indices_timestamp', None)
+        if columns_indices_timestamp is not None:
+            delta = now - columns_indices_timestamp
+            if delta < exists_timeout_seconds:
+                if debug:
+                    dprint(
+                        f"Returning cached `columns_indices` for {self} "
+                        f"({round(delta, 2)} seconds old)."
+                    )
+                return _columns_indices
+
+    with mrsm.Venv(get_connector_plugin(self.instance_connector)):
+        _columns_indices = (
+            self.instance_connector.get_pipe_columns_indices(self, debug=debug)
+            if hasattr(self.instance_connector, 'get_pipe_columns_indices')
+            else None
+        )
+
+    self.__dict__['_columns_indices'] = _columns_indices
+    self.__dict__['_columns_indices_timestamp'] = now
+    return _columns_indices or {}
 
 
 def get_id(self, **kw: Any) -> Union[int, None]:
@@ -289,7 +456,10 @@ def get_id(self, **kw: Any) -> Union[int, None]:
     from meerschaum.connectors import get_connector_plugin
 
     with Venv(get_connector_plugin(self.instance_connector)):
-        return self.instance_connector.get_pipe_id(self, **kw)
+        if hasattr(self.instance_connector, 'get_pipe_id'):
+            return self.instance_connector.get_pipe_id(self, **kw)
+
+    return None
 
 
 @property

meerschaum/core/Pipe/_clear.py

@@ -58,12 +58,14 @@ def clear(
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     if self.cache_pipe is not None:
         success, msg = self.cache_pipe.clear(
-            begin = begin,
-            end = end,
-            params = params,
-            debug = debug,
+            begin=begin,
+            end=end,
+            params=params,
+            debug=debug,
             **kwargs
         )
         if not success:
@@ -72,9 +74,9 @@ def clear(
     with Venv(get_connector_plugin(self.instance_connector)):
         return self.instance_connector.clear_pipe(
             self,
-            begin = begin,
-            end = end,
-            params = params,
-            debug = debug,
+            begin=begin,
+            end=end,
+            params=params,
+            debug=debug,
             **kwargs
         )

meerschaum/core/Pipe/_copy.py

@@ -55,6 +55,8 @@ def copy_to(
     if str(instance_keys) == self.instance_keys:
         return False, f"Cannot copy {self} to instance '{instance_keys}'."
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     new_pipe = mrsm.Pipe(
         self.connector_keys,
         self.metric_key,